From 95a8da86e4e414078027d4eea51400a5dd6fb11a Mon Sep 17 00:00:00 2001 From: TermiNexus Date: Mon, 26 Oct 2026 21:04:10 +0800 Subject: [PATCH] =?UTF-8?q?feat(lsp):=20Gate=209=20=E8=AF=AD=E4=B9=89?= =?UTF-8?q?=E9=AB=98=E4=BA=AE=E4=BF=9D=E7=9C=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit define 宏函数与调用点使用 macro token；常量保持 variable+readonly；多行字符串按行拆分；修正符号/出现点合并顺序导致整行 token 丢失；大文档诊断 scratch 按源大小扩容。 --- App/lib/lsp/LspDiagnostics.vp | 2 +- App/lib/lsp/LspDocuments.vp | 19 ++++++ App/lib/lsp/LspSemantic.vp | 92 +++++++++++++++++++------ Tools/lsp/fixtures/gate9.vp | 17 +++++ Tools/lsp/probe.py | 123 ++++++++++++++++++++++++++++++++++ 5 files changed, 231 insertions(+), 22 deletions(-) create mode 100644 Tools/lsp/fixtures/gate9.vp diff --git a/App/lib/lsp/LspDiagnostics.vp b/App/lib/lsp/LspDiagnostics.vp index fb839b3..9800462 100644 --- a/App/lib/lsp/LspDiagnostics.vp +++ b/App/lib/lsp/LspDiagnostics.vp @@ -96,7 +96,7 @@ def Publish(pool: memhub.MemManager*, tr: LspTransport.LspTransport*, doc: LspDocuments.LspDocument* = docs.Find(uri) if doc is None: return 0 - scratch: LspPools.LspPool* = docs.Scratch() + scratch: LspPools.LspPool* = docs.ScratchFor(sizet(string.strlen(doc.text))) if scratch is None: LspTransport.StderrWrite("viperc-lsp: diagnostics scratch pool is not available\n") return -1 diff --git a/App/lib/lsp/LspDocuments.vp b/App/lib/lsp/LspDocuments.vp index 2401c43..b2b17b2 100644 --- a/App/lib/lsp/LspDocuments.vp +++ b/App/lib/lsp/LspDocuments.vp @@ -9,7 +9,11 @@ define LSP_DOC_POOL_SIZE = 4194304 # Parse scratch arena shared by diagnostics: reset before each publish so ASTs and token lists do # not accumulate across edits. It lives in the per-documents object, whose owner is the permanent # pool, because the message pool is reset after every request. +# The arena grows with the largest published document: a 4 MiB arena is exhausted by a few hundred +# KiB of source, and allocation failure aborts the parse. define LSP_SCRATCH_POOL_SIZE = 4194304 +define LSP_SCRATCH_BYTES_PER_SOURCE = 128 +define LSP_SCRATCH_POOL_MAX = 134217728 class LspDocument: @@ -42,6 +46,21 @@ class LspDocuments: def Scratch(self) -> LspPools.LspPool*: return self.scratch + def ScratchFor(self, source_len: sizet) -> LspPools.LspPool*: + """Return a scratch arena large enough to parse a document of source_len bytes.""" + want: sizet = source_len * sizet(LSP_SCRATCH_BYTES_PER_SOURCE) + if want < sizet(LSP_SCRATCH_POOL_SIZE): + want = sizet(LSP_SCRATCH_POOL_SIZE) + if want > sizet(LSP_SCRATCH_POOL_MAX): + want = sizet(LSP_SCRATCH_POOL_MAX) + if self.scratch is not None and self.scratch.size >= want: + return self.scratch + if self.scratch is not None: + self.scratch.Destroy() + self.scratch = None + self.scratch = LspPools.Create(self.pool, want) + return self.scratch + def Find(self, uri: str) -> LspDocument*: return self._Find(uri) diff --git a/App/lib/lsp/LspSemantic.vp b/App/lib/lsp/LspSemantic.vp index c86b21b..cdf056a 100644 --- a/App/lib/lsp/LspSemantic.vp +++ b/App/lib/lsp/LspSemantic.vp @@ -12,7 +12,8 @@ import lib.lsp.LspTransport as LspTransport import lib.lsp.LspRpc as LspRpc -define LSP_MAX_SEMANTIC_TOKENS = 10000 +# Stays below 65535 so encoded deltas remain safe for clients that store token fields in 16 bits. +define LSP_MAX_SEMANTIC_TOKENS = 60000 # Indices must match the legend built by Provider(). define ST_NAMESPACE = 0 @@ -28,6 +29,7 @@ define ST_STRING = 9 define ST_NUMBER = 10 define ST_OPERATOR = 11 define ST_DECORATOR = 12 +define ST_MACRO = 13 class LspSemToken: @@ -79,6 +81,7 @@ def Provider(pool: memhub.MemManager*) -> json.JsonValue*: json.array_append(pool, types, json.string_val(pool, "number")) json.array_append(pool, types, json.string_val(pool, "operator")) json.array_append(pool, types, json.string_val(pool, "decorator")) + json.array_append(pool, types, json.string_val(pool, "macro")) json.array_append(pool, mods, json.string_val(pool, "declaration")) json.array_append(pool, mods, json.string_val(pool, "readonly")) json.object_set(pool, legend, "tokenTypes", types) @@ -89,6 +92,15 @@ def Provider(pool: memhub.MemManager*) -> json.JsonValue*: return provider +def _IsDefineMacro(s: LspSymbols.LspSymbol*) -> int: + if s is None or s.decl is None: + return 0 + if s.decl.kind() != ast.ASTKind.FunctionDef: + return 0 + fd: ast.FunctionDef* = (ast.FunctionDef*)(s.decl) + return 1 if (fd.flags & ast.FLAG_IS_DEFINE) != 0 else 0 + + def _TypeFor(s: LspSymbols.LspSymbol*) -> int: if s is None: return ST_VARIABLE @@ -97,8 +109,12 @@ def _TypeFor(s: LspSymbols.LspSymbol*) -> int: if s.kind == LspSymbols.LSP_KIND_CLASS: return ST_CLASS if s.kind == LspSymbols.LSP_KIND_FUNCTION: + if _IsDefineMacro(s) != 0: + return ST_MACRO return ST_FUNCTION if s.kind == LspSymbols.LSP_KIND_METHOD: + if _IsDefineMacro(s) != 0: + return ST_MACRO return ST_METHOD if s.kind == LspSymbols.LSP_KIND_FIELD: return ST_PROPERTY @@ -174,37 +190,43 @@ def _AstStream(pool: memhub.MemManager*, occs: list[LspSymbols.LspOccurrence*]* = a.occs si: int = 0 oi: int = 0 - while si < int(len(syms)) or oi < int(len(occs)): + n: int = int(len(syms)) + m: int = int(len(occs)) + while si < n or oi < m: + # Both inputs are in source order. Advance past declarations/occurrences that cannot + # produce a token so a skipped entry never holds back the other stream. sym_t: LspSemToken* = None - if si < int(len(syms)): + while si < n: s: LspSymbols.LspSymbol* = syms.get(sizet(si)) - sym_t = _TokenForSymbol(pool, s) - if sym_t is not None and s is not None: - if _SymbolTokenValid(pool, a, sym_t, s.name) == 0: - sym_t.typ = -1 + si += 1 + t: LspSemToken* = _TokenForSymbol(pool, s) + if t is not None and s is not None: + if _SymbolTokenValid(pool, a, t, s.name) == 0: + t.typ = -1 + if t is not None and t.typ >= 0: + sym_t = t + break occ_t: LspSemToken* = None - if oi < int(len(occs)): - occ_t = _TokenForOcc(pool, occs.get(sizet(oi))) + while oi < m: + t: LspSemToken* = _TokenForOcc(pool, occs.get(sizet(oi))) + oi += 1 + if t is not None and t.typ >= 0: + occ_t = t + break if sym_t is None and occ_t is None: - if sym_t is None: - si += 1 - if occ_t is None: - oi += 1 - continue + break if sym_t is None: out.append(occ_t) - oi += 1 continue if occ_t is None: out.append(sym_t) - si += 1 continue if _Before(sym_t, occ_t) != 0: out.append(sym_t) - si += 1 + oi -= 1 else: out.append(occ_t) - oi += 1 + si -= 1 return out @@ -235,6 +257,32 @@ def _ScanComments(pool: memhub.MemManager*, doc: LspDocuments.LspDocument*, off += 1 +def _AddSpanning(pool: memhub.MemManager*, doc: LspDocuments.LspDocument*, + sl: int, sc: int, el: int, ec: int, typ: int, + out: list[LspSemToken*]*): + """Split a multi-line literal/comment into one token per line. + + LSP semantic tokens encode positions per line; a token with a line span would be interpreted + with a single-line length and paint unrelated characters. + """ + count: int = doc.index.Count() + line: int = sl + while line <= el and line < count: + line_len: int = LspSource.ByteColToUtf16( + doc, line, LspSource.LineByteLength(doc, line)) + start: int = 0 + end: int = line_len + if line == sl: + start = sc + if line == el: + end = ec + if end > start: + t: LspSemToken* = _NewTok(pool, line, start, line, end, typ) + if t is not None: + out.append(t) + line += 1 + + def _LexStream(pool: memhub.MemManager*, doc: LspDocuments.LspDocument*) -> list[LspSemToken*]*: out: list[LspSemToken*]* = list[LspSemToken*](pool, 128) @@ -280,11 +328,13 @@ def _LexStream(pool: memhub.MemManager*, end_col = tok.col_offset + int(string.strlen(tok.str_val)) if end_col < tok.col_offset: end_col = tok.col_offset - if typ >= 0 and end_col > tok.col_offset: + if typ >= 0 and (end_line0 > line0 or end_col > tok.col_offset): sc: int = LspSource.ByteColToUtf16(doc, line0, tok.col_offset) ec: int = LspSource.ByteColToUtf16(doc, end_line0, end_col) - if ec > sc: - t: LspSemToken* = _NewTok(pool, line0, sc, end_line0, ec, typ) + if end_line0 > line0: + _AddSpanning(pool, doc, line0, sc, end_line0, ec, typ, out) + elif ec > sc: + t: LspSemToken* = _NewTok(pool, line0, sc, line0, ec, typ) if t is not None: out.append(t) new_off: int = int(doc.index.LineStart(end_line0)) + end_col diff --git a/Tools/lsp/fixtures/gate9.vp b/Tools/lsp/fixtures/gate9.vp new file mode 100644 index 0000000..e383484 --- /dev/null +++ b/Tools/lsp/fixtures/gate9.vp @@ -0,0 +1,17 @@ +define MAX_VALUE = 100 +define ADD(a, b): return a + b + + +def use(x: int) -> int: + s: str = """alpha +beta""" + n: int = 42 + # gate9 comment + return ADD(x, MAX_VALUE) + n + + +class Box: + value: int + + def get(self) -> int: + return self.value diff --git a/Tools/lsp/probe.py b/Tools/lsp/probe.py index 5d94789..72dc835 100644 --- a/Tools/lsp/probe.py +++ b/Tools/lsp/probe.py @@ -1235,6 +1235,126 @@ def gate8(exe: str) -> int: return 0 +def gate9(exe: str) -> int: + root = Path(__file__).resolve().parents[2] + source = root / "Tools" / "lsp" / "fixtures" / "gate9.vp" + uri = source.as_uri() + text = source.read_text(encoding="utf-8") + lines = text.split("\n") + + client = Client(exe) + response = client.request( + "initialize", + {"processId": os.getpid(), "rootUri": root.as_uri(), "capabilities": {}}, + ) + capabilities = response.get("result", {}).get("capabilities", {}) + sem = capabilities.get("semanticTokensProvider") + if not isinstance(sem, dict): + raise AssertionError(f"missing semanticTokensProvider: {response}") + legend = sem.get("legend", {}) + types = legend.get("tokenTypes", []) + mods = legend.get("tokenModifiers", []) + for required in ("macro", "variable", "keyword", "string", "number", "comment"): + if required not in types: + raise AssertionError(f"semantic legend misses {required}: {legend}") + if "readonly" not in mods: + raise AssertionError(f"semantic legend misses readonly: {legend}") + macro_idx = types.index("macro") + variable_idx = types.index("variable") + keyword_idx = types.index("keyword") + comment_idx = types.index("comment") + string_idx = types.index("string") + number_idx = types.index("number") + readonly_bit = 1 << mods.index("readonly") + mod_mask = (1 << len(mods)) - 1 + + client.notify("initialized", {}) + client.notify( + "textDocument/didOpen", + {"textDocument": {"uri": uri, "languageId": "viper", "version": 1, "text": text}}, + ) + client.wait_notification("textDocument/publishDiagnostics") + + semantic = client.request( + "textDocument/semanticTokens/full", {"textDocument": {"uri": uri}} + ).get("result") + if not isinstance(semantic, dict): + raise AssertionError(f"semanticTokens/full failed: {semantic}") + tokens = _decode_tokens(semantic.get("data", [])) + if not tokens: + raise AssertionError("semantic tokens are empty") + + last_line = -1 + last_end = -1 + for token in tokens: + if token["length"] <= 0: + raise AssertionError(f"zero-length token: {token}") + if token["line"] < 0 or token["line"] >= len(lines): + raise AssertionError(f"token line is out of range: {token}") + line_len = len(lines[token["line"]]) + if token["character"] < 0 or token["character"] + token["length"] > line_len: + raise AssertionError(f"token range is out of the line: {token}") + if token["type"] < 0 or token["type"] >= len(types): + raise AssertionError(f"token type is outside the legend: {token}") + if (token["modifiers"] & ~mod_mask) != 0: + raise AssertionError(f"token modifiers are outside the legend: {token}") + if token["type"] == keyword_idx: + raise AssertionError(f"keywords must be left to TextMate: {token}") + if token["line"] == last_line: + if token["character"] < last_end: + raise AssertionError(f"overlapping tokens on line {last_line}: {token}") + last_end = token["character"] + token["length"] + else: + last_line = token["line"] + last_end = token["character"] + token["length"] + + add_decl = _position(text, "define ADD") + add_decl["character"] += len("define ") + if not _find_token(tokens, add_decl["line"], add_decl["character"], macro_idx): + raise AssertionError(f"define macro declaration is not a macro token: {add_decl}") + add_call = _position(text, "ADD(x") + if not _find_token(tokens, add_call["line"], add_call["character"], macro_idx): + raise AssertionError(f"define macro call is not a macro token: {add_call}") + + max_decl = _position(text, "MAX_VALUE") + if not _find_token_mods(tokens, max_decl["line"], max_decl["character"], + variable_idx, readonly_bit): + raise AssertionError(f"define constant is not readonly: {max_decl}") + max_call = _position(text, "MAX_VALUE", 1) + if not _find_token_mods(tokens, max_call["line"], max_call["character"], + variable_idx, readonly_bit): + raise AssertionError(f"define constant use is not readonly: {max_call}") + + for keyword in ("define", "def", "class", "return"): + pos = _position(text, keyword) + if _find_token(tokens, pos["line"], pos["character"], keyword_idx): + raise AssertionError(f"keyword {keyword} must not be a semantic token") + + comment_pos = _position(text, "# gate9 comment") + if not _find_token(tokens, comment_pos["line"], comment_pos["character"], comment_idx): + raise AssertionError(f"missing comment token: {comment_pos}") + first_string = _position(text, '"""alpha') + if not _find_token(tokens, first_string["line"], first_string["character"], string_idx): + raise AssertionError(f"missing first line of the multi-line string: {first_string}") + second_string = _position(text, "beta") + if not _find_token(tokens, second_string["line"], second_string["character"], string_idx): + raise AssertionError(f"missing second line of the multi-line string: {second_string}") + number_pos = _position(text, "42") + if not _find_token(tokens, number_pos["line"], number_pos["character"], number_idx): + raise AssertionError(f"missing number token: {number_pos}") + + client.notify("textDocument/didClose", {"textDocument": {"uri": uri}}) + client.wait_notification("textDocument/publishDiagnostics") + client.request("shutdown") + code = client.close() + if code != 0: + raise AssertionError(f"exit code is {code}, expected 0") + print("semantic-legend: OK") + print("semantic-audit: OK") + print("semantic-macro: OK") + return 0 + + def main() -> int: parser = argparse.ArgumentParser(description="ViperC LSP probe") parser.add_argument("--selfcheck", action="store_true", help="run pure-Python framing checks") @@ -1245,6 +1365,7 @@ def main() -> int: parser.add_argument("--gate5", action="store_true", help="run rename/semantic/code-action checks") parser.add_argument("--gate7", action="store_true", help="run the product gate checks") parser.add_argument("--gate8", action="store_true", help="run cross-file references/rename checks") + parser.add_argument("--gate9", action="store_true", help="run semantic highlighting audits") parser.add_argument("--exe", help="path to viperc.exe") args = parser.parse_args() @@ -1256,6 +1377,8 @@ def main() -> int: if not os.path.isfile(exe): print(f"probe: executable not found: {exe}", file=sys.stderr) return 2 + if args.gate9: + return gate9(exe) if args.gate8: return gate8(exe) if args.gate7: