feat(lsp): Gate 9 语义高亮保真

define 宏函数与调用点使用 macro token；常量保持 variable+readonly；多行字符串按行拆分；修正符号/出现点合并顺序导致整行 token 丢失；大文档诊断 scratch 按源大小扩容。

App/lib/lsp/LspDiagnostics.vp

@@ -96,7 +96,7 @@ def Publish(pool: memhub.MemManager*, tr: LspTransport.LspTransport*,
     doc: LspDocuments.LspDocument* = docs.Find(uri)
     if doc is None:
         return 0
-    scratch: LspPools.LspPool* = docs.Scratch()
+    scratch: LspPools.LspPool* = docs.ScratchFor(sizet(string.strlen(doc.text)))
     if scratch is None:
         LspTransport.StderrWrite("viperc-lsp: diagnostics scratch pool is not available\n")
         return -1

App/lib/lsp/LspDocuments.vp

@@ -9,7 +9,11 @@ define LSP_DOC_POOL_SIZE = 4194304
 # Parse scratch arena shared by diagnostics: reset before each publish so ASTs and token lists do
 # not accumulate across edits. It lives in the per-documents object, whose owner is the permanent
 # pool, because the message pool is reset after every request.
+# The arena grows with the largest published document: a 4 MiB arena is exhausted by a few hundred
+# KiB of source, and allocation failure aborts the parse.
 define LSP_SCRATCH_POOL_SIZE = 4194304
+define LSP_SCRATCH_BYTES_PER_SOURCE = 128
+define LSP_SCRATCH_POOL_MAX = 134217728
 
 
 class LspDocument:
@@ -42,6 +46,21 @@ class LspDocuments:
     def Scratch(self) -> LspPools.LspPool*:
         return self.scratch
 
+    def ScratchFor(self, source_len: sizet) -> LspPools.LspPool*:
+        """Return a scratch arena large enough to parse a document of source_len bytes."""
+        want: sizet = source_len * sizet(LSP_SCRATCH_BYTES_PER_SOURCE)
+        if want < sizet(LSP_SCRATCH_POOL_SIZE):
+            want = sizet(LSP_SCRATCH_POOL_SIZE)
+        if want > sizet(LSP_SCRATCH_POOL_MAX):
+            want = sizet(LSP_SCRATCH_POOL_MAX)
+        if self.scratch is not None and self.scratch.size >= want:
+            return self.scratch
+        if self.scratch is not None:
+            self.scratch.Destroy()
+            self.scratch = None
+        self.scratch = LspPools.Create(self.pool, want)
+        return self.scratch
+
     def Find(self, uri: str) -> LspDocument*:
         return self._Find(uri)
 

App/lib/lsp/LspSemantic.vp

@@ -12,7 +12,8 @@ import lib.lsp.LspTransport as LspTransport
 import lib.lsp.LspRpc as LspRpc
 
 
-define LSP_MAX_SEMANTIC_TOKENS = 10000
+# Stays below 65535 so encoded deltas remain safe for clients that store token fields in 16 bits.
+define LSP_MAX_SEMANTIC_TOKENS = 60000
 
 # Indices must match the legend built by Provider().
 define ST_NAMESPACE = 0
@@ -28,6 +29,7 @@ define ST_STRING = 9
 define ST_NUMBER = 10
 define ST_OPERATOR = 11
 define ST_DECORATOR = 12
+define ST_MACRO = 13
 
 
 class LspSemToken:
@@ -79,6 +81,7 @@ def Provider(pool: memhub.MemManager*) -> json.JsonValue*:
     json.array_append(pool, types, json.string_val(pool, "number"))
     json.array_append(pool, types, json.string_val(pool, "operator"))
     json.array_append(pool, types, json.string_val(pool, "decorator"))
+    json.array_append(pool, types, json.string_val(pool, "macro"))
     json.array_append(pool, mods, json.string_val(pool, "declaration"))
     json.array_append(pool, mods, json.string_val(pool, "readonly"))
     json.object_set(pool, legend, "tokenTypes", types)
@@ -89,6 +92,15 @@ def Provider(pool: memhub.MemManager*) -> json.JsonValue*:
     return provider
 
 
+def _IsDefineMacro(s: LspSymbols.LspSymbol*) -> int:
+    if s is None or s.decl is None:
+        return 0
+    if s.decl.kind() != ast.ASTKind.FunctionDef:
+        return 0
+    fd: ast.FunctionDef* = (ast.FunctionDef*)(s.decl)
+    return 1 if (fd.flags & ast.FLAG_IS_DEFINE) != 0 else 0
+
+
 def _TypeFor(s: LspSymbols.LspSymbol*) -> int:
     if s is None:
         return ST_VARIABLE
@@ -97,8 +109,12 @@ def _TypeFor(s: LspSymbols.LspSymbol*) -> int:
     if s.kind == LspSymbols.LSP_KIND_CLASS:
         return ST_CLASS
     if s.kind == LspSymbols.LSP_KIND_FUNCTION:
+        if _IsDefineMacro(s) != 0:
+            return ST_MACRO
         return ST_FUNCTION
     if s.kind == LspSymbols.LSP_KIND_METHOD:
+        if _IsDefineMacro(s) != 0:
+            return ST_MACRO
         return ST_METHOD
     if s.kind == LspSymbols.LSP_KIND_FIELD:
         return ST_PROPERTY
@@ -174,37 +190,43 @@ def _AstStream(pool: memhub.MemManager*,
     occs: list[LspSymbols.LspOccurrence*]* = a.occs
     si: int = 0
     oi: int = 0
-    while si < int(len(syms)) or oi < int(len(occs)):
+    n: int = int(len(syms))
+    m: int = int(len(occs))
+    while si < n or oi < m:
+        # Both inputs are in source order. Advance past declarations/occurrences that cannot
+        # produce a token so a skipped entry never holds back the other stream.
         sym_t: LspSemToken* = None
-        if si < int(len(syms)):
+        while si < n:
             s: LspSymbols.LspSymbol* = syms.get(sizet(si))
-            sym_t = _TokenForSymbol(pool, s)
-            if sym_t is not None and s is not None:
-                if _SymbolTokenValid(pool, a, sym_t, s.name) == 0:
-                    sym_t.typ = -1
+            si += 1
+            t: LspSemToken* = _TokenForSymbol(pool, s)
+            if t is not None and s is not None:
+                if _SymbolTokenValid(pool, a, t, s.name) == 0:
+                    t.typ = -1
+            if t is not None and t.typ >= 0:
+                sym_t = t
+                break
         occ_t: LspSemToken* = None
-        if oi < int(len(occs)):
-            occ_t = _TokenForOcc(pool, occs.get(sizet(oi)))
+        while oi < m:
+            t: LspSemToken* = _TokenForOcc(pool, occs.get(sizet(oi)))
+            oi += 1
+            if t is not None and t.typ >= 0:
+                occ_t = t
+                break
         if sym_t is None and occ_t is None:
-            if sym_t is None:
-                si += 1
-            if occ_t is None:
-                oi += 1
-            continue
+            break
         if sym_t is None:
             out.append(occ_t)
-            oi += 1
             continue
         if occ_t is None:
             out.append(sym_t)
-            si += 1
             continue
         if _Before(sym_t, occ_t) != 0:
             out.append(sym_t)
-            si += 1
+            oi -= 1
         else:
             out.append(occ_t)
-            oi += 1
+            si -= 1
     return out
 
 
@@ -235,6 +257,32 @@ def _ScanComments(pool: memhub.MemManager*, doc: LspDocuments.LspDocument*,
         off += 1
 
 
+def _AddSpanning(pool: memhub.MemManager*, doc: LspDocuments.LspDocument*,
+                 sl: int, sc: int, el: int, ec: int, typ: int,
+                 out: list[LspSemToken*]*):
+    """Split a multi-line literal/comment into one token per line.
+
+    LSP semantic tokens encode positions per line; a token with a line span would be interpreted
+    with a single-line length and paint unrelated characters.
+    """
+    count: int = doc.index.Count()
+    line: int = sl
+    while line <= el and line < count:
+        line_len: int = LspSource.ByteColToUtf16(
+            doc, line, LspSource.LineByteLength(doc, line))
+        start: int = 0
+        end: int = line_len
+        if line == sl:
+            start = sc
+        if line == el:
+            end = ec
+        if end > start:
+            t: LspSemToken* = _NewTok(pool, line, start, line, end, typ)
+            if t is not None:
+                out.append(t)
+        line += 1
+
+
 def _LexStream(pool: memhub.MemManager*,
                doc: LspDocuments.LspDocument*) -> list[LspSemToken*]*:
     out: list[LspSemToken*]* = list[LspSemToken*](pool, 128)
@@ -280,11 +328,13 @@ def _LexStream(pool: memhub.MemManager*,
             end_col = tok.col_offset + int(string.strlen(tok.str_val))
         if end_col < tok.col_offset:
             end_col = tok.col_offset
-        if typ >= 0 and end_col > tok.col_offset:
+        if typ >= 0 and (end_line0 > line0 or end_col > tok.col_offset):
             sc: int = LspSource.ByteColToUtf16(doc, line0, tok.col_offset)
             ec: int = LspSource.ByteColToUtf16(doc, end_line0, end_col)
-            if ec > sc:
-                t: LspSemToken* = _NewTok(pool, line0, sc, end_line0, ec, typ)
+            if end_line0 > line0:
+                _AddSpanning(pool, doc, line0, sc, end_line0, ec, typ, out)
+            elif ec > sc:
+                t: LspSemToken* = _NewTok(pool, line0, sc, line0, ec, typ)
                 if t is not None:
                     out.append(t)
         new_off: int = int(doc.index.LineStart(end_line0)) + end_col

Tools/lsp/fixtures/gate9.vp

@@ -0,0 +1,17 @@
+define MAX_VALUE = 100
+define ADD(a, b): return a + b
+
+
+def use(x: int) -> int:
+    s: str = """alpha
+beta"""
+    n: int = 42
+    # gate9 comment
+    return ADD(x, MAX_VALUE) + n
+
+
+class Box:
+    value: int
+
+    def get(self) -> int:
+        return self.value

Tools/lsp/probe.py

@@ -1235,6 +1235,126 @@ def gate8(exe: str) -> int:
     return 0
 
 
+def gate9(exe: str) -> int:
+    root = Path(__file__).resolve().parents[2]
+    source = root / "Tools" / "lsp" / "fixtures" / "gate9.vp"
+    uri = source.as_uri()
+    text = source.read_text(encoding="utf-8")
+    lines = text.split("\n")
+
+    client = Client(exe)
+    response = client.request(
+        "initialize",
+        {"processId": os.getpid(), "rootUri": root.as_uri(), "capabilities": {}},
+    )
+    capabilities = response.get("result", {}).get("capabilities", {})
+    sem = capabilities.get("semanticTokensProvider")
+    if not isinstance(sem, dict):
+        raise AssertionError(f"missing semanticTokensProvider: {response}")
+    legend = sem.get("legend", {})
+    types = legend.get("tokenTypes", [])
+    mods = legend.get("tokenModifiers", [])
+    for required in ("macro", "variable", "keyword", "string", "number", "comment"):
+        if required not in types:
+            raise AssertionError(f"semantic legend misses {required}: {legend}")
+    if "readonly" not in mods:
+        raise AssertionError(f"semantic legend misses readonly: {legend}")
+    macro_idx = types.index("macro")
+    variable_idx = types.index("variable")
+    keyword_idx = types.index("keyword")
+    comment_idx = types.index("comment")
+    string_idx = types.index("string")
+    number_idx = types.index("number")
+    readonly_bit = 1 << mods.index("readonly")
+    mod_mask = (1 << len(mods)) - 1
+
+    client.notify("initialized", {})
+    client.notify(
+        "textDocument/didOpen",
+        {"textDocument": {"uri": uri, "languageId": "viper", "version": 1, "text": text}},
+    )
+    client.wait_notification("textDocument/publishDiagnostics")
+
+    semantic = client.request(
+        "textDocument/semanticTokens/full", {"textDocument": {"uri": uri}}
+    ).get("result")
+    if not isinstance(semantic, dict):
+        raise AssertionError(f"semanticTokens/full failed: {semantic}")
+    tokens = _decode_tokens(semantic.get("data", []))
+    if not tokens:
+        raise AssertionError("semantic tokens are empty")
+
+    last_line = -1
+    last_end = -1
+    for token in tokens:
+        if token["length"] <= 0:
+            raise AssertionError(f"zero-length token: {token}")
+        if token["line"] < 0 or token["line"] >= len(lines):
+            raise AssertionError(f"token line is out of range: {token}")
+        line_len = len(lines[token["line"]])
+        if token["character"] < 0 or token["character"] + token["length"] > line_len:
+            raise AssertionError(f"token range is out of the line: {token}")
+        if token["type"] < 0 or token["type"] >= len(types):
+            raise AssertionError(f"token type is outside the legend: {token}")
+        if (token["modifiers"] & ~mod_mask) != 0:
+            raise AssertionError(f"token modifiers are outside the legend: {token}")
+        if token["type"] == keyword_idx:
+            raise AssertionError(f"keywords must be left to TextMate: {token}")
+        if token["line"] == last_line:
+            if token["character"] < last_end:
+                raise AssertionError(f"overlapping tokens on line {last_line}: {token}")
+            last_end = token["character"] + token["length"]
+        else:
+            last_line = token["line"]
+            last_end = token["character"] + token["length"]
+
+    add_decl = _position(text, "define ADD")
+    add_decl["character"] += len("define ")
+    if not _find_token(tokens, add_decl["line"], add_decl["character"], macro_idx):
+        raise AssertionError(f"define macro declaration is not a macro token: {add_decl}")
+    add_call = _position(text, "ADD(x")
+    if not _find_token(tokens, add_call["line"], add_call["character"], macro_idx):
+        raise AssertionError(f"define macro call is not a macro token: {add_call}")
+
+    max_decl = _position(text, "MAX_VALUE")
+    if not _find_token_mods(tokens, max_decl["line"], max_decl["character"],
+                            variable_idx, readonly_bit):
+        raise AssertionError(f"define constant is not readonly: {max_decl}")
+    max_call = _position(text, "MAX_VALUE", 1)
+    if not _find_token_mods(tokens, max_call["line"], max_call["character"],
+                            variable_idx, readonly_bit):
+        raise AssertionError(f"define constant use is not readonly: {max_call}")
+
+    for keyword in ("define", "def", "class", "return"):
+        pos = _position(text, keyword)
+        if _find_token(tokens, pos["line"], pos["character"], keyword_idx):
+            raise AssertionError(f"keyword {keyword} must not be a semantic token")
+
+    comment_pos = _position(text, "# gate9 comment")
+    if not _find_token(tokens, comment_pos["line"], comment_pos["character"], comment_idx):
+        raise AssertionError(f"missing comment token: {comment_pos}")
+    first_string = _position(text, '"""alpha')
+    if not _find_token(tokens, first_string["line"], first_string["character"], string_idx):
+        raise AssertionError(f"missing first line of the multi-line string: {first_string}")
+    second_string = _position(text, "beta")
+    if not _find_token(tokens, second_string["line"], second_string["character"], string_idx):
+        raise AssertionError(f"missing second line of the multi-line string: {second_string}")
+    number_pos = _position(text, "42")
+    if not _find_token(tokens, number_pos["line"], number_pos["character"], number_idx):
+        raise AssertionError(f"missing number token: {number_pos}")
+
+    client.notify("textDocument/didClose", {"textDocument": {"uri": uri}})
+    client.wait_notification("textDocument/publishDiagnostics")
+    client.request("shutdown")
+    code = client.close()
+    if code != 0:
+        raise AssertionError(f"exit code is {code}, expected 0")
+    print("semantic-legend: OK")
+    print("semantic-audit: OK")
+    print("semantic-macro: OK")
+    return 0
+
+
 def main() -> int:
     parser = argparse.ArgumentParser(description="ViperC LSP probe")
     parser.add_argument("--selfcheck", action="store_true", help="run pure-Python framing checks")
@@ -1245,6 +1365,7 @@ def main() -> int:
     parser.add_argument("--gate5", action="store_true", help="run rename/semantic/code-action checks")
     parser.add_argument("--gate7", action="store_true", help="run the product gate checks")
     parser.add_argument("--gate8", action="store_true", help="run cross-file references/rename checks")
+    parser.add_argument("--gate9", action="store_true", help="run semantic highlighting audits")
     parser.add_argument("--exe", help="path to viperc.exe")
     args = parser.parse_args()
 
@@ -1256,6 +1377,8 @@ def main() -> int:
     if not os.path.isfile(exe):
         print(f"probe: executable not found: {exe}", file=sys.stderr)
         return 2
+    if args.gate9:
+        return gate9(exe)
     if args.gate8:
         return gate8(exe)
     if args.gate7: