feat(lsp): Gate 9 语义高亮保真

define 宏函数与调用点使用 macro token；常量保持 variable+readonly；多行字符串按行拆分；修正符号/出现点合并顺序导致整行 token 丢失；大文档诊断 scratch 按源大小扩容。

App/lib/lsp/LspDiagnostics.vp

@@ -96,7 +96,7 @@ def Publish(pool: memhub.MemManager*, tr: LspTransport.LspTransport*,
     doc: LspDocuments.LspDocument* = docs.Find(uri)
     if doc is None:
         return 0
-    scratch: LspPools.LspPool* = docs.Scratch()
+    scratch: LspPools.LspPool* = docs.ScratchFor(sizet(string.strlen(doc.text)))
     if scratch is None:
         LspTransport.StderrWrite("viperc-lsp: diagnostics scratch pool is not available\n")
         return -1

App/lib/lsp/LspDocuments.vp

@@ -9,7 +9,11 @@ define LSP_DOC_POOL_SIZE = 4194304
 # Parse scratch arena shared by diagnostics: reset before each publish so ASTs and token lists do
 # not accumulate across edits. It lives in the per-documents object, whose owner is the permanent
 # pool, because the message pool is reset after every request.
+# The arena grows with the largest published document: a 4 MiB arena is exhausted by a few hundred
+# KiB of source, and allocation failure aborts the parse.
 define LSP_SCRATCH_POOL_SIZE = 4194304
+define LSP_SCRATCH_BYTES_PER_SOURCE = 128
+define LSP_SCRATCH_POOL_MAX = 134217728
 
 
 class LspDocument:
@@ -42,6 +46,21 @@ class LspDocuments:
     def Scratch(self) -> LspPools.LspPool*:
         return self.scratch
 
+    def ScratchFor(self, source_len: sizet) -> LspPools.LspPool*:
+        """Return a scratch arena large enough to parse a document of source_len bytes."""
+        want: sizet = source_len * sizet(LSP_SCRATCH_BYTES_PER_SOURCE)
+        if want < sizet(LSP_SCRATCH_POOL_SIZE):
+            want = sizet(LSP_SCRATCH_POOL_SIZE)
+        if want > sizet(LSP_SCRATCH_POOL_MAX):
+            want = sizet(LSP_SCRATCH_POOL_MAX)
+        if self.scratch is not None and self.scratch.size >= want:
+            return self.scratch
+        if self.scratch is not None:
+            self.scratch.Destroy()
+            self.scratch = None
+        self.scratch = LspPools.Create(self.pool, want)
+        return self.scratch
+
     def Find(self, uri: str) -> LspDocument*:
         return self._Find(uri)
 

App/lib/lsp/LspSemantic.vp

@@ -12,7 +12,8 @@ import lib.lsp.LspTransport as LspTransport
 import lib.lsp.LspRpc as LspRpc
 
 
-define LSP_MAX_SEMANTIC_TOKENS = 10000
+# Stays below 65535 so encoded deltas remain safe for clients that store token fields in 16 bits.
+define LSP_MAX_SEMANTIC_TOKENS = 60000
 
 # Indices must match the legend built by Provider().
 define ST_NAMESPACE = 0
@@ -28,6 +29,7 @@ define ST_STRING = 9
 define ST_NUMBER = 10
 define ST_OPERATOR = 11
 define ST_DECORATOR = 12
+define ST_MACRO = 13
 
 
 class LspSemToken:
@@ -79,6 +81,7 @@ def Provider(pool: memhub.MemManager*) -> json.JsonValue*:
     json.array_append(pool, types, json.string_val(pool, "number"))
     json.array_append(pool, types, json.string_val(pool, "operator"))
     json.array_append(pool, types, json.string_val(pool, "decorator"))
+    json.array_append(pool, types, json.string_val(pool, "macro"))
     json.array_append(pool, mods, json.string_val(pool, "declaration"))
     json.array_append(pool, mods, json.string_val(pool, "readonly"))
     json.object_set(pool, legend, "tokenTypes", types)
@@ -89,6 +92,15 @@ def Provider(pool: memhub.MemManager*) -> json.JsonValue*:
     return provider
 
 
+def _IsDefineMacro(s: LspSymbols.LspSymbol*) -> int:
+    if s is None or s.decl is None:
+        return 0
+    if s.decl.kind() != ast.ASTKind.FunctionDef:
+        return 0
+    fd: ast.FunctionDef* = (ast.FunctionDef*)(s.decl)
+    return 1 if (fd.flags & ast.FLAG_IS_DEFINE) != 0 else 0
+
+
 def _TypeFor(s: LspSymbols.LspSymbol*) -> int:
     if s is None:
         return ST_VARIABLE
@@ -97,8 +109,12 @@ def _TypeFor(s: LspSymbols.LspSymbol*) -> int:
     if s.kind == LspSymbols.LSP_KIND_CLASS:
         return ST_CLASS
     if s.kind == LspSymbols.LSP_KIND_FUNCTION:
+        if _IsDefineMacro(s) != 0:
+            return ST_MACRO
         return ST_FUNCTION
     if s.kind == LspSymbols.LSP_KIND_METHOD:
+        if _IsDefineMacro(s) != 0:
+            return ST_MACRO
         return ST_METHOD
     if s.kind == LspSymbols.LSP_KIND_FIELD:
         return ST_PROPERTY
@@ -174,37 +190,43 @@ def _AstStream(pool: memhub.MemManager*,
     occs: list[LspSymbols.LspOccurrence*]* = a.occs
     si: int = 0
     oi: int = 0
-    while si < int(len(syms)) or oi < int(len(occs)):
+    n: int = int(len(syms))
+    m: int = int(len(occs))
+    while si < n or oi < m:
+        # Both inputs are in source order. Advance past declarations/occurrences that cannot
+        # produce a token so a skipped entry never holds back the other stream.
         sym_t: LspSemToken* = None
-        if si < int(len(syms)):
+        while si < n:
             s: LspSymbols.LspSymbol* = syms.get(sizet(si))
-            sym_t = _TokenForSymbol(pool, s)
-            if sym_t is not None and s is not None:
-                if _SymbolTokenValid(pool, a, sym_t, s.name) == 0:
-                    sym_t.typ = -1
+            si += 1
+            t: LspSemToken* = _TokenForSymbol(pool, s)
+            if t is not None and s is not None:
+                if _SymbolTokenValid(pool, a, t, s.name) == 0:
+                    t.typ = -1
+            if t is not None and t.typ >= 0:
+                sym_t = t
+                break
         occ_t: LspSemToken* = None
-        if oi < int(len(occs)):
-            occ_t = _TokenForOcc(pool, occs.get(sizet(oi)))
+        while oi < m:
+            t: LspSemToken* = _TokenForOcc(pool, occs.get(sizet(oi)))
+            oi += 1
+            if t is not None and t.typ >= 0:
+                occ_t = t
+                break
         if sym_t is None and occ_t is None:
-            if sym_t is None:
-                si += 1
-            if occ_t is None:
-                oi += 1
-            continue
+            break
         if sym_t is None:
             out.append(occ_t)
-            oi += 1
             continue
         if occ_t is None:
             out.append(sym_t)
-            si += 1
             continue
         if _Before(sym_t, occ_t) != 0:
             out.append(sym_t)
-            si += 1
+            oi -= 1
         else:
             out.append(occ_t)
-            oi += 1
+            si -= 1
     return out
 
 
@@ -235,6 +257,32 @@ def _ScanComments(pool: memhub.MemManager*, doc: LspDocuments.LspDocument*,
         off += 1
 
 
+def _AddSpanning(pool: memhub.MemManager*, doc: LspDocuments.LspDocument*,
+                 sl: int, sc: int, el: int, ec: int, typ: int,
+                 out: list[LspSemToken*]*):
+    """Split a multi-line literal/comment into one token per line.
+
+    LSP semantic tokens encode positions per line; a token with a line span would be interpreted
+    with a single-line length and paint unrelated characters.
+    """
+    count: int = doc.index.Count()
+    line: int = sl
+    while line <= el and line < count:
+        line_len: int = LspSource.ByteColToUtf16(
+            doc, line, LspSource.LineByteLength(doc, line))
+        start: int = 0
+        end: int = line_len
+        if line == sl:
+            start = sc
+        if line == el:
+            end = ec
+        if end > start:
+            t: LspSemToken* = _NewTok(pool, line, start, line, end, typ)
+            if t is not None:
+                out.append(t)
+        line += 1
+
+
 def _LexStream(pool: memhub.MemManager*,
                doc: LspDocuments.LspDocument*) -> list[LspSemToken*]*:
     out: list[LspSemToken*]* = list[LspSemToken*](pool, 128)
@@ -280,11 +328,13 @@ def _LexStream(pool: memhub.MemManager*,
             end_col = tok.col_offset + int(string.strlen(tok.str_val))
         if end_col < tok.col_offset:
             end_col = tok.col_offset
-        if typ >= 0 and end_col > tok.col_offset:
+        if typ >= 0 and (end_line0 > line0 or end_col > tok.col_offset):
             sc: int = LspSource.ByteColToUtf16(doc, line0, tok.col_offset)
             ec: int = LspSource.ByteColToUtf16(doc, end_line0, end_col)
-            if ec > sc:
-                t: LspSemToken* = _NewTok(pool, line0, sc, end_line0, ec, typ)
+            if end_line0 > line0:
+                _AddSpanning(pool, doc, line0, sc, end_line0, ec, typ, out)
+            elif ec > sc:
+                t: LspSemToken* = _NewTok(pool, line0, sc, line0, ec, typ)
                 if t is not None:
                     out.append(t)
         new_off: int = int(doc.index.LineStart(end_line0)) + end_col