From c42ae889d183ce46a892dfa5f6f380f467a7483b Mon Sep 17 00:00:00 2001 From: TermiNexus Date: Mon, 26 Oct 2026 14:23:12 +0800 Subject: [PATCH] =?UTF-8?q?feat(lsp):=20Gate=201=20=E6=96=87=E6=A1=A3?= =?UTF-8?q?=E5=90=8C=E6=AD=A5=E3=80=81UTF-16=20=E4=BD=8D=E7=BD=AE=E4=B8=8E?= =?UTF-8?q?=E5=B7=A5=E4=BD=9C=E5=8C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - LspPositions：UTF-8 字节列 / UTF-16 code unit 转换，支持 LF/CRLF/emoji - LspDocuments：didOpen/didChange/didClose/didSave，增量与全量替换，双池复用 - LspWorkspace：file URI 解码、project.vpj 向上查找、source_dir/includes - LspServer：请求池 mark/reset、textDocumentSync、viper/lspStats 与 viper/lspDocument - probe --gate1：文档编辑、UTF-16 代理对、10k 次编辑与池稳定性 - 依赖 includes/memhub 的 NewMemBuddy 堆构造工厂（单独提交 fa7591e） --- App/lib/lsp/LspDocuments.vp | 196 +++++++++++++++++++++++++++++++ App/lib/lsp/LspPools.vp | 72 ++++++++++++ App/lib/lsp/LspPositions.vp | 209 +++++++++++++++++++++++++++++++++ App/lib/lsp/LspRpc.vp | 13 +++ App/lib/lsp/LspServer.vp | 226 +++++++++++++++++++++++++++++++++--- App/lib/lsp/LspTransport.vp | 11 +- App/lib/lsp/LspWorkspace.vp | 141 ++++++++++++++++++++++ App/main.vp | 2 +- Tools/lsp/probe.py | 150 +++++++++++++++++++++++- 9 files changed, 997 insertions(+), 23 deletions(-) create mode 100644 App/lib/lsp/LspDocuments.vp create mode 100644 App/lib/lsp/LspPools.vp create mode 100644 App/lib/lsp/LspPositions.vp create mode 100644 App/lib/lsp/LspWorkspace.vp diff --git a/App/lib/lsp/LspDocuments.vp b/App/lib/lsp/LspDocuments.vp new file mode 100644 index 0000000..3f03bb8 --- /dev/null +++ b/App/lib/lsp/LspDocuments.vp @@ -0,0 +1,196 @@ +from stdint import * +import string +import memhub +import lib.lsp.LspPools as LspPools +import lib.lsp.LspPositions as LspPositions + + +define LSP_DOC_POOL_SIZE = 4194304 + + +class LspDocument: + uri: str + text: str + version: int + index: LspPositions.LspLineIndex* + pool: LspPools.LspPool* + spare: LspPools.LspPool* + + +class LspDocuments: + pool: memhub.MemManager* + docs: list[LspDocument*]* + count: int + + def __new__(self, pool: memhub.MemManager*) -> rawptr: + p: LspDocuments* = (LspDocuments*)(pool.alloc(LspDocuments.__sizeof__())) + if p: + string.memset(p, 0, LspDocuments.__sizeof__()) + return rawptr(p) + + def __init__(self, pool: memhub.MemManager*): + self.pool = pool + self.docs = list[LspDocument*](pool, 8) + self.count = 0 + + def Find(self, uri: str) -> LspDocument*: + return self._Find(uri) + + def _Find(self, uri: str) -> LspDocument*: + if uri is None: + return None + i: int = 0 + while i < self.count: + doc: LspDocument* = self.docs[i] + if doc is not None and doc.uri is not None: + if string.strcmp(doc.uri, uri) == 0: + return doc + i += 1 + return None + + def _DestroyDoc(self, doc: LspDocument*) -> int: + if doc is None: + return -1 + if doc.pool is not None: + doc.pool.Destroy() + doc.pool = None + if doc.spare is not None: + doc.spare.Destroy() + doc.spare = None + return 0 + + def Open(self, uri: str, text: str, version: int) -> LspDocument*: + if uri is None or text is None: + return None + existing: LspDocument* = self._Find(uri) + if existing is not None: + self.Close(uri) + + doc: LspDocument* = (LspDocument*)(self.pool.alloc(LspDocument.__sizeof__())) + if doc is None: + return None + string.memset(doc, 0, LspDocument.__sizeof__()) + doc.uri = LspPools.DupStr(self.pool, uri) + if doc.uri is None: + return None + + p1: LspPools.LspPool* = LspPools.Create(self.pool, sizet(LSP_DOC_POOL_SIZE)) + p2: LspPools.LspPool* = LspPools.Create(self.pool, sizet(LSP_DOC_POOL_SIZE)) + if p1 is None or p2 is None: + if p1 is not None: + p1.Destroy() + if p2 is not None: + p2.Destroy() + return None + doc.pool = p1 + doc.spare = p2 + doc.text = LspPools.DupStr(p1.mb, text) + if doc.text is None: + self._DestroyDoc(doc) + return None + doc.index = LspPositions.LspLineIndex(p1.mb) + if doc.index is None or doc.index.Build(doc.text) != 0: + self._DestroyDoc(doc) + return None + doc.version = version + self.docs.append(doc) + self.count += 1 + return doc + + def ApplyChange(self, uri: str, text: str, has_range: int, + start_line: int, start_char: int, + end_line: int, end_char: int, + version: int) -> int: + doc: LspDocument* = self._Find(uri) + if doc is None or doc.index is None or doc.spare is None: + return -1 + new_pool: LspPools.LspPool* = doc.spare + if new_pool.Reset() < 0: + return -1 + + old_len: sizet = doc.index.Length() + start: sizet = 0 + end: sizet = old_len + if has_range != 0: + if doc.index.RangeToOffsets(start_line, start_char, end_line, end_char, + &start, &end) != 0: + return -1 + if end > old_len: + return -1 + + add_len: sizet = 0 + if text is not None: + add_len = string.strlen(text) + new_len: sizet = start + add_len + (old_len - end) + buf: bytes = new_pool.mb.alloc(new_len + 1) + if buf is None: + return -1 + if start > 0: + string.memcpy(buf, doc.text, start) + if add_len > 0: + string.memcpy(buf + start, text, add_len) + if old_len > end: + string.memcpy(buf + start + add_len, doc.text + end, old_len - end) + buf[new_len] = 0 + + new_index: LspPositions.LspLineIndex* = LspPositions.LspLineIndex(new_pool.mb) + if new_index is None or new_index.Build(str(buf)) != 0: + return -1 + + old_pool: LspPools.LspPool* = doc.pool + doc.pool = new_pool + doc.spare = old_pool + doc.text = str(buf) + doc.index = new_index + doc.version = version + if old_pool is not None: + old_pool.Reset() + return 0 + + def Close(self, uri: str) -> int: + i: int = 0 + while i < self.count: + doc: LspDocument* = self.docs[i] + if doc is not None and doc.uri is not None: + if string.strcmp(doc.uri, uri) == 0: + self._DestroyDoc(doc) + self.docs[i] = self.docs[self.count - 1] + self.docs.pop() + self.count -= 1 + return 0 + i += 1 + return -1 + + def Count(self) -> int: + return self.count + + def DocAt(self, index: int) -> LspDocument*: + if index < 0 or index >= self.count: + return None + return self.docs[index] + + def UsedBytes(self) -> sizet: + total: sizet = 0 + i: int = 0 + while i < self.count: + doc: LspDocument* = self.docs[i] + if doc is not None: + if doc.pool is not None: + total += doc.pool.UsedBytes() + if doc.spare is not None: + total += doc.spare.UsedBytes() + i += 1 + return total + + def FreeBytes(self) -> sizet: + total: sizet = 0 + i: int = 0 + while i < self.count: + doc: LspDocument* = self.docs[i] + if doc is not None: + if doc.pool is not None: + total += doc.pool.FreeBytes() + if doc.spare is not None: + total += doc.spare.FreeBytes() + i += 1 + return total diff --git a/App/lib/lsp/LspPools.vp b/App/lib/lsp/LspPools.vp new file mode 100644 index 0000000..70f7cf7 --- /dev/null +++ b/App/lib/lsp/LspPools.vp @@ -0,0 +1,72 @@ +from stdint import * +import stdlib +import string +import memhub + + +class LspPool: + owner: memhub.MemManager* + arena: bytes + mb: memhub.MemBuddy* + size: sizet + + def __new__(self, owner: memhub.MemManager*, size: sizet) -> rawptr: + p: LspPool* = (LspPool*)(owner.alloc(LspPool.__sizeof__())) + if p: + string.memset(p, 0, LspPool.__sizeof__()) + return rawptr(p) + + def __init__(self, owner: memhub.MemManager*, size: sizet): + self.owner = owner + self.size = size + self.arena = stdlib.malloc(size) + self.mb = None + if self.arena is not None: + self.mb = memhub.NewMemBuddy(self.arena, size, 0, 0) + + def IsValid(self) -> bool: + return self.mb is not None + + def Reset(self) -> int: + if self.mb is None: + return -1 + return self.mb.reset() + + def Destroy(self) -> int: + if self.arena is not None: + stdlib.free(self.arena) + if self.mb is not None: + stdlib.free(self.mb) + self.arena = None + self.mb = None + return 0 + + def UsedBytes(self) -> sizet: + if self.mb is None: + return 0 + return self.mb.used_bytes() + + def FreeBytes(self) -> sizet: + if self.mb is None: + return 0 + return self.mb.free_bytes() + + +def Create(owner: memhub.MemManager*, size: sizet) -> LspPool*: + p: LspPool* = LspPool(owner, size) + if p is None or not p.IsValid(): + if p is not None: + p.Destroy() + return None + return p + + +def DupStr(pool: memhub.MemManager*, s: str) -> str: + if s is None: + return None + n: sizet = string.strlen(s) + buf: bytes = pool.alloc(n + 1) + if buf is None: + return None + string.memcpy(buf, s, n + 1) + return str(buf) diff --git a/App/lib/lsp/LspPositions.vp b/App/lib/lsp/LspPositions.vp new file mode 100644 index 0000000..9419695 --- /dev/null +++ b/App/lib/lsp/LspPositions.vp @@ -0,0 +1,209 @@ +from stdint import * +import string +import memhub + + +def _Utf8Seq(b: uint8) -> int: + if b < 0x80: + return 1 + if (b & 0xE0) == 0xC0: + return 2 + if (b & 0xF0) == 0xE0: + return 3 + if (b & 0xF8) == 0xF0: + return 4 + return 1 + + +def _Utf8Units(b: uint8) -> int: + if b >= 0xF0: + return 2 + return 1 + + +class LspLineIndex: + pool: memhub.MemManager* + text: str + starts: sizet* + ends: sizet* + count: int + length: sizet + + def __new__(self, pool: memhub.MemManager*) -> rawptr: + p: LspLineIndex* = (LspLineIndex*)(pool.alloc(LspLineIndex.__sizeof__())) + if p: + string.memset(p, 0, LspLineIndex.__sizeof__()) + return rawptr(p) + + def __init__(self, pool: memhub.MemManager*): + self.pool = pool + self.text = None + self.starts = None + self.ends = None + self.count = 0 + self.length = 0 + + def Build(self, text: str) -> int: + if text is None: + return -1 + self.text = text + self.length = string.strlen(text) + total: int = int(self.length) + + count: int = 1 + i: int = 0 + while i < total: + ch: char = self.text[i] + if ch == '\n': + count += 1 + i += 1 + elif ch == '\r': + count += 1 + if i + 1 < total and self.text[i + 1] == '\n': + i += 2 + else: + i += 1 + else: + i += 1 + self.count = count + self.starts = (sizet*)(self.pool.alloc(sizet(count) * 8)) + self.ends = (sizet*)(self.pool.alloc(sizet(count) * 8)) + if self.starts is None or self.ends is None: + return -1 + + idx: int = 0 + start: sizet = 0 + i = 0 + while i < total: + ch = self.text[i] + if ch == '\n': + self.starts[idx] = start + self.ends[idx] = sizet(i) + idx += 1 + start = sizet(i + 1) + i += 1 + elif ch == '\r': + self.starts[idx] = start + self.ends[idx] = sizet(i) + idx += 1 + if i + 1 < total and self.text[i + 1] == '\n': + i += 2 + else: + i += 1 + start = sizet(i) + else: + i += 1 + self.starts[idx] = start + self.ends[idx] = self.length + return 0 + + def Count(self) -> int: + return self.count + + def Length(self) -> sizet: + return self.length + + def LineStart(self, line: int) -> sizet: + if line < 0: + line = 0 + if line >= self.count: + line = self.count - 1 + return self.starts[line] + + def LineEnd(self, line: int) -> sizet: + if line < 0: + line = 0 + if line >= self.count: + line = self.count - 1 + return self.ends[line] + + def ByteColToUtf16(self, line: int, byte_col: int) -> int: + if line < 0: + line = 0 + if line >= self.count: + line = self.count - 1 + if byte_col < 0: + byte_col = 0 + start: sizet = self.starts[line] + end: sizet = self.ends[line] + limit: sizet = start + sizet(byte_col) + if limit > end: + limit = end + i: sizet = start + units: int = 0 + while i < limit: + b: uint8 = uint8(self.text[i]) + units += _Utf8Units(b) + seq: int = _Utf8Seq(b) + if i + sizet(seq) > limit: + i += 1 + else: + i += sizet(seq) + return units + + def Utf16ColToByte(self, line: int, utf16_col: int) -> int: + if line < 0: + line = 0 + if line >= self.count: + line = self.count - 1 + if utf16_col < 0: + utf16_col = 0 + start: sizet = self.starts[line] + end: sizet = self.ends[line] + i: sizet = start + units: int = 0 + while i < end: + if units >= utf16_col: + break + b: uint8 = uint8(self.text[i]) + u: int = _Utf8Units(b) + if units + u > utf16_col: + break + units += u + seq: int = _Utf8Seq(b) + if i + sizet(seq) > end: + i += 1 + else: + i += sizet(seq) + return int(i - start) + + def OffsetToPosition(self, offset: sizet, out_line: int*, out_char: int*) -> int: + if out_line is None or out_char is None: + return -1 + if offset > self.length: + offset = self.length + lo: int = 0 + hi: int = self.count - 1 + line: int = 0 + while lo <= hi: + mid: int = (lo + hi) >> 1 + if self.starts[mid] <= offset: + line = mid + lo = mid + 1 + else: + hi = mid - 1 + byte_col: int = int(offset - self.starts[line]) + out_line[0] = line + out_char[0] = self.ByteColToUtf16(line, byte_col) + return 0 + + def PositionToOffset(self, line: int, utf16_col: int) -> sizet: + if line < 0: + line = 0 + if line >= self.count: + line = self.count - 1 + byte_col: int = self.Utf16ColToByte(line, utf16_col) + return self.starts[line] + sizet(byte_col) + + def RangeToOffsets(self, start_line: int, start_char: int, + end_line: int, end_char: int, + out_start: sizet*, out_end: sizet*) -> int: + if out_start is None or out_end is None: + return -1 + start: sizet = self.PositionToOffset(start_line, start_char) + end: sizet = self.PositionToOffset(end_line, end_char) + if end < start: + return -1 + out_start[0] = start + out_end[0] = end + return 0 diff --git a/App/lib/lsp/LspRpc.vp b/App/lib/lsp/LspRpc.vp index 760dd70..674ba1d 100644 --- a/App/lib/lsp/LspRpc.vp +++ b/App/lib/lsp/LspRpc.vp @@ -19,6 +19,19 @@ def GetStringField(obj: json.JsonValue*, key: str) -> str: return v.as_string() +def GetIntField(obj: json.JsonValue*, key: str, default: int) -> int: + if obj is None: + return default + v: json.JsonValue* = obj[key] + if v is None: + return default + if v.is_int(): + return int(v.as_int()) + if v.is_float(): + return int(v.as_float()) + return default + + def _NewEnvelope(pool: memhub.MemManager*, id_node: json.JsonValue*) -> json.JsonValue*: resp: json.JsonValue* = json.object(pool) if resp is None: diff --git a/App/lib/lsp/LspServer.vp b/App/lib/lsp/LspServer.vp index f095512..5a0d518 100644 --- a/App/lib/lsp/LspServer.vp +++ b/App/lib/lsp/LspServer.vp @@ -2,13 +2,25 @@ from stdint import * import string import json import memhub +import os +import os._win32 +import os._posix +import os.path import lib.lsp.LspProtocol as LspProtocol import lib.lsp.LspTransport as LspTransport import lib.lsp.LspRpc as LspRpc +import lib.lsp.LspDocuments as LspDocuments +import lib.lsp.LspWorkspace as LspWorkspace define LSP_SERVER_NAME = "viperc-lsp" -define LSP_SERVER_VERSION = "0.1.0-gate0" +define LSP_SERVER_VERSION = "0.1.0-gate1" + + +def _StringOrNull(pool: memhub.MemManager*, s: str) -> json.JsonValue*: + if s is None: + return json.null(pool) + return json.string_val(pool, s) def _BuildInitializeResult(pool: memhub.MemManager*) -> json.JsonValue*: @@ -18,11 +30,13 @@ def _BuildInitializeResult(pool: memhub.MemManager*) -> json.JsonValue*: enc: json.JsonValue* = json.string_val(pool, "utf-16") name: json.JsonValue* = json.string_val(pool, LSP_SERVER_NAME) ver: json.JsonValue* = json.string_val(pool, LSP_SERVER_VERSION) + sync: json.JsonValue* = json.int_val(pool, 2) if result is None or caps is None or info is None: return None - if enc is None or name is None or ver is None: + if enc is None or name is None or ver is None or sync is None: return None json.object_set(pool, caps, "positionEncoding", enc) + json.object_set(pool, caps, "textDocumentSync", sync) json.object_set(pool, result, "capabilities", caps) json.object_set(pool, info, "name", name) json.object_set(pool, info, "version", ver) @@ -30,16 +44,182 @@ def _BuildInitializeResult(pool: memhub.MemManager*) -> json.JsonValue*: return result -def Run(pool: memhub.MemManager*) -> int: - tr: LspTransport.LspTransport* = LspTransport.LspTransport(pool) +def _HandleInitialize(pool: memhub.MemManager*, tr: LspTransport.LspTransport*, + ws: LspWorkspace.LspWorkspace*, id_node: json.JsonValue*, + params: json.JsonValue*) -> int: + root_uri: str = None + if params is not None: + root_uri = LspRpc.GetStringField(params, "rootUri") + if root_uri is None: + folders: json.JsonValue* = params["workspaceFolders"] + if folders is not None and folders.is_array() and len(folders) > 0: + first: json.JsonValue* = folders.get_item(0) + root_uri = LspRpc.GetStringField(first, "uri") + if root_uri is None: + root_uri = LspRpc.GetStringField(params, "rootPath") + if ws.Init(root_uri) != 0: + LspTransport.StderrWrite("viperc-lsp: workspace initialization failed; using no workspace\n") + result: json.JsonValue* = _BuildInitializeResult(pool) + if result is None: + return -1 + return LspRpc.WriteResponse(pool, tr, id_node, result) + + +def _HandleDidOpen(docs: LspDocuments.LspDocuments*, params: json.JsonValue*) -> int: + if params is None: + return -1 + td: json.JsonValue* = params["textDocument"] + uri: str = LspRpc.GetStringField(td, "uri") + text: str = LspRpc.GetStringField(td, "text") + version: int = LspRpc.GetIntField(td, "version", 0) + if uri is None or text is None: + LspTransport.StderrWrite("viperc-lsp: didOpen missing uri or text\n") + return -1 + if docs.Open(uri, text, version) is None: + LspTransport.StderrWrite("viperc-lsp: didOpen failed to store document\n") + return -1 + return 0 + + +def _HandleDidChange(docs: LspDocuments.LspDocuments*, params: json.JsonValue*) -> int: + if params is None: + return -1 + td: json.JsonValue* = params["textDocument"] + uri: str = LspRpc.GetStringField(td, "uri") + version: int = LspRpc.GetIntField(td, "version", 0) + changes: json.JsonValue* = params["contentChanges"] + if uri is None or changes is None or not changes.is_array(): + LspTransport.StderrWrite("viperc-lsp: didChange missing uri or contentChanges\n") + return -1 + n: int = int(len(changes)) + i: int = 0 + while i < n: + ch: json.JsonValue* = changes.get_item(sizet(i)) + if ch is None or not ch.is_object(): + return -1 + text: str = LspRpc.GetStringField(ch, "text") + has_range: int = 0 + sl: int = 0 + sc: int = 0 + el: int = 0 + ec: int = 0 + rng: json.JsonValue* = ch["range"] + if rng is not None: + has_range = 1 + start: json.JsonValue* = rng["start"] + end: json.JsonValue* = rng["end"] + sl = LspRpc.GetIntField(start, "line", 0) + sc = LspRpc.GetIntField(start, "character", 0) + el = LspRpc.GetIntField(end, "line", 0) + ec = LspRpc.GetIntField(end, "character", 0) + if docs.ApplyChange(uri, text, has_range, sl, sc, el, ec, version) != 0: + LspTransport.StderrWrite("viperc-lsp: didChange failed to apply a content change\n") + return -1 + i += 1 + return 0 + + +def _HandleDidClose(docs: LspDocuments.LspDocuments*, params: json.JsonValue*) -> int: + if params is None: + return -1 + td: json.JsonValue* = params["textDocument"] + uri: str = LspRpc.GetStringField(td, "uri") + if uri is None: + return -1 + docs.Close(uri) + return 0 + + +def _StatsResult(pool: memhub.MemBuddy*, docs: LspDocuments.LspDocuments*, + ws: LspWorkspace.LspWorkspace*) -> json.JsonValue*: + result: json.JsonValue* = json.object(pool) + if result is None: + return None + json.object_set(pool, result, "documentCount", + json.int_val(pool, int64(docs.Count()))) + json.object_set(pool, result, "usedBytes", + json.int_val(pool, int64(docs.UsedBytes()))) + json.object_set(pool, result, "freeBytes", + json.int_val(pool, int64(docs.FreeBytes()))) + json.object_set(pool, result, "requestUsedBytes", + json.int_val(pool, int64(pool.used_bytes()))) + json.object_set(pool, result, "requestFreeBytes", + json.int_val(pool, int64(pool.free_bytes()))) + json.object_set(pool, result, "sourceDir", _StringOrNull(pool, ws.SourceDir())) + json.object_set(pool, result, "includes", _StringOrNull(pool, ws.Includes())) + json.object_set(pool, result, "project", _StringOrNull(pool, ws.ProjectPath())) + + arr: json.JsonValue* = json.array(pool) + if arr is None: + return None + i: int = 0 + while i < docs.Count(): + doc: LspDocuments.LspDocument* = docs.DocAt(i) + if doc is not None: + item: json.JsonValue* = json.object(pool) + if item is not None: + json.object_set(pool, item, "uri", _StringOrNull(pool, doc.uri)) + json.object_set(pool, item, "version", json.int_val(pool, int64(doc.version))) + if doc.index is not None: + json.object_set(pool, item, "textLength", + json.int_val(pool, int64(doc.index.Length()))) + json.array_append(pool, arr, item) + i += 1 + json.object_set(pool, result, "documents", arr) + return result + + +def _HandleStats(pool: memhub.MemBuddy*, tr: LspTransport.LspTransport*, + docs: LspDocuments.LspDocuments*, ws: LspWorkspace.LspWorkspace*, + id_node: json.JsonValue*) -> int: + result: json.JsonValue* = _StatsResult(pool, docs, ws) + if result is None: + return -1 + return LspRpc.WriteResponse(pool, tr, id_node, result) + + +def _HandleDocument(pool: memhub.MemManager*, tr: LspTransport.LspTransport*, + docs: LspDocuments.LspDocuments*, id_node: json.JsonValue*, + params: json.JsonValue*) -> int: + uri: str = None + if params is not None: + uri = LspRpc.GetStringField(params, "uri") + doc: LspDocuments.LspDocument* = docs.Find(uri) + if doc is None: + return LspRpc.WriteResponse(pool, tr, id_node, json.null(pool)) + result: json.JsonValue* = json.object(pool) + if result is None: + return -1 + json.object_set(pool, result, "uri", _StringOrNull(pool, doc.uri)) + json.object_set(pool, result, "version", json.int_val(pool, int64(doc.version))) + if doc.index is not None: + json.object_set(pool, result, "textLength", json.int_val(pool, int64(doc.index.Length()))) + json.object_set(pool, result, "text", _StringOrNull(pool, doc.text)) + return LspRpc.WriteResponse(pool, tr, id_node, result) + + +def Run(pool: memhub.MemBuddy*, perm_pool: memhub.MemBuddy*) -> int: + os._mbuddy = perm_pool + os._win32._mbuddy = perm_pool + os._posix._mbuddy = perm_pool + os.path._mbuddy = perm_pool + + tr: LspTransport.LspTransport* = LspTransport.LspTransport(perm_pool, pool) if tr is None or not tr.IsValid(): LspTransport.StderrWrite("viperc-lsp: invalid stdio handles or transport allocation failed\n") return 1 + docs: LspDocuments.LspDocuments* = LspDocuments.LspDocuments(perm_pool) + ws: LspWorkspace.LspWorkspace* = LspWorkspace.LspWorkspace(perm_pool) + if docs is None or ws is None: + LspTransport.StderrWrite("viperc-lsp: failed to allocate the document or workspace store\n") + return 1 shutdown_requested: int = 0 while True: + mark: sizet = pool.mark() body: str = tr.ReadMessage() if body is None: + pool.reset_to(mark) if tr.error != 0: LspTransport.StderrWrite("viperc-lsp: malformed message framing\n") return 1 @@ -50,6 +230,7 @@ def Run(pool: memhub.MemManager*) -> int: if LspRpc.WriteError(pool, tr, None, LspProtocol.LSP_ERROR_PARSE, "Invalid JSON") != 0: return 1 + pool.reset_to(mark) continue id_node: json.JsonValue* = msg["id"] @@ -60,26 +241,39 @@ def Run(pool: memhub.MemManager*) -> int: LspProtocol.LSP_ERROR_INVALID_REQUEST, "Invalid Request") != 0: return 1 + pool.reset_to(mark) continue + params: json.JsonValue* = msg["params"] + rc: int = 0 if string.strcmp(method, LspProtocol.LSP_METHOD_INITIALIZE) == 0: - result: json.JsonValue* = _BuildInitializeResult(pool) - if result is None: - return 1 - if LspRpc.WriteResponse(pool, tr, id_node, result) != 0: - return 1 + rc = _HandleInitialize(pool, tr, ws, id_node, params) elif string.strcmp(method, LspProtocol.LSP_METHOD_INITIALIZED) == 0: - continue + pass + elif string.strcmp(method, "textDocument/didOpen") == 0: + _HandleDidOpen(docs, params) + elif string.strcmp(method, "textDocument/didChange") == 0: + _HandleDidChange(docs, params) + elif string.strcmp(method, "textDocument/didClose") == 0: + _HandleDidClose(docs, params) + elif string.strcmp(method, "textDocument/didSave") == 0: + pass + elif string.strcmp(method, "viper/lspStats") == 0: + rc = _HandleStats(pool, tr, docs, ws, id_node) + elif string.strcmp(method, "viper/lspDocument") == 0: + rc = _HandleDocument(pool, tr, docs, id_node, params) elif string.strcmp(method, LspProtocol.LSP_METHOD_SHUTDOWN) == 0: shutdown_requested = 1 - if LspRpc.WriteResponse(pool, tr, id_node, json.null(pool)) != 0: - return 1 + rc = LspRpc.WriteResponse(pool, tr, id_node, json.null(pool)) elif string.strcmp(method, LspProtocol.LSP_METHOD_EXIT) == 0: return 0 if shutdown_requested != 0 else 1 else: if id_node is not None: - if LspRpc.WriteError(pool, tr, id_node, - LspProtocol.LSP_ERROR_METHOD_NOT_FOUND, - "Method not found") != 0: - return 1 + rc = LspRpc.WriteError(pool, tr, id_node, + LspProtocol.LSP_ERROR_METHOD_NOT_FOUND, + "Method not found") + + if rc != 0: + return 1 + pool.reset_to(mark) return 0 diff --git a/App/lib/lsp/LspTransport.vp b/App/lib/lsp/LspTransport.vp index 695702b..d20856c 100644 --- a/App/lib/lsp/LspTransport.vp +++ b/App/lib/lsp/LspTransport.vp @@ -1,4 +1,5 @@ from stdint import * +import stdlib import string import viperlib import memhub @@ -22,18 +23,18 @@ class LspTransport: eof: int error: int - def __new__(self, pool: memhub.MemManager*) -> rawptr: - ptr: LspTransport* = (LspTransport*)(pool.alloc(LspTransport.__sizeof__())) + def __new__(self, owner: memhub.MemManager*, request: memhub.MemManager*) -> rawptr: + ptr: LspTransport* = (LspTransport*)(owner.alloc(LspTransport.__sizeof__())) if ptr: string.memset(ptr, 0, LspTransport.__sizeof__()) return rawptr(ptr) - def __init__(self, pool: memhub.MemManager*): - self.pool = pool + def __init__(self, owner: memhub.MemManager*, request: memhub.MemManager*): + self.pool = request self.in_handle = win32.win32file.GetStdHandle(win32.win32file.STD_INPUT_HANDLE) self.out_handle = win32.win32file.GetStdHandle(win32.win32file.STD_OUTPUT_HANDLE) self.buf_cap = sizet(LSP_READ_BUFFER_SIZE) - self.buf = pool.alloc(self.buf_cap) + self.buf = stdlib.malloc(self.buf_cap) self.buf_len = 0 self.buf_pos = 0 self.eof = 0 diff --git a/App/lib/lsp/LspWorkspace.vp b/App/lib/lsp/LspWorkspace.vp new file mode 100644 index 0000000..2ad4226 --- /dev/null +++ b/App/lib/lsp/LspWorkspace.vp @@ -0,0 +1,141 @@ +from stdint import * +import string +import os +import os._win32 +import os._posix +import os.path +import memhub +import lib.lsp.LspPools as LspPools +import lib.Projectrans.Config as Config + + +def _HexVal(ch: char) -> int: + if ch >= '0' and ch <= '9': + return int(ch) - int('0') + if ch >= 'a' and ch <= 'f': + return int(ch) - int('a') + 10 + if ch >= 'A' and ch <= 'F': + return int(ch) - int('A') + 10 + return -1 + + +def DecodeUri(pool: memhub.MemManager*, uri: str) -> str: + if uri is None or uri[0] == 0: + return None + p: str = uri + if string.strncmp(uri, "file://", 7) == 0: + p = uri + 7 + if p[0] == '/' and len(p) >= 3 and p[2] == ':': + p = p + 1 + n: sizet = string.strlen(p) + out: bytes = pool.alloc(n + 1) + if out is None: + return None + i: sizet = 0 + o: sizet = 0 + while i < n: + ch: char = p[i] + if ch == '%' and i + 2 < n: + hi: int = _HexVal(p[i + 1]) + lo: int = _HexVal(p[i + 2]) + if hi >= 0 and lo >= 0: + out[o] = char(hi * 16 + lo) + o += 1 + i += 3 + continue + out[o] = ch + o += 1 + i += 1 + out[o] = 0 + return str(out) + + +class LspWorkspace: + pool: memhub.MemBuddy* + root: str + project: str + source_dir: str + includes: str + ready: int + + def __new__(self, pool: memhub.MemBuddy*) -> rawptr: + p: LspWorkspace* = (LspWorkspace*)(pool.alloc(LspWorkspace.__sizeof__())) + if p: + string.memset(p, 0, LspWorkspace.__sizeof__()) + return rawptr(p) + + def __init__(self, pool: memhub.MemBuddy*): + self.pool = pool + self.root = None + self.project = None + self.source_dir = None + self.includes = None + self.ready = 0 + + def FindProject(self, start: str) -> str: + if start is None or start[0] == 0: + return None + current: str = start + i: int = 0 + while i < 32: + cand: str = os.path.join(current, "project.vpj") + if cand is not None and os.isfile(cand): + return cand + parent: str = os.path.dirname(current) + if parent is None or parent[0] == 0: + break + if string.strcmp(parent, current) == 0: + break + current = parent + i += 1 + return None + + def Init(self, root_uri: str) -> int: + path: str = DecodeUri(self.pool, root_uri) + if path is None or path[0] == 0: + cwd: str = os.getcwd() + if cwd is None: + cwd = "." + path = LspPools.DupStr(self.pool, cwd) + if path is None: + return -1 + self.root = path + self.project = self.FindProject(path) + + if self.project is not None: + old_pool: memhub.MemBuddy* = Config._mbuddy + Config._mbuddy = self.pool + rc: int = Config.Load_project_config(self.project) + if rc == 0: + Config.resolve_paths(os.path.dirname(self.project)) + self.source_dir = Config.SourceDir + self.includes = Config.IncludesDirs + Config._mbuddy = old_pool + + if self.source_dir is None or self.source_dir[0] == 0: + self.source_dir = self.root + + if self.includes is None or self.includes[0] == 0: + sibling: str = os.path.join(self.root, "includes") + if os.isdir(sibling): + self.includes = sibling + else: + parent_sibling: str = os.path.join(self.root, "../includes") + if os.isdir(parent_sibling): + self.includes = parent_sibling + else: + self.includes = "" + self.ready = 1 + return 0 + + def Root(self) -> str: + return self.root + + def ProjectPath(self) -> str: + return self.project + + def SourceDir(self) -> str: + return self.source_dir + + def Includes(self) -> str: + return self.includes diff --git a/App/main.vp b/App/main.vp index bb3b661..d24548e 100644 --- a/App/main.vp +++ b/App/main.vp @@ -227,7 +227,7 @@ Target: %s argparse.release(args) return 2 argparse.release(args) - return LspServer.Run(mb) + return LspServer.Run(mb, perm_pool) # --build-self is a standalone mode: it compiles this project into __vpcbuild_2__, # then launches copyself to replace __vpcbuild__ after a delay. diff --git a/Tools/lsp/probe.py b/Tools/lsp/probe.py index 7bf869f..f16a83f 100644 --- a/Tools/lsp/probe.py +++ b/Tools/lsp/probe.py @@ -10,6 +10,7 @@ import queue import subprocess import sys import threading +from pathlib import Path from typing import Any @@ -138,9 +139,154 @@ def lifecycle(exe: str) -> int: return 0 +def documents(exe: str) -> int: + root = Path(__file__).resolve().parents[2] + root_uri = root.as_uri() + uri = (root / "Tools" / "lsp" / "fixtures" / "gate1.vp").as_uri() + client = Client(exe) + response = client.request( + "initialize", + {"processId": os.getpid(), "rootUri": root_uri, "capabilities": {}}, + ) + result = response.get("result") + if not isinstance(result, dict): + raise AssertionError(f"initialize result is not an object: {response}") + capabilities = result.get("capabilities") + if not isinstance(capabilities, dict): + raise AssertionError(f"missing capabilities: {response}") + if capabilities.get("positionEncoding") != "utf-16": + raise AssertionError(f"positionEncoding is not utf-16: {response}") + if capabilities.get("textDocumentSync") != 2: + raise AssertionError(f"textDocumentSync is not incremental: {response}") + client.notify("initialized", {}) + + text = "def f(a: int) -> int:\r\n return a\r\n\r\n# 中文🙂\r\nx: int = 1\r\n" + client.notify( + "textDocument/didOpen", + {"textDocument": {"uri": uri, "languageId": "viper", "version": 1, "text": text}}, + ) + + stats = client.request("viper/lspStats").get("result") + if not isinstance(stats, dict) or stats.get("documentCount") != 1: + raise AssertionError(f"didOpen was not stored: {stats}") + source_dir = str(stats.get("sourceDir", "")).replace("\\", "/") + if not source_dir.endswith("ViperC/App"): + raise AssertionError(f"workspace sourceDir is wrong: {stats}") + if "includes" not in str(stats.get("includes", "")): + raise AssertionError(f"workspace includes is wrong: {stats}") + if not str(stats.get("project", "")).endswith("project.vpj"): + raise AssertionError(f"workspace project is wrong: {stats}") + + doc = client.request("viper/lspDocument", {"uri": uri}).get("result") + if not isinstance(doc, dict) or doc.get("text") != text or doc.get("version") != 1: + raise AssertionError(f"initial document text/version mismatch: {doc}") + + client.notify( + "textDocument/didChange", + { + "textDocument": {"uri": uri, "version": 2}, + "contentChanges": [ + { + "range": { + "start": {"line": 1, "character": 11}, + "end": {"line": 1, "character": 12}, + }, + "text": "a + 1", + } + ], + }, + ) + doc = client.request("viper/lspDocument", {"uri": uri}).get("result") + if not isinstance(doc, dict) or "return a + 1" not in doc.get("text", ""): + raise AssertionError(f"incremental ASCII edit failed: {doc}") + if doc.get("version") != 2: + raise AssertionError(f"version was not updated: {doc}") + + client.notify( + "textDocument/didChange", + { + "textDocument": {"uri": uri, "version": 3}, + "contentChanges": [ + { + "range": { + "start": {"line": 3, "character": 4}, + "end": {"line": 3, "character": 6}, + }, + "text": "X", + } + ], + }, + ) + doc = client.request("viper/lspDocument", {"uri": uri}).get("result") + if not isinstance(doc, dict) or "# 中文X" not in doc.get("text", ""): + raise AssertionError(f"UTF-16 surrogate edit failed: {doc}") + + replacement = "def f() -> int:\r\n return 42\r\n" + client.notify( + "textDocument/didChange", + { + "textDocument": {"uri": uri, "version": 4}, + "contentChanges": [{"text": replacement}], + }, + ) + doc = client.request("viper/lspDocument", {"uri": uri}).get("result") + if not isinstance(doc, dict) or doc.get("text") != replacement or doc.get("version") != 4: + raise AssertionError(f"full replacement failed: {doc}") + + client.notify("textDocument/didSave", {"textDocument": {"uri": uri}}) + client.notify("textDocument/didClose", {"textDocument": {"uri": uri}}) + stats = client.request("viper/lspStats").get("result") + if not isinstance(stats, dict) or stats.get("documentCount") != 0: + raise AssertionError(f"didClose was not applied: {stats}") + + edit_text = "x = 0\n" + client.notify( + "textDocument/didOpen", + {"textDocument": {"uri": uri, "languageId": "viper", "version": 1, "text": edit_text}}, + ) + for i in range(10000): + digit = "1" if (i % 2) == 0 else "0" + client.notify( + "textDocument/didChange", + { + "textDocument": {"uri": uri, "version": 2 + i}, + "contentChanges": [ + { + "range": { + "start": {"line": 0, "character": 4}, + "end": {"line": 0, "character": 5}, + }, + "text": digit, + } + ], + }, + ) + stats = client.request("viper/lspStats").get("result") + if not isinstance(stats, dict) or stats.get("documentCount") != 1: + raise AssertionError(f"10k edits lost the document: {stats}") + doc = client.request("viper/lspDocument", {"uri": uri}).get("result") + if not isinstance(doc, dict) or doc.get("text") != edit_text: + raise AssertionError(f"10k edits produced unexpected text: {doc}") + if doc.get("version") != 10001: + raise AssertionError(f"10k edits produced unexpected version: {doc}") + if int(stats.get("usedBytes", 0)) > 8 * 1024 * 1024: + raise AssertionError(f"document pool grew unexpectedly: {stats}") + + client.notify("textDocument/didClose", {"textDocument": {"uri": uri}}) + client.request("shutdown") + code = client.close() + if code != 0: + raise AssertionError(f"exit code is {code}, expected 0") + print("documents: OK") + print("positions: OK") + print("10k-edits: OK") + return 0 + + def main() -> int: - parser = argparse.ArgumentParser(description="ViperC LSP probe (Gate 0)") + parser = argparse.ArgumentParser(description="ViperC LSP probe") parser.add_argument("--selfcheck", action="store_true", help="run pure-Python framing checks") + parser.add_argument("--gate1", action="store_true", help="run document/position checks") parser.add_argument("--exe", help="path to viperc.exe") args = parser.parse_args() @@ -152,6 +298,8 @@ def main() -> int: if not os.path.isfile(exe): print(f"probe: executable not found: {exe}", file=sys.stderr) return 2 + if args.gate1: + return documents(exe) return lifecycle(exe)