TermiNexus
1968748ea5
Adds LspSigCache: per-file signature cache under <project>/.viperc-lsp-cache, keyed by content hash+size, invalidated on didChange/didSave; resolves from-import, module-attribute and wildcard calls without building the workspace index. Extends LspTypes with list/array/pointer structural checks, array/string initializer lengths and list-literal element checks; adds return-type consistency, augmented-assignment narrowing, constant folding for unary/binop and string/list subscripts. Diagnostics use a per-line occurrence bucket, a dedicated cache work pool and statement/expression budgets so large include files stay responsive. gate13 covers the new rules.
489 lines
15 KiB
Plaintext
489 lines
15 KiB
Plaintext
from stdint import *
|
|
import string
|
|
import stdio
|
|
import os
|
|
import io
|
|
import memhub
|
|
import lib.lsp.LspFiles as LspFiles
|
|
import lib.lsp.LspSource as LspSource
|
|
import lib.lsp.LspDocuments as LspDocuments
|
|
import lib.lsp.LspSymbols as LspSymbols
|
|
import lib.lsp.LspIndex as LspIndex
|
|
import lib.lsp.LspWorkspace as LspWorkspace
|
|
import lib.lsp.LspPools as LspPools
|
|
|
|
|
|
# On-demand cross-file signature cache.
|
|
#
|
|
# Diagnostics must not build the whole workspace symbol index on every publish. This module keeps
|
|
# one small text file per referenced source under `<project>/.viperc-lsp-cache/`, keyed by the
|
|
# file's FNV hash and validated by its size. A cache miss parses only that file with
|
|
# LspSymbols.Analyze, stores its module/class signatures, and returns the requested entry.
|
|
|
|
define LSP_SIG_CACHE_DIR = ".viperc-lsp-cache"
|
|
define LSP_SIG_CACHE_LINE_CAP = 4096
|
|
|
|
|
|
class LspSigEntry:
|
|
file: str
|
|
container: str
|
|
name: str
|
|
kind: int
|
|
detail: str
|
|
|
|
def __new__(self, pool: memhub.MemManager*) -> rawptr:
|
|
p: LspSigEntry* = (LspSigEntry*)(pool.alloc(LspSigEntry.__sizeof__()))
|
|
if p:
|
|
string.memset(p, 0, LspSigEntry.__sizeof__())
|
|
return rawptr(p)
|
|
|
|
|
|
_cache_dir: str = None
|
|
_entries: LspSigEntry** = None
|
|
_entry_count: int = 0
|
|
_work: LspPools.LspPool* = None
|
|
_ready: int = 0
|
|
_map_vals: LspSigEntry** = None
|
|
|
|
define LSP_SIG_MAP_CAP = 262144
|
|
define LSP_SIG_MAP_MASK = 262143
|
|
|
|
|
|
def _HashPath(path: str) -> uint32:
|
|
h: uint32 = uint32(2166136261)
|
|
if path is None:
|
|
return h
|
|
i: sizet = 0
|
|
while path[i] != 0:
|
|
h = (h ^ uint32(path[i])) * uint32(16777619)
|
|
i += 1
|
|
return h
|
|
|
|
|
|
def _HashText(text: str) -> uint32:
|
|
h: uint32 = uint32(2166136261)
|
|
if text is None:
|
|
return h
|
|
i: sizet = 0
|
|
while text[i] != 0:
|
|
h = (h ^ uint32(text[i])) * uint32(16777619)
|
|
i += 1
|
|
return h
|
|
|
|
|
|
def _HexVal(ch: char) -> int:
|
|
if ch >= '0' and ch <= '9':
|
|
return int(ch) - 48
|
|
if ch >= 'a' and ch <= 'f':
|
|
return int(ch) - 87
|
|
if ch >= 'A' and ch <= 'F':
|
|
return int(ch) - 55
|
|
return -1
|
|
|
|
|
|
def _Hex32(pool: memhub.MemManager*, v: uint32) -> str:
|
|
buf: bytes = pool.alloc(sizet(16))
|
|
if buf is None:
|
|
return None
|
|
stdio.snprintf(buf, sizet(16), "%08X", v)
|
|
return str(buf)
|
|
|
|
|
|
def _Sanitize(pool: memhub.MemManager*, s: str) -> str:
|
|
"""Replace line breaks/tabs so one cache line holds one signature."""
|
|
if s is None:
|
|
return ""
|
|
n: sizet = string.strlen(s)
|
|
buf: bytes = pool.alloc(n + 1)
|
|
if buf is None:
|
|
return ""
|
|
i: sizet = 0
|
|
while i < n:
|
|
ch: char = s[i]
|
|
if ch == '\t' or ch == '\n' or ch == '\r':
|
|
buf[i] = ' '
|
|
else:
|
|
buf[i] = ch
|
|
i += 1
|
|
buf[n] = 0
|
|
return str(buf)
|
|
|
|
|
|
def _Dup(pool: memhub.MemManager*, s: str) -> str:
|
|
if s is None:
|
|
return None
|
|
n: sizet = string.strlen(s)
|
|
buf: bytes = pool.alloc(n + 1)
|
|
if buf is None:
|
|
return None
|
|
string.memcpy(buf, s, n + 1)
|
|
return str(buf)
|
|
|
|
|
|
def Ensure(ws: LspWorkspace.LspWorkspace*, table_pool: memhub.MemManager*) -> int:
|
|
global _cache_dir, _entries, _entry_count, _work, _ready, _map_vals
|
|
if _ready != 0:
|
|
return 0
|
|
base: str = None
|
|
if ws is not None:
|
|
project: str = ws.ProjectPath()
|
|
if project is not None:
|
|
base = LspFiles.DirName(table_pool, project)
|
|
if base is None:
|
|
base = ws.SourceDir()
|
|
if base is None:
|
|
base = LspFiles.Normalize(table_pool, os.getcwd())
|
|
if base is None:
|
|
return -1
|
|
cache: str = LspFiles.Join(table_pool, base, LSP_SIG_CACHE_DIR)
|
|
if cache is None:
|
|
return -1
|
|
if os.isdir(cache) == 0:
|
|
os.mkdir(cache)
|
|
_cache_dir = cache
|
|
_entries = (LspSigEntry**)(table_pool.alloc(sizet(65536) * sizet(8)))
|
|
if _entries is None:
|
|
_ready = 0
|
|
return -1
|
|
string.memset(rawptr(_entries), 0, sizet(65536) * sizet(8))
|
|
_entry_count = 0
|
|
_work = LspPools.Create(table_pool, sizet(33554432))
|
|
if _work is None:
|
|
_ready = 0
|
|
return -1
|
|
_ready = 1
|
|
return 0
|
|
|
|
|
|
def _CachePath(pool: memhub.MemManager*, file: str) -> str:
|
|
if _cache_dir is None or file is None:
|
|
return None
|
|
h: str = _Hex32(pool, _HashPath(file))
|
|
if h is None:
|
|
return None
|
|
return LspFiles.Join(pool, _cache_dir, _Cat(pool, h, ".sig"))
|
|
|
|
|
|
def _MapHash(file: str, name: str) -> int:
|
|
return int(_HashPath(file) ^ _HashPath(name)) & LSP_SIG_MAP_MASK
|
|
|
|
|
|
def _InsertMap(e: LspSigEntry*):
|
|
# Linear entry lookup is used; this hook stays for call-site symmetry.
|
|
return
|
|
|
|
|
|
def _RebuildMap():
|
|
return
|
|
|
|
|
|
def _FindEntry(file: str, name: str) -> LspSigEntry*:
|
|
if _entries is None or file is None or name is None:
|
|
return None
|
|
i: int = 0
|
|
while i < _entry_count:
|
|
e: LspSigEntry* = _entries[i]
|
|
i += 1
|
|
if e is None or e.container is not None:
|
|
continue
|
|
if LspFiles.PathEqual(e.file, file) != 0 and string.strcmp(e.name, name) == 0:
|
|
return e
|
|
return None
|
|
|
|
|
|
def _AddEntry(pool: memhub.MemManager*, file: str, container: str, name: str,
|
|
kind: int, detail: str) -> LspSigEntry*:
|
|
global _entry_count
|
|
if _entries is None or name is None:
|
|
return None
|
|
if _entry_count >= 65536:
|
|
return None
|
|
e: LspSigEntry* = LspSigEntry(pool)
|
|
if e is None:
|
|
return None
|
|
e.file = _Dup(pool, file)
|
|
e.container = _Dup(pool, container)
|
|
e.name = _Dup(pool, name)
|
|
e.kind = kind
|
|
e.detail = _Dup(pool, detail)
|
|
_entries[_entry_count] = e
|
|
_entry_count += 1
|
|
_InsertMap(e)
|
|
return e
|
|
|
|
|
|
def _LoadCache(pool: memhub.MemManager*, table_pool: memhub.MemManager*,
|
|
file: str, cache_path: str,
|
|
file_size: int64, file_hash: uint32) -> int:
|
|
if not os.isfile(cache_path):
|
|
return 0
|
|
text: str = LspFiles.ReadSource(pool, cache_path)
|
|
if text is None:
|
|
return 0
|
|
n: sizet = string.strlen(text)
|
|
if n < 12 or string.strncmp(text, "hash=", 5) != 0:
|
|
return 0
|
|
i: sizet = 5
|
|
hv: uint32 = uint32(0)
|
|
while i < n:
|
|
d: int = _HexVal(text[i])
|
|
if d < 0:
|
|
break
|
|
hv = hv * uint32(16) + uint32(d)
|
|
i += 1
|
|
if i >= n or text[i] != ' ':
|
|
return 0
|
|
i += 1
|
|
if i + 5 >= n or string.strncmp(text + i, "size=", 5) != 0:
|
|
return 0
|
|
i += 5
|
|
sz: int64 = 0
|
|
while i < n and text[i] >= '0' and text[i] <= '9':
|
|
sz = sz * 10 + int64(int(text[i]) - 48)
|
|
i += 1
|
|
if hv != file_hash or sz != file_size:
|
|
return 0
|
|
while i < n and text[i] != '\n':
|
|
i += 1
|
|
if i < n:
|
|
i += 1
|
|
while i < n:
|
|
start: sizet = i
|
|
while i < n and text[i] != '\n':
|
|
i += 1
|
|
line_end: sizet = i
|
|
if i < n:
|
|
i += 1
|
|
if line_end <= start:
|
|
continue
|
|
# kind \t container \t name \t detail
|
|
f1: sizet = start
|
|
f2: sizet = f1
|
|
while f1 < line_end and text[f1] != '\t':
|
|
f1 += 1
|
|
f2 = f1 + 1
|
|
c1: sizet = f2
|
|
c2: sizet = c1
|
|
while c2 < line_end and text[c2] != '\t':
|
|
c2 += 1
|
|
n1: sizet = c2 + 1
|
|
n2: sizet = n1
|
|
while n2 < line_end and text[n2] != '\t':
|
|
n2 += 1
|
|
d1: sizet = n2 + 1
|
|
if f1 >= line_end or c2 > line_end or n2 > line_end:
|
|
continue
|
|
kind_txt: str = _CopyRange(table_pool, text, start, f1)
|
|
cont_txt: str = _CopyRange(table_pool, text, f2, c2)
|
|
name_txt: str = _CopyRange(table_pool, text, n1, n2)
|
|
det_txt: str = _CopyRange(table_pool, text, d1, line_end)
|
|
kind: int = string.atoi(kind_txt)
|
|
_AddEntry(table_pool, file, cont_txt if string.strlen(cont_txt) > 0 else None,
|
|
name_txt, kind, det_txt)
|
|
return 1
|
|
|
|
|
|
def _CopyRange(pool: memhub.MemManager*, s: str, a: sizet, b: sizet) -> str:
|
|
if s is None or b < a:
|
|
return ""
|
|
n: sizet = b - a
|
|
buf: bytes = pool.alloc(n + 1)
|
|
if buf is None:
|
|
return ""
|
|
string.memcpy(buf, s + a, n)
|
|
buf[n] = 0
|
|
return str(buf)
|
|
|
|
|
|
def _WriteCache(pool: memhub.MemManager*, file: str, cache_path: str,
|
|
file_size: int64, file_hash: uint32):
|
|
f: io.File* = io.File(cache_path, io.MODE.W)
|
|
if f is None or f.closed:
|
|
return
|
|
hdr: bytes = pool.alloc(sizet(64))
|
|
if hdr is not None:
|
|
stdio.snprintf(hdr, sizet(64), "hash=%08X size=%lld\n", file_hash, file_size)
|
|
hstr: str = str(hdr)
|
|
f.write(hstr, sizet(string.strlen(hstr)))
|
|
i: int = 0
|
|
while i < _entry_count:
|
|
e: LspSigEntry* = _entries[i]
|
|
i += 1
|
|
if e is None or LspFiles.PathEqual(e.file, file) == 0:
|
|
continue
|
|
line: str = _Itoa(pool, int64(e.kind))
|
|
line = _Cat(pool, line, "\t")
|
|
line = _Cat(pool, line, e.container if e.container is not None else "")
|
|
line = _Cat(pool, line, "\t")
|
|
line = _Cat(pool, line, e.name)
|
|
line = _Cat(pool, line, "\t")
|
|
line = _Cat(pool, line, _Sanitize(pool, e.detail))
|
|
if line is not None:
|
|
f.write(line, sizet(string.strlen(line)))
|
|
f.write("\n", sizet(1))
|
|
f.close()
|
|
|
|
|
|
def _Itoa(pool: memhub.MemManager*, v: int64) -> str:
|
|
b: bytes = pool.alloc(sizet(32))
|
|
if b is None:
|
|
return "0"
|
|
stdio.snprintf(b, sizet(32), "%lld", v)
|
|
return str(b)
|
|
|
|
|
|
def _Cat(pool: memhub.MemManager*, a: str, b: str) -> str:
|
|
if a is None:
|
|
return b
|
|
if b is None:
|
|
return a
|
|
n1: sizet = string.strlen(a)
|
|
n2: sizet = string.strlen(b)
|
|
buf: bytes = pool.alloc(n1 + n2 + 1)
|
|
if buf is None:
|
|
return None
|
|
string.memcpy(buf, a, n1)
|
|
string.memcpy(buf + n1, b, n2)
|
|
buf[n1 + n2] = 0
|
|
return str(buf)
|
|
|
|
|
|
def _ParseFile(pool: memhub.MemManager*, table_pool: memhub.MemManager*,
|
|
file: str, cache_path: str, text: str, file_hash: uint32):
|
|
if text is None:
|
|
return
|
|
doc: LspDocuments.LspDocument* = LspSource.NewTempDoc(pool, text)
|
|
if doc is None:
|
|
return
|
|
a: LspSymbols.LspAnalysis* = LspSymbols.Analyze(pool, table_pool, doc, file)
|
|
if a is None:
|
|
return
|
|
i: int = 0
|
|
while i < int(len(a.symbols)):
|
|
s: LspSymbols.LspSymbol* = a.symbols.get(sizet(i))
|
|
i += 1
|
|
if s is None or s.name is None:
|
|
continue
|
|
if s.scope != a.module_scope and (s.parent_sym is None
|
|
or s.parent_sym.kind != LspSymbols.LSP_KIND_CLASS):
|
|
continue
|
|
if s.kind == LspSymbols.LSP_KIND_MODULE:
|
|
continue
|
|
_AddEntry(table_pool, file, s.container, s.name, s.kind, s.detail)
|
|
_WriteCache(pool, file, cache_path, os.getsize(file), file_hash)
|
|
|
|
|
|
def Invalidate(pool: memhub.MemManager*, table_pool: memhub.MemManager*, file: str):
|
|
"""Drop cached signatures for a file (open/edited/saved)."""
|
|
global _entries, _entry_count
|
|
if _entries is not None and file is not None:
|
|
j: int = 0
|
|
i: int = 0
|
|
while i < _entry_count:
|
|
e: LspSigEntry* = _entries[i]
|
|
i += 1
|
|
if e is None or LspFiles.PathEqual(e.file, file) == 0:
|
|
_entries[j] = e
|
|
j += 1
|
|
_entry_count = j
|
|
_RebuildMap()
|
|
if _cache_dir is not None and file is not None:
|
|
cp: str = _CachePath(pool, file)
|
|
if cp is not None and os.isfile(cp):
|
|
os.remove(cp)
|
|
|
|
|
|
def Lookup(pool: memhub.MemManager*, table_pool: memhub.MemManager*,
|
|
ws: LspWorkspace.LspWorkspace*, file: str, name: str) -> LspSigEntry*:
|
|
if file is None or name is None:
|
|
return None
|
|
if Ensure(ws, table_pool) != 0:
|
|
return None
|
|
norm: str = LspFiles.Normalize(table_pool, file)
|
|
e: LspSigEntry* = _FindEntry(norm, name)
|
|
if e is not None:
|
|
return e
|
|
cp: str = _CachePath(pool, norm)
|
|
if cp is None:
|
|
return None
|
|
size: int64 = os.getsize(norm)
|
|
if _work is None:
|
|
return None
|
|
_work.Reset()
|
|
text: str = LspFiles.ReadSource(_work.mb, norm)
|
|
h: uint32 = _HashText(text)
|
|
if _LoadCache(_work.mb, table_pool, norm, cp, size, h) == 0:
|
|
_ParseFile(_work.mb, table_pool, norm, cp, text, h)
|
|
return _FindEntry(norm, name)
|
|
|
|
|
|
def TargetFor(pool: memhub.MemManager*, e: LspSigEntry*) -> LspIndex.LspTarget*:
|
|
if e is None:
|
|
return None
|
|
t: LspIndex.LspTarget* = LspIndex.LspTarget(pool)
|
|
if t is None:
|
|
return None
|
|
t.file = e.file
|
|
t.name = e.name
|
|
t.kind = e.kind
|
|
t.detail = e.detail
|
|
t.container = e.container
|
|
t.has_sel = 0
|
|
return t
|
|
|
|
|
|
def ResolveOccurrence(pool: memhub.MemManager*, table_pool: memhub.MemManager*,
|
|
ws: LspWorkspace.LspWorkspace*, a: LspSymbols.LspAnalysis*,
|
|
occ: LspSymbols.LspOccurrence*) -> LspIndex.LspTarget*:
|
|
"""Resolve an imported function/variable through the signature cache."""
|
|
if a is None or occ is None:
|
|
return None
|
|
if occ.rest is not None and occ.sym is None and occ.head is not None:
|
|
h: LspSymbols.LspSymbol* = occ.head
|
|
rest: str = occ.rest
|
|
dot: int = -1
|
|
i: int = 0
|
|
while i < int(string.strlen(rest)):
|
|
if rest[sizet(i)] == '.':
|
|
dot = i
|
|
break
|
|
i += 1
|
|
if dot >= 0:
|
|
return None # member chains need class layout knowledge
|
|
file: str = None
|
|
if h.import_kind == LspSymbols.LSP_IMPORT_MODULE:
|
|
file = LspIndex._ModuleFile(ws, pool, a.file, h)
|
|
elif h.import_kind == LspSymbols.LSP_IMPORT_FROM:
|
|
file = LspIndex._ResolveFromFile(ws, pool, a.file, h)
|
|
if file is None:
|
|
return None
|
|
e: LspSigEntry* = Lookup(pool, table_pool, ws, file, rest)
|
|
return TargetFor(pool, e)
|
|
if occ.sym is not None:
|
|
s: LspSymbols.LspSymbol* = occ.sym
|
|
if s.import_kind == LspSymbols.LSP_IMPORT_FROM and s.import_attr is not None:
|
|
file: str = LspIndex._ResolveFromFile(ws, pool, a.file, s)
|
|
if file is None:
|
|
return None
|
|
e: LspSigEntry* = Lookup(pool, table_pool, ws, file, s.import_attr)
|
|
return TargetFor(pool, e)
|
|
if s.import_kind == LspSymbols.LSP_IMPORT_MODULE and occ.rest is not None:
|
|
file: str = LspIndex._ModuleFile(ws, pool, a.file, s)
|
|
if file is None:
|
|
return None
|
|
e: LspSigEntry* = Lookup(pool, table_pool, ws, file, occ.rest)
|
|
return TargetFor(pool, e)
|
|
return None
|
|
if occ.name is not None and a.wildcards is not None:
|
|
i: int = 0
|
|
while i < int(len(a.wildcards)):
|
|
w: LspSymbols.LspSymbol* = a.wildcards.get(sizet(i))
|
|
i += 1
|
|
file: str = LspIndex._ResolveFromFile(ws, pool, a.file, w)
|
|
if file is None:
|
|
continue
|
|
e: LspSigEntry* = Lookup(pool, table_pool, ws, file, occ.name)
|
|
if e is not None:
|
|
return TargetFor(pool, e)
|
|
return None
|