feat(includes): 软关键字 fn/fun/func/function 与 match 语句判定; 解析 fail-fast 错误记录(get_parse_err_*) + 相邻字符串字面量隐式拼接; ord/chr 内置函数; llvmlite build_alloca_aligned; memhub 超块请求返回 NULL; numpy 显式 string./stdio. 前缀; this 数组迭代修复
This commit is contained in:
TermiNexus committed
1 parent
dd5a639437
commit
a6444a4e1f
9 files changed
+320
-38
No files matched your search
+242
-10
@@ -93,6 +93,110 @@ def _cur_kw(ps: Parser | t.CPtr) -> t.CInt:
|
||||
return ps.cur.kw_subtype
|
||||
|
||||
|
||||
def _is_funcdef_kw(ps: Parser | t.CPtr, kw: t.CInt) -> int:
|
||||
"""判断当前 token 是否为函数定义关键字（def 及其别名 fn/fun/func/function）
|
||||
|
||||
def 是硬关键字直接判定；fn/fun/func/function 是软关键字，
|
||||
仅当下一个 token 是 Name 且再下一个是 '('（或泛型 '[T]' 前的 '['）时
|
||||
判定为函数定义。否则按普通标识符处理，兼容既有代码把 func 用作
|
||||
字段/参数名（如 AST 节点 func 字段、Call.func 注解 `func: AST | t.CPtr`）。
|
||||
"""
|
||||
if kw == Keyword.Def:
|
||||
return 1
|
||||
if kw == Keyword.Fn or kw == Keyword.Fun \
|
||||
or kw == Keyword.Func or kw == Keyword.Function:
|
||||
# 软关键字 lookahead: 下个是 Name，再下个是 '(' 或 '[T...]' 泛型参数（PEP 695）
|
||||
nxt: Token | t.CPtr = ps.cur.next
|
||||
if nxt is None:
|
||||
return 0
|
||||
if nxt.type != TokenType.Name:
|
||||
return 0
|
||||
nxt2: Token | t.CPtr = nxt.next
|
||||
if nxt2 is None:
|
||||
return 0
|
||||
if nxt2.type == TokenType.Op \
|
||||
and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb):
|
||||
return 1
|
||||
return 0
|
||||
return 0
|
||||
|
||||
|
||||
def _is_match_stmt(ps: Parser | t.CPtr) -> int:
|
||||
"""match 是软关键字（CPython 3.10+）：lookahead 判断是否为 match 语句
|
||||
|
||||
match 后直接跟 ':'（变量注解，如 lexer.vp 的 `match: t.CInt = 1`）、
|
||||
'='（赋值，如 `match = 0`）、换行/文件尾 → 按普通标识符；
|
||||
否则扫描至行尾，在括号深度 0 处出现 ':' → match 语句（subject 后必须冒号）。
|
||||
case/type 同理为软关键字（case 仅在 match 套件内消费，type 无语句分发）。
|
||||
"""
|
||||
nxt: Token | t.CPtr = ps.cur.next
|
||||
if nxt is None:
|
||||
return 0
|
||||
if nxt.type == TokenType.NewLine or nxt.type == TokenType.EndMarker:
|
||||
return 0
|
||||
if nxt.type == TokenType.Op:
|
||||
if nxt.op_subtype == TokOp.Colon:
|
||||
return 0
|
||||
if nxt.op_subtype == TokOp.Equal:
|
||||
return 0
|
||||
walk: Token | t.CPtr = nxt
|
||||
depth: t.CInt = 0
|
||||
while walk is not None:
|
||||
if walk.type == TokenType.NewLine or walk.type == TokenType.EndMarker:
|
||||
return 0
|
||||
if walk.type == TokenType.Op:
|
||||
wop: t.CInt = walk.op_subtype
|
||||
if wop == TokOp.LPar or wop == TokOp.Lsqb or wop == TokOp.LBrace:
|
||||
depth += 1
|
||||
elif wop == TokOp.RPar or wop == TokOp.Rsqb or wop == TokOp.RBrace:
|
||||
if depth > 0:
|
||||
depth -= 1
|
||||
elif wop == TokOp.Colon and depth == 0:
|
||||
return 1
|
||||
walk = walk.next
|
||||
return 0
|
||||
|
||||
|
||||
# 解析错误位置与消息（fail-fast 仿 Python SyntaxError 报告，首次错误记录）
|
||||
_g_parse_err_line: t.CInt = 0
|
||||
_g_parse_err_col: t.CInt = 0
|
||||
_g_parse_err_msg: str = None
|
||||
|
||||
|
||||
def _record_parse_err(ps: Parser | t.CPtr):
|
||||
"""记录首个解析错误的位置与消息（只记录第一次，后续错误不覆盖）"""
|
||||
global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg
|
||||
if _g_parse_err_line != 0:
|
||||
return
|
||||
_g_parse_err_line = _cur_lineno(ps)
|
||||
_g_parse_err_col = _cur_col(ps)
|
||||
err_s: str = _cur_str(ps)
|
||||
if err_s is None:
|
||||
err_s = "(none)"
|
||||
# 消息文本用 stdlib 拷贝（token 的 str_val 在 pool 中，生命周期覆盖本次编译但保险起见独立存储）
|
||||
buf_m: str = stdlib.malloc(128)
|
||||
if buf_m is not None:
|
||||
stdio.snprintf(buf_m, 128, "意外的 token '%s'", err_s)
|
||||
_g_parse_err_msg = buf_m
|
||||
else:
|
||||
_g_parse_err_msg = err_s
|
||||
|
||||
|
||||
def get_parse_err_line() -> t.CInt:
|
||||
"""获取最近一次解析错误的行号（0=无错误）"""
|
||||
return _g_parse_err_line
|
||||
|
||||
|
||||
def get_parse_err_col() -> t.CInt:
|
||||
"""获取最近一次解析错误的列号（0=无错误）"""
|
||||
return _g_parse_err_col
|
||||
|
||||
|
||||
def get_parse_err_msg() -> str:
|
||||
"""获取最近一次解析错误的错误消息（仿 Python SyntaxError 消息文本）"""
|
||||
return _g_parse_err_msg
|
||||
|
||||
|
||||
def _cur_str(ps: Parser | t.CPtr) -> str:
|
||||
if ps.cur is None: return None
|
||||
return ps.cur.str_val
|
||||
@@ -128,6 +232,7 @@ def _expect_type(ps: Parser | t.CPtr, ttype: t.CInt) -> t.CInt:
|
||||
if _cur_type(ps) == ttype:
|
||||
_advance(ps)
|
||||
return 1
|
||||
_record_parse_err(ps)
|
||||
ps.error_count += 1
|
||||
return 0
|
||||
|
||||
@@ -137,6 +242,7 @@ def _expect_op(ps: Parser | t.CPtr, op_id: t.CInt) -> t.CInt:
|
||||
if _cur_type(ps) == TokenType.Op and _cur_op(ps) == op_id:
|
||||
_advance(ps)
|
||||
return 1
|
||||
_record_parse_err(ps)
|
||||
ps.error_count += 1
|
||||
return 0
|
||||
|
||||
@@ -187,9 +293,6 @@ def _new_const_float(pool: memhub.MemManager | t.CPtr, val: t.CDouble,
|
||||
_g_dbg_pts[_g_dbg_cnt] = t.CUInt64T(new_c)
|
||||
_g_dbg_vals[_g_dbg_cnt] = val
|
||||
_g_dbg_cnt += 1
|
||||
# NEWF-DBG: 打印浮点常量构造值+节点指针+存储后读回（定位浮点 CDefine 值错误）
|
||||
stdio.printf("[NEWF-DBG] val=%.17g ptr=%p readback=%.17g int_val=%lld\n",
|
||||
val, t.CSizeT(new_c), new_c.float_val, new_c.int_val)
|
||||
return new_c
|
||||
|
||||
|
||||
@@ -387,6 +490,7 @@ def _parse_or_test(ps: Parser | t.CPtr) -> AST | t.CPtr:
|
||||
values.append(left)
|
||||
while _cur_kw(ps) == Keyword.Or:
|
||||
_advance(ps)
|
||||
_skip_nl(ps) # 括号内跨行: a or\n b
|
||||
right: AST | t.CPtr = _parse_and_test(ps)
|
||||
if right is None: break
|
||||
values.append(right)
|
||||
@@ -405,6 +509,7 @@ def _parse_and_test(ps: Parser | t.CPtr) -> AST | t.CPtr:
|
||||
values.append(left)
|
||||
while _cur_kw(ps) == Keyword.And:
|
||||
_advance(ps)
|
||||
_skip_nl(ps) # 括号内跨行: a and\n b
|
||||
right: AST | t.CPtr = _parse_not_test(ps)
|
||||
if right is None: break
|
||||
values.append(right)
|
||||
@@ -440,6 +545,7 @@ def _parse_comparison(ps: Parser | t.CPtr) -> AST | t.CPtr:
|
||||
if op_id == OpKind.NoneOp:
|
||||
return left
|
||||
_advance_comp_op(ps)
|
||||
_skip_nl(ps) # 括号内跨行: a ==\n b
|
||||
right: AST | t.CPtr = _parse_bitor(ps)
|
||||
# Single comparison: construct ops and comparators list
|
||||
ops: list[t.CInt] | t.CPtr = list[t.CInt](ps.pool, 8)
|
||||
@@ -460,6 +566,7 @@ def _parse_comparison(ps: Parser | t.CPtr) -> AST | t.CPtr:
|
||||
cur_left: AST | t.CPtr = right
|
||||
while next_op != OpKind.NoneOp:
|
||||
_advance_comp_op(ps)
|
||||
_skip_nl(ps) # 括号内跨行链式比较
|
||||
cur_right: AST | t.CPtr = _parse_bitor(ps)
|
||||
ops2: list[t.CInt] | t.CPtr = list[t.CInt](ps.pool, 8)
|
||||
ops2.append(next_op)
|
||||
@@ -837,9 +944,30 @@ def _parse_atom(ps: Parser | t.CPtr) -> AST | t.CPtr:
|
||||
is_dquote: t.CInt = ps.cur.kw_subtype & 16
|
||||
is_char: t.CInt = ps.cur.kw_subtype & 32
|
||||
_advance(ps)
|
||||
node: AST | t.CPtr
|
||||
if is_fstr:
|
||||
return _parse_fstring(ps, sval, lineno, col)
|
||||
return _new_const_str(ps.pool, sval, lineno, col, is_dquote, is_triple, is_char)
|
||||
node = _parse_fstring(ps, sval, lineno, col)
|
||||
else:
|
||||
node = _new_const_str(ps.pool, sval, lineno, col, is_dquote, is_triple, is_char)
|
||||
# Python 相邻字符串字面量隐式拼接："a" "b" → "ab"
|
||||
# Nl 是括号内换行 token（允许括号内跨行拼接）；语句级换行是 NewLine，不跳过
|
||||
while True:
|
||||
_skip_nl(ps)
|
||||
if _cur_type(ps) != TokenType.String:
|
||||
break
|
||||
n_sval: str = _cur_str(ps)
|
||||
n_fstr: t.CInt = ps.cur.kw_subtype & 4
|
||||
n_triple: t.CInt = ps.cur.kw_subtype & 8
|
||||
n_dquote: t.CInt = ps.cur.kw_subtype & 16
|
||||
n_char: t.CInt = ps.cur.kw_subtype & 32
|
||||
_advance(ps)
|
||||
n_node: AST | t.CPtr
|
||||
if n_fstr:
|
||||
n_node = _parse_fstring(ps, n_sval, lineno, col)
|
||||
else:
|
||||
n_node = _new_const_str(ps.pool, n_sval, lineno, col, n_dquote, n_triple, n_char)
|
||||
node = _concat_str_nodes(ps.pool, node, n_node)
|
||||
return node
|
||||
|
||||
if _match_op(ps, TokOp.Ellipsis):
|
||||
_advance(ps)
|
||||
@@ -885,10 +1013,12 @@ def _parse_paren(ps: Parser | t.CPtr) -> AST | t.CPtr:
|
||||
break
|
||||
e: AST | t.CPtr = _parse_expr(ps)
|
||||
elts.append(e)
|
||||
_skip_nl(ps) # 末元素无尾逗号时收尾（多行元组）
|
||||
_expect_op(ps, TokOp.RPar)
|
||||
node = Tuple(ps.pool, elts, ASTCtx.Load)
|
||||
_set_pos(node, lineno, col, 0, 0)
|
||||
return node
|
||||
_skip_nl(ps) # (expr\n) 单表达式括号跨行收尾
|
||||
_expect_op(ps, TokOp.RPar)
|
||||
# Check for generator expression
|
||||
if _cur_kw(ps) == Keyword.For:
|
||||
@@ -921,6 +1051,7 @@ def _parse_list(ps: Parser | t.CPtr) -> AST | t.CPtr:
|
||||
break
|
||||
e: AST | t.CPtr = _parse_expr(ps)
|
||||
elts.append(e)
|
||||
_skip_nl(ps) # 末元素无尾逗号时收尾（base64.vp 多行列表）
|
||||
_expect_op(ps, TokOp.Rsqb)
|
||||
node = List(ps.pool, elts, ASTCtx.Load)
|
||||
_set_pos(node, lineno, col, 0, 0)
|
||||
@@ -965,6 +1096,7 @@ def _parse_brace(ps: Parser | t.CPtr) -> AST | t.CPtr:
|
||||
v: AST | t.CPtr = _parse_expr(ps)
|
||||
keys.append(k)
|
||||
values.append(v)
|
||||
_skip_nl(ps) # 末元素无尾逗号时收尾
|
||||
_expect_op(ps, TokOp.RBrace)
|
||||
node = Dict(ps.pool, keys, values)
|
||||
_set_pos(node, lineno, col, 0, 0)
|
||||
@@ -977,10 +1109,12 @@ def _parse_brace(ps: Parser | t.CPtr) -> AST | t.CPtr:
|
||||
elts: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
|
||||
elts.append(first)
|
||||
while _accept_op(ps, TokOp.Comma):
|
||||
_skip_nl(ps) # 括号内跨行集合字面量
|
||||
if _match_op(ps, TokOp.RBrace):
|
||||
break
|
||||
e: AST | t.CPtr = _parse_expr(ps)
|
||||
elts.append(e)
|
||||
_skip_nl(ps) # 末元素无尾逗号时收尾
|
||||
_expect_op(ps, TokOp.RBrace)
|
||||
node = Set(ps.pool, elts)
|
||||
_set_pos(node, lineno, col, 0, 0)
|
||||
@@ -1005,6 +1139,58 @@ def _parse_sub_expr(ps: Parser | t.CPtr, src: str) -> AST | t.CPtr:
|
||||
return _parse_test(sub_ps)
|
||||
|
||||
|
||||
def _append_str_value(pool: memhub.MemManager | t.CPtr,
|
||||
values: list[AST | t.CPtr] | t.CPtr,
|
||||
node: AST | t.CPtr):
|
||||
"""将字符串节点展开加入 values（JoinedStr 展开其 values，其余直接追加）"""
|
||||
if node is None:
|
||||
return
|
||||
if node.kind() == ASTKind.JoinedStr:
|
||||
js_val: JoinedStr | t.CPtr = (JoinedStr | t.CPtr)(node)
|
||||
if js_val.values is not None:
|
||||
n_v: t.CSizeT = js_val.values.__len__()
|
||||
i_v: t.CSizeT
|
||||
for i_v in range(n_v):
|
||||
values.append(js_val.values.get(i_v))
|
||||
else:
|
||||
values.append(node)
|
||||
|
||||
|
||||
def _concat_str_nodes(pool: memhub.MemManager | t.CPtr,
|
||||
a: AST | t.CPtr, b: AST | t.CPtr) -> AST | t.CPtr:
|
||||
"""合并相邻字符串字面量节点（Python 隐式拼接 "a" "b" → "ab"）
|
||||
|
||||
纯常量字符串直接拼接文本；涉及 f-string（JoinedStr/FormattedValue）时
|
||||
合并到 JoinedStr 的 values 列表。
|
||||
"""
|
||||
if a is None:
|
||||
return b
|
||||
if b is None:
|
||||
return a
|
||||
# 两个都是常量字符串 → 直接拼接文本
|
||||
if a.kind() == ASTKind.Constant and b.kind() == ASTKind.Constant:
|
||||
ca_c: Constant | t.CPtr = (Constant | t.CPtr)(a)
|
||||
cb_c: Constant | t.CPtr = (Constant | t.CPtr)(b)
|
||||
if ca_c.const_kind == CONST_STR and cb_c.const_kind == CONST_STR \
|
||||
and ca_c.str_val is not None and cb_c.str_val is not None:
|
||||
la_c: t.CSizeT = string.strlen(ca_c.str_val)
|
||||
lb_c: t.CSizeT = string.strlen(cb_c.str_val)
|
||||
buf_c: str = pool.alloc(la_c + lb_c + 1)
|
||||
if buf_c is not None:
|
||||
string.memcpy(buf_c, ca_c.str_val, la_c)
|
||||
string.memcpy(buf_c + la_c, cb_c.str_val, lb_c + 1)
|
||||
return _new_const_str(pool, buf_c, a.lineno, a.col_offset, 0, 0, 0)
|
||||
# 涉及 f-string → 合并到 JoinedStr
|
||||
values_c: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](pool, 8)
|
||||
_append_str_value(pool, values_c, a)
|
||||
_append_str_value(pool, values_c, b)
|
||||
if values_c.__len__() == 1:
|
||||
return values_c.get(0)
|
||||
node_c: AST | t.CPtr = JoinedStr(pool, values_c)
|
||||
_set_pos(node_c, a.lineno, a.col_offset, 0, 0)
|
||||
return node_c
|
||||
|
||||
|
||||
def _parse_fstring(ps: Parser | t.CPtr, content: str,
|
||||
lineno: t.CInt, col: t.CInt) -> AST | t.CPtr:
|
||||
"""Parse f-string content, building JoinedStr or FormattedValue nodes"""
|
||||
@@ -1375,12 +1561,12 @@ def _parse_statement_into(ps: Parser | t.CPtr, parent: AST | t.CPtr):
|
||||
if stmt is not None:
|
||||
parent.append(stmt)
|
||||
return
|
||||
if kw == Keyword.With or kw == Keyword.Def or kw == Keyword.Class:
|
||||
if kw == Keyword.With or _is_funcdef_kw(ps, kw) != 0 or kw == Keyword.Class:
|
||||
stmt = _parse_compound(ps)
|
||||
if stmt is not None:
|
||||
parent.append(stmt)
|
||||
return
|
||||
if kw == Keyword.Match or kw == Keyword.Async:
|
||||
if (kw == Keyword.Match and _is_match_stmt(ps) != 0) or kw == Keyword.Async:
|
||||
stmt = _parse_compound(ps)
|
||||
if stmt is not None:
|
||||
parent.append(stmt)
|
||||
@@ -1399,7 +1585,45 @@ def _parse_simple_stmt_into(ps: Parser | t.CPtr, parent: AST | t.CPtr):
|
||||
s: AST | t.CPtr = _parse_small_stmt(ps)
|
||||
if s is None: break
|
||||
parent.append(s)
|
||||
# 先跳过 Nl（括号内换行，如多行函数调用参数间的换行，属合法 token）
|
||||
_skip_nl(ps)
|
||||
# 语句结束检查：simple_stmt 必须以换行/文件尾结束。
|
||||
# 残留 token（如相邻表达式 `func12344444 avg3(...)` 被静默拆成多条语句、
|
||||
# `def` 缺失的关键字函数定义残留）是语法错误（fail-fast），禁止静默，
|
||||
# 并仿 Python 报告错误位置（SyntaxError at line/col）。
|
||||
if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker:
|
||||
_record_parse_err(ps)
|
||||
err_str: str = _cur_str(ps)
|
||||
if err_str is None:
|
||||
err_str = "(none)"
|
||||
# 定位：从链表头找 cur，记录前一个 token 与下一个 token 的上下文
|
||||
prev_tok: Token | t.CPtr = None
|
||||
walk_tok: Token | t.CPtr = ps.tokens
|
||||
while walk_tok is not None and walk_tok is not ps.cur:
|
||||
prev_tok = walk_tok
|
||||
walk_tok = walk_tok.next
|
||||
pv_str: str = "(head)"
|
||||
pv_line: t.CInt = 0
|
||||
pv_col: t.CInt = 0
|
||||
if prev_tok is not None:
|
||||
pv_str = prev_tok.str_val
|
||||
pv_line = prev_tok.lineno
|
||||
pv_col = prev_tok.col_offset
|
||||
nxt_tok: Token | t.CPtr = ps.cur.next
|
||||
nxt_str: str = "(none)"
|
||||
nxt_line: t.CInt = 0
|
||||
nxt_col: t.CInt = 0
|
||||
if nxt_tok is not None:
|
||||
nxt_str = nxt_tok.str_val
|
||||
nxt_line = nxt_tok.lineno
|
||||
nxt_col = nxt_tok.col_offset
|
||||
stdio.printf("[AST] SyntaxError: line %d col %d (end %d %d), 意外的 token '%s' (type=%d) | prev='%s' (line %d col %d) | next='%s' (line %d col %d)\n",
|
||||
_g_parse_err_line, _g_parse_err_col, ps.cur.end_lineno, ps.cur.end_col_offset,
|
||||
err_str, _cur_type(ps),
|
||||
pv_str, pv_line, pv_col,
|
||||
nxt_str, nxt_line, nxt_col)
|
||||
ps.error_count += 1
|
||||
return
|
||||
if _cur_type(ps) == TokenType.NewLine:
|
||||
_advance(ps)
|
||||
|
||||
@@ -1774,13 +1998,13 @@ def _parse_compound(ps: Parser | t.CPtr) -> AST | t.CPtr:
|
||||
return _parse_try(ps)
|
||||
if kw == Keyword.With:
|
||||
return _parse_with(ps, 0)
|
||||
if kw == Keyword.Def:
|
||||
if _is_funcdef_kw(ps, kw) != 0:
|
||||
return _parse_funcdef(ps, 0)
|
||||
if kw == Keyword.Class:
|
||||
return _parse_classdef(ps)
|
||||
if kw == Keyword.Async:
|
||||
_advance(ps)
|
||||
if _cur_kw(ps) == Keyword.Def:
|
||||
if _is_funcdef_kw(ps, _cur_kw(ps)) != 0:
|
||||
return _parse_funcdef(ps, 1)
|
||||
if _cur_kw(ps) == Keyword.For:
|
||||
return _parse_for(ps, 1)
|
||||
@@ -1788,7 +2012,7 @@ def _parse_compound(ps: Parser | t.CPtr) -> AST | t.CPtr:
|
||||
return _parse_with(ps, 1)
|
||||
ps.error_count += 1
|
||||
return None
|
||||
if kw == Keyword.Match:
|
||||
if kw == Keyword.Match and _is_match_stmt(ps) != 0:
|
||||
return _parse_match(ps)
|
||||
ps.error_count += 1
|
||||
return None
|
||||
@@ -2460,9 +2684,17 @@ def parse_tokens(pool: memhub.MemManager | t.CPtr,
|
||||
"""Parse an AST Module from a token list."""
|
||||
if pool is None or tokens is None:
|
||||
return None
|
||||
global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg
|
||||
_g_parse_err_line = 0
|
||||
_g_parse_err_col = 0
|
||||
_g_parse_err_msg = None
|
||||
ps: Parser | t.CPtr = new_parser(pool)
|
||||
if ps is None: return None
|
||||
_parser_init(ps, tokens, pool)
|
||||
module_node: Module | t.CPtr = Module(pool)
|
||||
_parse_module_body(ps, module_node)
|
||||
# fail-fast: 解析出错（error_count > 0）时返回 None，由调用方（TPV 等）报错终止，
|
||||
# 避免错误代码静默编译（如 `func12344444 avg3(...)` 缺 def 关键字的函数定义残留）。
|
||||
if ps.error_count > 0:
|
||||
return None
|
||||
return module_node
|
||||
Reference in new issue
Block a user