feat(vast): C 解析器补齐 T | None 与语句级声明

支持 T | None、typedef/define/static/export/extern/inline 变量形态；C 解析失败不再回退纯 Python，直接 fail-fast；同步重建 _parser.pyd。
2026-10-18 19:26:52 +08:00
1 parent 109e71a477
commit c8e49a60ed
@@ -183,13 +183,7 @@ def parse(source: str, filename: str = '<unknown>', mode: str = 'exec', *,
        try:
            return _VAST_C_PARSE(source, mode)
        except SyntaxError as _exc:
-            if _os.environ.get('VAST_USE_C') == 'strict':
-                raise _HumanizeSyntaxError(_exc, source, filename) from None
-            _ReportCFallback(filename, _exc)
-        except Exception as _exc:
-            if _os.environ.get('VAST_USE_C') == 'strict':
-                raise
-            _ReportCFallback(filename, _exc)
+            raise _HumanizeSyntaxError(_exc, source, filename) from None
    try:
        return _ParsePure(source, mode)
    except SyntaxError as _exc:
@@ -2992,6 +2992,93 @@ static void ApplyTypeModifiers(PyObject *Node, PyObject *Markers, PyObject *Attr
    }
 }

+/* 把语句级修饰词追加到注解节点的 Markers（保序去重，对齐 Parser._AddTypeMarkers）。 */
+static void AddTypeMarkers(PyObject *Node, const char *Marker) {
+    if (Node == NULL || Node == Py_None) {
+        return;
+    }
+    if (!PyObject_HasAttrString((PyObject *)Py_TYPE(Node), "Markers")) {
+        return;
+    }
+    PyObject *Existing = PyObject_GetAttrString(Node, "Markers");
+    PyObject *Items = NULL;
+    if (Existing == NULL) {
+        PyErr_Clear();
+        Items = PyTuple_New(0);
+    } else {
+        Items = PySequence_Tuple(Existing);
+        Py_DECREF(Existing);
+    }
+    if (Items == NULL) {
+        PyErr_Clear();
+        return;
+    }
+    PyObject *MarkerStr = PyUnicode_FromString(Marker);
+    if (MarkerStr == NULL) {
+        Py_DECREF(Items);
+        PyErr_Clear();
+        return;
+    }
+    int Hit = PySequence_Contains(Items, MarkerStr);
+    if (Hit != 0) {
+        if (Hit < 0) {
+            PyErr_Clear();
+        }
+        Py_DECREF(Items);
+        Py_DECREF(MarkerStr);
+        return;
+    }
+    Py_ssize_t N = PyTuple_GET_SIZE(Items);
+    PyObject *Merged = PyTuple_New(N + 1);
+    if (Merged == NULL) {
+        Py_DECREF(Items);
+        Py_DECREF(MarkerStr);
+        PyErr_Clear();
+        return;
+    }
+    for (Py_ssize_t I = 0; I < N; I++) {
+        PyObject *Item = PyTuple_GET_ITEM(Items, I);
+        Py_INCREF(Item);
+        PyTuple_SET_ITEM(Merged, I, Item);
+    }
+    PyTuple_SET_ITEM(Merged, N, MarkerStr); /* steals MarkerStr */
+    if (PyObject_SetAttrString(Node, "Markers", Merged) < 0) {
+        PyErr_Clear();
+    }
+    Py_DECREF(Merged);
+    Py_DECREF(Items);
+}
+
+/* 注解联合中的 None 字面量：Constant(None) 或 Name('None')。 */
+static int IsNoneLiteralNode(PyObject *Node) {
+    if (Node == NULL || Node == Py_None) {
+        return 0;
+    }
+    if (NodeIsConstant(Node)) {
+        PyObject *V = PyObject_GetAttrString(Node, "value");
+        int Hit = (V == Py_None);
+        if (V == NULL) {
+            PyErr_Clear();
+        } else {
+            Py_DECREF(V);
+        }
+        return Hit;
+    }
+    if (NodeIs(Node, "Name")) {
+        PyObject *Id = PyObject_GetAttrString(Node, "id");
+        int Hit = 0;
+        if (Id == NULL) {
+            PyErr_Clear();
+        } else {
+            Hit = PyUnicode_Check(Id)
+                && PyUnicode_CompareWithASCIIString(Id, "None") == 0;
+            Py_DECREF(Id);
+        }
+        return Hit;
+    }
+    return 0;
+}
+
 /* 类型注解入口：置注解模式标志（对齐 Python ExprParser._ParseTypeAnnotation）。
 *
 * 值引用参数注解（仅形参位有效，对齐纯 Python 端）：
@@ -3039,6 +3126,42 @@ static PyObject *ParseTypeAnnotation(Parser *P) {
        Py_XDECREF(Attributes);
        return NULL;
    }
+    /* `T | None` / `None | T` 等价于 T；两侧同为 None 或均非 None 仍报错。
+     * 对齐 Pure Python ExprParser._ParseTypeAnnotation 的联合处理。 */
+    if (NodeIs(Node, "BinOp")) {
+        PyObject *Op = PyObject_GetAttrString(Node, "op");
+        int IsBitOr = (Op != NULL && NodeIs(Op, "BitOr"));
+        Py_XDECREF(Op);
+        if (IsBitOr) {
+            PyObject *Left = PyObject_GetAttrString(Node, "left");
+            PyObject *Right = PyObject_GetAttrString(Node, "right");
+            if (Left == NULL || Right == NULL) {
+                PyErr_Clear();
+                Py_XDECREF(Left);
+                Py_XDECREF(Right);
+            } else {
+                int LeftNone = IsNoneLiteralNode(Left);
+                int RightNone = IsNoneLiteralNode(Right);
+                if (LeftNone != RightNone) {
+                    PyObject *Keep = LeftNone ? Right : Left;
+                    Py_INCREF(Keep);
+                    Py_DECREF(Node);
+                    Node = Keep;
+                } else {
+                    SyntaxErrorAt(P, PeekTok(P, 0),
+                        "unsupported type union 'T1 | T2': type annotation must be a single type");
+                    Py_DECREF(Left);
+                    Py_DECREF(Right);
+                    Py_DECREF(Node);
+                    Py_XDECREF(Markers);
+                    Py_XDECREF(Attributes);
+                    return NULL;
+                }
+                Py_DECREF(Left);
+                Py_DECREF(Right);
+            }
+        }
+    }
    if (RefTemp != 0) {
        PyObject *Wrap = MakeTypeWrapperSubscript(
            P, RefTemp == 2 ? "CTempRef" : "CRef", Node, RefTok);
@@ -4800,6 +4923,10 @@ static PyObject *ParseFunctionDef(Parser *P, int IsAsync, PyObject *Decorators);
 static PyObject *ParseClassDef(Parser *P, PyObject *Decorators);
 static PyObject *ParseExprList(Parser *P);
 static PyObject *ParseExternDef(Parser *P, int IsExport, PyObject *Decorators);
+static int IsStorageVarStart(Parser *P);
+static PyObject *ParseStorageVar(Parser *P, const char *Marker);
+static PyObject *ParseDefineVar(Parser *P);
+static PyObject *ParseTypedefVar(Parser *P);
 static PyObject *TryParseAnnAssign(Parser *P);
 static PyObject *ParseExprStmt(Parser *P);
 static PyObject *ParseImport(Parser *P);
@@ -4863,14 +4990,32 @@ static PyObject *ParseStatement(Parser *P) {
    Token *T = PeekTok(P, 0);
    if (T->Kind == TK_NAME) {
        if (TokIsName(T, "extern")) {
+            if (IsStorageVarStart(P)) {
+                return ParseStorageVar(P, "extern");
+            }
            return ParseExternDef(P, 0, NULL);
        }
        if (TokIsName(T, "export")) {
+            if (IsStorageVarStart(P)) {
+                return ParseStorageVar(P, "export");
+            }
            return ParseExternDef(P, 1, NULL);
        }
+        if (TokIsName(T, "static") && IsStorageVarStart(P)) {
+            return ParseStorageVar(P, "static");
+        }
+        if (TokIsName(T, "inline") && IsStorageVarStart(P)) {
+            return ParseStorageVar(P, "inline");
+        }
        if (TokIsName(T, "define") && PeekTok(P, 1)->Kind == TK_NAME) {
+            if (IsStorageVarStart(P)) {
+                return ParseDefineVar(P);
+            }
            return ParseFunctionDef(P, 0, NULL);
        }
+        if (TokIsName(T, "typedef") && PeekTok(P, 1)->Kind == TK_NAME) {
+            return ParseTypedefVar(P);
+        }
        if (TokIsName(T, "virtual") || TokIsName(T, "novirtual")) {
            return ParseVirtualModified(P, NULL);
        }
@@ -5048,7 +5193,7 @@ static PyObject *ParseSimpleStmt(Parser *P) {
    if (Ann != NULL) {
        return Ann;
    }
-    if (PyErr_Occurred()) {
+    if (P->Error || PyErr_Occurred()) {
        return NULL;
    }
    PyObject *Expr = ParseExprList(P);
@@ -5175,6 +5320,145 @@ static int IsCTypedefAnnotation(PyObject *Ann) {
    return 0;
 }

+/* 语句级变量声明判据：`修饰词 名字 :` 或 `修饰词 名字 =`
+ * （对齐 vast Parser._IsStorageVarStart）。 */
+static int IsStorageVarStart(Parser *P) {
+    if (PeekTok(P, 1)->Kind != TK_NAME) {
+        return 0;
+    }
+    Token *Next = PeekTok(P, 2);
+    return Next->Kind == TK_OP && (TokIsOp(Next, ":") || TokIsOp(Next, "="));
+}
+
+/* 构造 AnnAssign(target, annotation, value, simple=1) 并设置位置。 */
+static PyObject *MakeAnnAssignSimple(Parser *P, Token *Start, Token *NameTok,
+                                     PyObject *Annotation, PyObject *Value) {
+    PyObject *Target = MakeNameNode(P, NameTok, "Store");
+    if (Target == NULL) {
+        return NULL;
+    }
+    PyObject *Simple = PyLong_FromLong(1);
+    if (Simple == NULL) {
+        Py_DECREF(Target);
+        return NULL;
+    }
+    PyObject *Argv[4] = { Target, Annotation, Value ? Value : Py_None, Simple };
+    PyObject *Node = NewNode(P, "AnnAssign", Argv, 4);
+    Py_DECREF(Simple);
+    Py_DECREF(Target);
+    if (Node != NULL) {
+        FixLoc(P, Node, Start);
+    }
+    return Node;
+}
+
+/* `static|export|extern|inline 名字: 类型 [= 初值]` → AnnAssign（extern 禁止初值）。
+ * 对齐 vast Parser._ParseStorageVar。 */
+static PyObject *ParseStorageVar(Parser *P, const char *Marker) {
+    Token *Start = NextTok(P); /* 修饰词 */
+    Token *NameTok = ExpectName(P, NULL);
+    if (NameTok == NULL) {
+        return NULL;
+    }
+    if (!AtOp(P, ":")) {
+        SyntaxErrorAt(P, PeekTok(P, 0),
+            "storage variable declaration requires a type annotation");
+        return NULL;
+    }
+    NextTok(P); /* ':' */
+    PyObject *Annotation = ParseTypeAnnotation(P);
+    if (Annotation == NULL) {
+        return NULL;
+    }
+    PyObject *Value = NULL;
+    if (AtOp(P, "=")) {
+        if (strcmp(Marker, "extern") == 0) {
+            SyntaxErrorAt(P, PeekTok(P, 0),
+                "extern variable declaration cannot have an initializer");
+            Py_DECREF(Annotation);
+            return NULL;
+        }
+        NextTok(P); /* '=' */
+        Value = ParseExpr(P);
+        if (Value == NULL) {
+            Py_DECREF(Annotation);
+            return NULL;
+        }
+    }
+    AddTypeMarkers(Annotation, Marker);
+    PyObject *Node = MakeAnnAssignSimple(P, Start, NameTok, Annotation, Value);
+    Py_DECREF(Annotation);
+    Py_XDECREF(Value);
+    return Node;
+}
+
+/* `define 名字[: 类型] = 值` → AnnAssign；未标注类型时注解占位 Name('CDefine')。
+ * 对齐 vast Parser._ParseDefineVar。 */
+static PyObject *ParseDefineVar(Parser *P) {
+    Token *Start = NextTok(P); /* 'define' */
+    Token *NameTok = ExpectName(P, NULL);
+    if (NameTok == NULL) {
+        return NULL;
+    }
+    PyObject *Annotation = NULL;
+    if (AtOp(P, ":")) {
+        NextTok(P);
+        Annotation = ParseTypeAnnotation(P);
+        if (Annotation == NULL) {
+            return NULL;
+        }
+    }
+    if (!AtOp(P, "=")) {
+        SyntaxErrorAt(P, PeekTok(P, 0),
+            "define constant declaration requires an initializer");
+        Py_XDECREF(Annotation);
+        return NULL;
+    }
+    NextTok(P); /* '=' */
+    PyObject *Value = ParseExpr(P);
+    if (Value == NULL) {
+        Py_XDECREF(Annotation);
+        return NULL;
+    }
+    if (Annotation == NULL) {
+        Annotation = MakeTAttrNode(P, "CDefine");
+        if (Annotation == NULL) {
+            Py_DECREF(Value);
+            return NULL;
+        }
+        FixLoc(P, Annotation, Start);
+    }
+    AddTypeMarkers(Annotation, "define");
+    PyObject *Node = MakeAnnAssignSimple(P, Start, NameTok, Annotation, Value);
+    Py_DECREF(Annotation);
+    Py_DECREF(Value);
+    return Node;
+}
+
+/* `typedef 名字 = 类型` → AnnAssign（annotation 与 value 同一节点）。
+ * 对齐 vast Parser._ParseTypedefVar。 */
+static PyObject *ParseTypedefVar(Parser *P) {
+    Token *Start = NextTok(P); /* 'typedef' */
+    Token *NameTok = ExpectName(P, NULL);
+    if (NameTok == NULL) {
+        return NULL;
+    }
+    if (!AtOp(P, "=")) {
+        SyntaxErrorAt(P, PeekTok(P, 0),
+            "typedef syntax error: expected `typedef NAME = type`");
+        return NULL;
+    }
+    NextTok(P); /* '=' */
+    PyObject *TypeNode = ParseTypeAnnotation(P);
+    if (TypeNode == NULL) {
+        return NULL;
+    }
+    AddTypeMarkers(TypeNode, "typedef");
+    PyObject *Node = MakeAnnAssignSimple(P, Start, NameTok, TypeNode, TypeNode);
+    Py_DECREF(TypeNode);
+    return Node;
+}
+
 /* 尝试解析带注解赋值（对齐 vast _TryParseAnnAssign） */
 static PyObject *TryParseAnnAssign(Parser *P) {
    Token *Start = PeekTok(P, 0);