diff --git a/lib/vast_c/vast_c.c b/lib/vast_c/vast_c.c index daa06c6..ae04fd4 100644 --- a/lib/vast_c/vast_c.c +++ b/lib/vast_c/vast_c.c @@ -1,7 +1,8 @@ /* vast_c - C 版 Viper/Python 语法解析器(vast 加速层) * - * 阶段 0:最小可行性验证——Tokenizer + 表达式/赋值/模块解析, - * 直接构造 lib.vast.Nodes 的节点类实例,输出与 vast.parse 结构完全一致。 + * 完整 Python 3.12 + Viper 软关键字(fn/fun/func/function → def)解析器, + * 直接构造 lib.vast.Nodes 的节点类实例,输出与 vast.parse 结构完全一致 + * (vast.dump 逐字节相同)。 * * 编译(MSYS2 MinGW gcc,Python 3.12,注意 -l 必须在源文件之后): * gcc -shared -O2 -std=c11 -I D:\Python312\include lib\vast_c\vast_c.c \ @@ -19,33 +20,68 @@ #include /* ================= 记号(Token) ================= */ + typedef enum { - TK_EOF = 0, + TK_ENDMARKER = 0, TK_NAME, TK_NUMBER, - TK_EQUAL, - TK_LPAREN, - TK_RPAREN, + TK_STRING, + TK_FSTRING, + TK_OP, TK_NEWLINE, + TK_NL, + TK_INDENT, + TK_DEDENT, } TokenKind; typedef struct { TokenKind Kind; - const char *Start; - Py_ssize_t Len; - int Line; - int Col; + const char *Start; /* 指向源码(UTF-8 字节) */ + const char *End; + int Line; /* 1-based 行号 */ + int Col; /* 0-based UTF-8 字节列偏移 */ + int EndLine; + int EndCol; + int StartOff; /* 源码字节偏移 */ + int EndOff; } Token; -/* ================= 解析器(Parser) ================= */ +/* ================= 词法分析器(Lexer) ================= */ + typedef struct { const char *Src; Py_ssize_t Len; Py_ssize_t Pos; - int Line; - int Col; - Token Cur; + int BracketDepth; + int AtLineStart; + int ContinuationLine; + int BlankLine; + int *Indents; + int NIndents; + int IndentsCap; + Token *Tokens; + int NTokens; + int TokensCap; + int *LineStarts; /* 行起始字节偏移表 */ + int NLineStarts; + int LineStartsCap; int Error; +} Lexer; + +/* ================= 解析器(Parser) ================= */ + +typedef struct { + const char *Src; + Py_ssize_t Len; + Token *Tokens; + int NTokens; + int Idx; + int *LineStarts; + int NLineStarts; + int Error; /* 语法错误标志(可回退) */ + const char *ErrMsg; + int ErrLine; + int FStringExprEnd; /* f-string 表达式段结束位置(body 内偏移) */ } Parser; /* vast.nodes 模块与节点类缓存(进程级惰性初始化) */ @@ -93,7 +129,7 @@ static PyObject *ClassGet(Parser *P, const char *Name) { } /* 构造节点:按位置参数数组调用类(vast.nodes 的 __init__(*args, **kwargs)) */ -static PyObject *NodeMake(Parser *P, const char *Name, PyObject **Args, Py_ssize_t NArgs) { +static PyObject *NewNode(Parser *P, const char *Name, PyObject **Args, Py_ssize_t NArgs) { PyObject *Cls = ClassGet(P, Name); if (Cls == NULL) { return NULL; @@ -101,28 +137,64 @@ static PyObject *NodeMake(Parser *P, const char *Name, PyObject **Args, Py_ssize return PyObject_Vectorcall(Cls, Args, NArgs, NULL); } -/* 设置位置属性(vast.parse 默认携带 lineno/col_offset/end_lineno/end_col_offset) */ -static void NodePos(Parser *P, PyObject *Node, int Line, int Col) { - PyObject *LineObj = PyLong_FromLong(Line); - PyObject *ColObj = PyLong_FromLong(Col); - PyObject *EndLineObj = PyLong_FromLong(P->Line); - PyObject *EndColObj = PyLong_FromLong(P->Col); - if (LineObj && ColObj && EndLineObj && EndColObj) { - PyObject_SetAttrString(Node, "lineno", LineObj); - PyObject_SetAttrString(Node, "col_offset", ColObj); - PyObject_SetAttrString(Node, "end_lineno", EndLineObj); - PyObject_SetAttrString(Node, "end_col_offset", EndColObj); +/* 设置整数属性;失败(无槽位)时清除异常 */ +static void SetAttrLong(PyObject *Node, const char *Name, long V) { + PyObject *O = PyLong_FromLong(V); + if (O == NULL) { + PyErr_Clear(); + return; } - Py_XDECREF(LineObj); - Py_XDECREF(ColObj); - Py_XDECREF(EndLineObj); - Py_XDECREF(EndColObj); + if (PyObject_SetAttrString(Node, Name, O) < 0) { + PyErr_Clear(); + } + Py_DECREF(O); } -/* ================= Tokenizer ================= */ +/* 设置 4 个位置属性 */ +static void NodePos(Parser *P, PyObject *Node, int Line, int Col, int EndLine, int EndCol) { + SetAttrLong(Node, "lineno", Line); + SetAttrLong(Node, "col_offset", Col); + SetAttrLong(Node, "end_lineno", EndLine); + SetAttrLong(Node, "end_col_offset", EndCol); +} + +/* 仅设置 lineno/col_offset(f-string 内部字面量常量用) */ +static void NodePosNoEnd(Parser *P, PyObject *Node, int Line, int Col) { + SetAttrLong(Node, "lineno", Line); + SetAttrLong(Node, "col_offset", Col); +} + +/* 前一个 token(借用的 _Prev 语义) */ +static Token *PrevTok(Parser *P) { + int I = P->Idx > 0 ? P->Idx - 1 : 0; + return &P->Tokens[I]; +} + +/* FixLoc:start 用给定 token,end 缺省取前一个 token(vast _FixLoc 语义) */ +static void FixLocEnd(Parser *P, PyObject *Node, Token *Start, Token *End) { + if (End == NULL) { + End = PrevTok(P); + } + NodePos(P, Node, Start->Line, Start->Col, End->EndLine, End->EndCol); +} + +static void FixLoc(Parser *P, PyObject *Node, Token *Start) { + FixLocEnd(P, Node, Start, NULL); +} + +/* 节点类名(动态 type() 创建的类 tp_name 即类名) */ +static const char *TypeName(PyObject *Node) { + return Py_TYPE(Node)->tp_name; +} + +static int NodeIs(PyObject *Node, const char *Name) { + return strcmp(Py_TYPE(Node)->tp_name, Name) == 0; +} + +/* ================= 字符分类 ================= */ static int IsNameStart(int C) { - return (C >= 'a' && C <= 'z') || (C >= 'A' && C <= 'Z') || C == '_'; + return (C >= 'a' && C <= 'z') || (C >= 'A' && C <= 'Z') || C == '_' || C >= 0x80; } static int IsNameChar(int C) { @@ -133,296 +205,5954 @@ static int IsDigit(int C) { return C >= '0' && C <= '9'; } -static void NextToken(Parser *P) { - /* 跳过空白(不含换行) */ - while (P->Pos < P->Len) { - char C = P->Src[P->Pos]; - if (C == ' ' || C == '\t' || C == '\r') { - P->Pos++; - P->Col++; +static int IsAlnum(int C) { + return (C >= 'a' && C <= 'z') || (C >= 'A' && C <= 'Z') || (C >= '0' && C <= '9'); +} + +static int IsStringPrefixChar(int C) { + return C == 'r' || C == 'R' || C == 'b' || C == 'B' || C == 'u' || C == 'U' + || C == 'f' || C == 'F'; +} + +/* ================= 行起始表 ================= */ + +/* bisect_right(LineStarts, Off) - 1 → (行号, 字节列) */ +static void PosInfoLS(const int *LineStarts, int N, int Off, int *Line, int *Col) { + int Lo = 0, Hi = N - 1, Ans = 0; + while (Lo <= Hi) { + int Mid = (Lo + Hi) / 2; + if (LineStarts[Mid] <= Off) { + Ans = Mid; + Lo = Mid + 1; + } else { + Hi = Mid - 1; + } + } + *Line = Ans + 1; + *Col = Off - LineStarts[Ans]; +} + +/* ================= Lexer 基础 ================= */ + +static void RegisterNewline(Lexer *L, int Pos) { + if (L->NLineStarts >= L->LineStartsCap) { + int NCap = L->LineStartsCap ? L->LineStartsCap * 2 : 32; + int *NS = (int *)PyMem_Realloc(L->LineStarts, (size_t)NCap * sizeof(int)); + if (NS == NULL) { + L->Error = 1; + return; + } + L->LineStarts = NS; + L->LineStartsCap = NCap; + } + L->LineStarts[L->NLineStarts++] = Pos; +} + +static void EmitToken(Lexer *L, TokenKind Kind, int StartOff, int EndOff) { + if (L->NTokens >= L->TokensCap) { + int NCap = L->TokensCap ? L->TokensCap * 2 : 128; + Token *NT = (Token *)PyMem_Realloc(L->Tokens, (size_t)NCap * sizeof(Token)); + if (NT == NULL) { + L->Error = 1; + return; + } + L->Tokens = NT; + L->TokensCap = NCap; + } + Token *T = &L->Tokens[L->NTokens]; + T->Kind = Kind; + T->Start = L->Src + StartOff; + T->End = L->Src + EndOff; + T->StartOff = StartOff; + T->EndOff = EndOff; + PosInfoLS(L->LineStarts, L->NLineStarts, StartOff, &T->Line, &T->Col); + PosInfoLS(L->LineStarts, L->NLineStarts, EndOff, &T->EndLine, &T->EndCol); + L->NTokens++; +} + +/* 运算符最长匹配(与 vast Tokenizer._OPERATORS 一致) */ +static int MatchOperator(const char *S, Py_ssize_t N) { + if (N <= 0) { + return 0; + } + char C = S[0]; + switch (C) { + case '*': + if (N >= 3 && S[1] == '*' && S[2] == '=') return 3; + if (N >= 2 && S[1] == '*') return 2; + if (N >= 2 && S[1] == '=') return 2; + return 1; + case '/': + if (N >= 3 && S[1] == '/' && S[2] == '=') return 3; + if (N >= 2 && S[1] == '/') return 2; + if (N >= 2 && S[1] == '=') return 2; + return 1; + case '<': + if (N >= 3 && S[1] == '<' && S[2] == '=') return 3; + if (N >= 2 && S[1] == '<') return 2; + if (N >= 2 && S[1] == '=') return 2; + return 1; + case '>': + if (N >= 3 && S[1] == '>' && S[2] == '=') return 3; + if (N >= 2 && S[1] == '>') return 2; + if (N >= 2 && S[1] == '=') return 2; + return 1; + case '=': + return (N >= 2 && S[1] == '=') ? 2 : 1; + case '!': + return (N >= 2 && S[1] == '=') ? 2 : 1; + case ':': + return (N >= 2 && S[1] == '=') ? 2 : 1; + case '-': + if (N >= 2 && S[1] == '>') return 2; + if (N >= 2 && S[1] == '=') return 2; + return 1; + case '+': + return (N >= 2 && S[1] == '=') ? 2 : 1; + case '%': + return (N >= 2 && S[1] == '=') ? 2 : 1; + case '@': + return (N >= 2 && S[1] == '=') ? 2 : 1; + case '&': + return (N >= 2 && S[1] == '=') ? 2 : 1; + case '|': + return (N >= 2 && S[1] == '=') ? 2 : 1; + case '^': + return (N >= 2 && S[1] == '=') ? 2 : 1; + case '.': + return (N >= 3 && S[1] == '.' && S[2] == '.') ? 3 : 1; + default: + if (C == '(' || C == ')' || C == '[' || C == ']' || C == '{' || C == '}' + || C == ',' || C == ';' || C == '~') { + return 1; + } + return 0; + } +} + +static void EmitOp(Lexer *L) { + int Start = (int)L->Pos; + int Len = MatchOperator(L->Src + L->Pos, L->Len - L->Pos); + if (Len <= 0) { + Len = 1; + } + L->Pos += Len; + EmitToken(L, TK_OP, Start, (int)L->Pos); + L->AtLineStart = 0; +} + +static void HandleNewline(Lexer *L) { + int NlStart = (int)L->Pos - 1; + if (NlStart > 0 && L->Src[NlStart] == '\n' && L->Src[NlStart - 1] == '\r') { + NlStart--; + } + if (L->BracketDepth > 0 || L->BlankLine + || (L->AtLineStart && !L->ContinuationLine)) { + EmitToken(L, TK_NL, NlStart, (int)L->Pos); + } else { + EmitToken(L, TK_NEWLINE, NlStart, (int)L->Pos); + } + RegisterNewline(L, (int)L->Pos); + L->AtLineStart = 1; + L->ContinuationLine = 0; + L->BlankLine = 0; +} + +static void HandleIndent(Lexer *L) { + int Start = (int)L->Pos; + int Col = 0; + while (L->Pos < L->Len) { + char Ch = L->Src[L->Pos]; + if (Ch == ' ') { + Col++; + L->Pos++; + } else if (Ch == '\t') { + Col = (Col / 8 + 1) * 8; + L->Pos++; + } else if (Ch == '\f') { + L->Pos++; } else { break; } } - Token T; - T.Start = P->Src + P->Pos; - T.Len = 0; - T.Line = P->Line; - T.Col = P->Col; - if (P->Pos >= P->Len) { - T.Kind = TK_EOF; - P->Cur = T; + L->AtLineStart = 0; + if (L->Pos >= L->Len || L->Src[L->Pos] == '\r' || L->Src[L->Pos] == '\n' + || L->Src[L->Pos] == '#') { + L->BlankLine = 1; return; } - char C = P->Src[P->Pos]; - if (C == '\n') { - T.Kind = TK_NEWLINE; - T.Len = 1; - P->Pos++; - P->Line++; - P->Col = 0; - } else if (IsNameStart(C)) { - Py_ssize_t Start = P->Pos; - while (P->Pos < P->Len && IsNameChar(P->Src[P->Pos])) { - P->Pos++; + int Cur = L->Indents[L->NIndents - 1]; + if (Col > Cur) { + if (L->NIndents >= L->IndentsCap) { + int NCap = L->IndentsCap ? L->IndentsCap * 2 : 16; + int *NI = (int *)PyMem_Realloc(L->Indents, (size_t)NCap * sizeof(int)); + if (NI == NULL) { + L->Error = 1; + return; + } + L->Indents = NI; + L->IndentsCap = NCap; } - T.Kind = TK_NAME; - T.Len = P->Pos - Start; - P->Col += (int)T.Len; - } else if (IsDigit(C)) { - Py_ssize_t Start = P->Pos; - while (P->Pos < P->Len && (IsDigit(P->Src[P->Pos]) || P->Src[P->Pos] == '.')) { - P->Pos++; - } - T.Kind = TK_NUMBER; - T.Len = P->Pos - Start; - P->Col += (int)T.Len; - } else { - switch (C) { - case '=': - T.Kind = TK_EQUAL; - T.Len = 1; - P->Pos++; - P->Col++; - break; - case '(': - T.Kind = TK_LPAREN; - T.Len = 1; - P->Pos++; - P->Col++; - break; - case ')': - T.Kind = TK_RPAREN; - T.Len = 1; - P->Pos++; - P->Col++; - break; - default: - T.Kind = TK_EOF; - P->Error = 1; - break; + L->Indents[L->NIndents++] = Col; + EmitToken(L, TK_INDENT, Start, (int)L->Pos); + } else if (Col < Cur) { + while (L->NIndents > 0 && L->Indents[L->NIndents - 1] > Col) { + L->NIndents--; + EmitToken(L, TK_DEDENT, (int)L->Pos, (int)L->Pos); } } - P->Cur = T; } -/* ================= Parser ================= */ +static void ScanComment(Lexer *L) { + while (L->Pos < L->Len && L->Src[L->Pos] != '\r' && L->Src[L->Pos] != '\n') { + L->Pos++; + } +} -static PyObject *ParseExpression(Parser *P); +static void ScanContinuation(Lexer *L) { + if (L->Pos + 1 < L->Len + && (L->Src[L->Pos + 1] == '\r' || L->Src[L->Pos + 1] == '\n')) { + L->Pos += 1; + if (L->Pos < L->Len && L->Src[L->Pos] == '\r') { + L->Pos += 1; + } + if (L->Pos < L->Len && L->Src[L->Pos] == '\n') { + L->Pos += 1; + } + RegisterNewline(L, (int)L->Pos); + L->AtLineStart = 1; + L->ContinuationLine = 1; + } else { + L->Pos += 1; + } +} -/* 原子表达式:数字常量 / 名字 */ -static PyObject *ParsePrimary(Parser *P) { - if (P->Cur.Kind == TK_NUMBER) { - const char *Text = P->Cur.Start; - Py_ssize_t Len = P->Cur.Len; - int Line = P->Cur.Line; - int Col = P->Cur.Col; +static void ScanName(Lexer *L) { + int Start = (int)L->Pos; + while (L->Pos < L->Len && IsNameChar((unsigned char)L->Src[L->Pos])) { + L->Pos++; + } + EmitToken(L, TK_NAME, Start, (int)L->Pos); + L->AtLineStart = 0; +} + +static void ScanNumber(Lexer *L) { + int Start = (int)L->Pos; + if (L->Src[L->Pos] == '0' && L->Pos + 1 < L->Len) { + char Nxt = L->Src[L->Pos + 1]; + if (Nxt == 'x' || Nxt == 'X' || Nxt == 'o' || Nxt == 'O' + || Nxt == 'b' || Nxt == 'B') { + L->Pos += 2; + while (L->Pos < L->Len + && (IsAlnum((unsigned char)L->Src[L->Pos]) || L->Src[L->Pos] == '_')) { + L->Pos++; + } + EmitToken(L, TK_NUMBER, Start, (int)L->Pos); + L->AtLineStart = 0; + return; + } + } + while (L->Pos < L->Len + && (IsDigit((unsigned char)L->Src[L->Pos]) || L->Src[L->Pos] == '_')) { + L->Pos++; + } + if (L->Pos < L->Len && L->Src[L->Pos] == '.') { + L->Pos++; + while (L->Pos < L->Len + && (IsDigit((unsigned char)L->Src[L->Pos]) || L->Src[L->Pos] == '_')) { + L->Pos++; + } + } + if (L->Pos < L->Len && (L->Src[L->Pos] == 'e' || L->Src[L->Pos] == 'E')) { + Py_ssize_t Save = L->Pos; + L->Pos++; + if (L->Pos < L->Len && (L->Src[L->Pos] == '+' || L->Src[L->Pos] == '-')) { + L->Pos++; + } + if (L->Pos < L->Len + && (IsDigit((unsigned char)L->Src[L->Pos]) || L->Src[L->Pos] == '_')) { + while (L->Pos < L->Len + && (IsDigit((unsigned char)L->Src[L->Pos]) || L->Src[L->Pos] == '_')) { + L->Pos++; + } + } else { + L->Pos = Save; + } + } + if (L->Pos < L->Len && (L->Src[L->Pos] == 'j' || L->Src[L->Pos] == 'J')) { + L->Pos++; + } + EmitToken(L, TK_NUMBER, Start, (int)L->Pos); + L->AtLineStart = 0; +} + +static void SkipInnerString(Lexer *L); +static void ScanFStringExpr(Lexer *L); + +static void ScanStringBody(Lexer *L, char Quote, int Triple, int IsFString) { + while (L->Pos < L->Len) { + char Ch = L->Src[L->Pos]; + if (Ch == '\\') { + if (L->Pos + 1 < L->Len + && (L->Src[L->Pos + 1] == '{' || L->Src[L->Pos + 1] == '}')) { + L->Pos += 1; + } else { + L->Pos += 2; + } + continue; + } + if (Ch == '\n') { + L->Pos++; + RegisterNewline(L, (int)L->Pos); + continue; + } + if (IsFString && Ch == '{') { + if (L->Pos + 1 < L->Len && L->Src[L->Pos + 1] == '{') { + L->Pos += 2; + continue; + } + ScanFStringExpr(L); + continue; + } + if (IsFString && Ch == '}') { + if (L->Pos + 1 < L->Len && L->Src[L->Pos + 1] == '}') { + L->Pos += 2; + continue; + } + L->Pos += 1; + continue; + } + if (Ch == Quote) { + if (Triple) { + if (L->Pos + 2 < L->Len && L->Src[L->Pos] == Quote + && L->Src[L->Pos + 1] == Quote && L->Src[L->Pos + 2] == Quote) { + L->Pos += 3; + return; + } + L->Pos += 1; + continue; + } else { + L->Pos += 1; + return; + } + } + L->Pos += 1; + } +} + +static void ScanFStringExpr(Lexer *L) { + L->Pos += 1; /* 跳过 { */ + int Depth = 1; + while (L->Pos < L->Len) { + char Ch = L->Src[L->Pos]; + if (Ch == '\\') { + L->Pos += 2; + continue; + } + if (Ch == '\n') { + L->Pos++; + RegisterNewline(L, (int)L->Pos); + continue; + } + if (Ch == '\'' || Ch == '"') { + SkipInnerString(L); + continue; + } + if (Ch == '(' || Ch == '[' || Ch == '{') { + Depth++; + L->Pos++; + continue; + } + if (Ch == ')' || Ch == ']') { + Depth--; + L->Pos++; + continue; + } + if (Ch == '}') { + if (Depth == 1) { + L->Pos++; + return; + } + Depth--; + L->Pos++; + continue; + } + L->Pos++; + } +} + +static void SkipInnerString(Lexer *L) { + Py_ssize_t Save = L->Pos; + while (L->Pos < L->Len && IsStringPrefixChar((unsigned char)L->Src[L->Pos])) { + L->Pos++; + } + if (L->Pos >= L->Len || (L->Src[L->Pos] != '\'' && L->Src[L->Pos] != '"')) { + L->Pos = Save; + L->Pos += 1; + return; + } + char InnerQuote = L->Src[L->Pos]; + int Triple = (L->Pos + 2 < L->Len && L->Src[L->Pos] == InnerQuote + && L->Src[L->Pos + 1] == InnerQuote && L->Src[L->Pos + 2] == InnerQuote); + if (Triple) { + L->Pos += 3; + } else { + L->Pos += 1; + } + ScanStringBody(L, InnerQuote, Triple, 0); +} + +static int TryScanString(Lexer *L) { + Py_ssize_t Start = L->Pos; + while (L->Pos < L->Len && IsStringPrefixChar((unsigned char)L->Src[L->Pos])) { + L->Pos++; + } + if (L->Pos >= L->Len || (L->Src[L->Pos] != '\'' && L->Src[L->Pos] != '"')) { + L->Pos = Start; + return 0; + } + int IsFString = 0; + for (Py_ssize_t K = Start; K < L->Pos; K++) { + char C = L->Src[K]; + if (C == 'f' || C == 'F') { + IsFString = 1; + } + } + char Quote = L->Src[L->Pos]; + int Triple = (L->Pos + 2 < L->Len && L->Src[L->Pos] == Quote + && L->Src[L->Pos + 1] == Quote && L->Src[L->Pos + 2] == Quote); + if (Triple) { + L->Pos += 3; + } else { + L->Pos += 1; + } + ScanStringBody(L, Quote, Triple, IsFString); + EmitToken(L, IsFString ? TK_FSTRING : TK_STRING, (int)Start, (int)L->Pos); + L->AtLineStart = 0; + return 1; +} + +static void FinishFile(Lexer *L) { + if (L->NTokens > 0) { + TokenKind K = L->Tokens[L->NTokens - 1].Kind; + if (K != TK_NEWLINE && K != TK_DEDENT && K != TK_INDENT) { + EmitToken(L, TK_NEWLINE, (int)L->Pos, (int)L->Pos); + } + } + while (L->NIndents > 1) { + L->NIndents--; + EmitToken(L, TK_DEDENT, (int)L->Pos, (int)L->Pos); + } + EmitToken(L, TK_ENDMARKER, (int)L->Pos, (int)L->Pos); +} + +/* 词法分析主循环(对齐 vast Tokenizer.Tokenize) */ +static void TokenizeMainLoop(Lexer *L) { + while (L->Pos < L->Len) { + unsigned char Ch = (unsigned char)L->Src[L->Pos]; + if (L->AtLineStart && L->BracketDepth == 0 && !L->ContinuationLine + && Ch != '\r' && Ch != '\n') { + HandleIndent(L); + if (L->Pos >= L->Len) { + break; + } + continue; + } + if (Ch == ' ' || Ch == '\t' || Ch == '\f') { + L->Pos++; + continue; + } + if (Ch == '\r') { + L->Pos++; + if (L->Pos < L->Len && L->Src[L->Pos] == '\n') { + L->Pos++; + } + HandleNewline(L); + continue; + } + if (Ch == '\n') { + L->Pos++; + HandleNewline(L); + continue; + } + if (Ch == '#') { + ScanComment(L); + continue; + } + if (Ch == '\\') { + ScanContinuation(L); + continue; + } + if (Ch == '(' || Ch == '[' || Ch == '{') { + L->BracketDepth++; + EmitOp(L); + continue; + } + if (Ch == ')' || Ch == ']' || Ch == '}') { + if (L->BracketDepth > 0) { + L->BracketDepth--; + } + EmitOp(L); + continue; + } + if (Ch == '\'' || Ch == '"' || IsNameStart(Ch)) { + if (TryScanString(L)) { + continue; + } + if (IsDigit(Ch)) { + ScanNumber(L); + continue; + } + if (IsNameStart(Ch)) { + ScanName(L); + continue; + } + } + if (IsDigit(Ch) || (Ch == '.' && L->Pos + 1 < L->Len + && IsDigit((unsigned char)L->Src[L->Pos + 1]))) { + ScanNumber(L); + continue; + } + EmitOp(L); + } +} + +/* 对任意文本执行完整 tokenize。 + * OutLineStarts/OutNLineStarts 可为 NULL(子表达式不需要)。 + * 返回 1 成功,0 失败(内存不足)。 */ +static int TokenizeText(const char *Src, Py_ssize_t Len, + Token **OutTokens, int *OutN, + int **OutLineStarts, int *OutNLineStarts) { + Lexer L; + memset(&L, 0, sizeof(L)); + L.Src = Src; + L.Len = Len; + L.Pos = 0; + L.AtLineStart = 1; + L.Indents = (int *)PyMem_Malloc(sizeof(int)); + if (L.Indents == NULL) { + return 0; + } + L.Indents[0] = 0; + L.NIndents = 1; + L.IndentsCap = 1; + RegisterNewline(&L, 0); + TokenizeMainLoop(&L); + if (!L.Error) { + FinishFile(&L); + } + *OutTokens = L.Tokens; + *OutN = L.NTokens; + if (OutLineStarts) { + *OutLineStarts = L.LineStarts; + *OutNLineStarts = L.NLineStarts; + } + PyMem_Free(L.Indents); + if (OutLineStarts == NULL) { + PyMem_Free(L.LineStarts); + } + return !L.Error; +} + +/* ================= Parser 游标辅助 ================= */ + +static Token *PeekTok(Parser *P, int Off) { + int I = P->Idx + Off; + if (I >= P->NTokens) { + I = P->NTokens - 1; + } + if (I < 0) { + I = 0; + } + return &P->Tokens[I]; +} + +static Token *NextTok(Parser *P) { + Token *T = &P->Tokens[P->Idx]; + if (P->Idx < P->NTokens - 1) { + P->Idx++; + } + return T; +} + +static int TokIsOp(Token *T, const char *S) { + if (T->Kind != TK_OP) { + return 0; + } + Py_ssize_t L = (Py_ssize_t)strlen(S); + return (T->End - T->Start == L) && memcmp(T->Start, S, (size_t)L) == 0; +} + +static int TokIsName(Token *T, const char *S) { + if (T->Kind != TK_NAME) { + return 0; + } + Py_ssize_t L = (Py_ssize_t)strlen(S); + return (T->End - T->Start == L) && memcmp(T->Start, S, (size_t)L) == 0; +} + +static int AtOp(Parser *P, const char *S) { + return TokIsOp(PeekTok(P, 0), S); +} + +static int AtName(Parser *P, const char *S) { + return TokIsName(PeekTok(P, 0), S); +} + +static void SkipNL(Parser *P) { + while (PeekTok(P, 0)->Kind == TK_NL) { + NextTok(P); + } +} + +static void SkipNEWLINE(Parser *P) { + while (PeekTok(P, 0)->Kind == TK_NEWLINE || PeekTok(P, 0)->Kind == TK_NL) { + NextTok(P); + } +} + +static void SyntaxErrorAt(Parser *P, Token *T, const char *Msg) { + P->Error = 1; + P->ErrMsg = Msg; + P->ErrLine = T ? T->Line : 0; +} + +static Token *ExpectOp(Parser *P, const char *S) { + Token *T = NextTok(P); + if (!TokIsOp(T, S)) { + SyntaxErrorAt(P, T, "expected operator"); + return NULL; + } + return T; +} + +static Token *ExpectName(Parser *P, const char *Value) { + Token *T = NextTok(P); + if (T->Kind != TK_NAME || (Value != NULL && !TokIsName(T, Value))) { + SyntaxErrorAt(P, T, "expected name"); + return NULL; + } + return T; +} + +static int ExpectNEWLINE(Parser *P) { + Token *T = NextTok(P); + if (T->Kind != TK_NEWLINE) { + SyntaxErrorAt(P, T, "expected newline"); + return 0; + } + return 1; +} + +/* ================= 节点构造辅助 ================= */ + +static PyObject *MakeCtx(Parser *P, const char *Name) { + PyObject *Cls = ClassGet(P, Name); + if (Cls == NULL) { + return NULL; + } + return PyObject_Vectorcall(Cls, NULL, 0, NULL); +} + +static PyObject *MakeSingleton(Parser *P, const char *Name) { + PyObject *Cls = ClassGet(P, Name); + if (Cls == NULL) { + return NULL; + } + return PyObject_Vectorcall(Cls, NULL, 0, NULL); +} + +static PyObject *NewList(void) { + return PyList_New(0); +} + +static int ListAppend(PyObject *List, PyObject *Item) { + return PyList_Append(List, Item); +} + +/* Name(id, ctx) 并设置位置 */ +static PyObject *MakeNameNode(Parser *P, Token *T, const char *CtxName) { + PyObject *Id = PyUnicode_FromStringAndSize(T->Start, T->End - T->Start); + PyObject *Ctx = MakeCtx(P, CtxName); + if (Id == NULL || Ctx == NULL) { + Py_XDECREF(Id); + Py_XDECREF(Ctx); + return NULL; + } + PyObject *Argv[2] = { Id, Ctx }; + PyObject *Node = NewNode(P, "Name", Argv, 2); + Py_DECREF(Id); + Py_DECREF(Ctx); + if (Node != NULL) { + FixLocEnd(P, Node, T, NULL); + } + return Node; +} + +/* Constant(value, kind=None) */ +static PyObject *MakeConstant(Parser *P, PyObject *Value) { + PyObject *Argv[2] = { Value, Py_None }; + return NewNode(P, "Constant", Argv, 2); +} + +static PyObject *MakeConstantPos(Parser *P, PyObject *Value, Token *Start) { + PyObject *N = MakeConstant(P, Value); + if (N != NULL) { + FixLocEnd(P, N, Start, NULL); + } + return N; +} + +/* keyword(arg, value):arg 为 NULL 表示 **kwargs(arg=None) */ +static PyObject *MakeKeyword(Parser *P, PyObject *Arg, PyObject *Value) { + PyObject *Argv[2] = { Arg ? Arg : Py_None, Value }; + return NewNode(P, "keyword", Argv, 2); +} + +static PyObject *MakeStarred(Parser *P, PyObject *Value) { + PyObject *Load = MakeCtx(P, "Load"); + if (Load == NULL) { + return NULL; + } + PyObject *Argv[2] = { Value, Load }; + PyObject *N = NewNode(P, "Starred", Argv, 2); + Py_DECREF(Load); + return N; +} + +/* Attribute(value, attr, Load) */ +static PyObject *MakeAttrNode(Parser *P, PyObject *Value, Token *AttrTok, Token *Start) { + PyObject *Attr = PyUnicode_FromStringAndSize(AttrTok->Start, AttrTok->End - AttrTok->Start); + PyObject *Load = MakeCtx(P, "Load"); + if (Attr == NULL || Load == NULL) { + Py_XDECREF(Attr); + Py_XDECREF(Load); + return NULL; + } + PyObject *Argv[3] = { Value, Attr, Load }; + PyObject *N = NewNode(P, "Attribute", Argv, 3); + Py_DECREF(Attr); + Py_DECREF(Load); + if (N != NULL) { + FixLoc(P, N, Start); + } + return N; +} + +/* 替换节点 ctx 属性为指定上下文单例 */ +static void ReplaceCtx(Parser *P, PyObject *Node, const char *CtxName) { + PyObject *Ctx = MakeCtx(P, CtxName); + if (Ctx == NULL) { + return; + } + if (PyObject_SetAttrString(Node, "ctx", Ctx) < 0) { + PyErr_Clear(); + } + Py_DECREF(Ctx); +} + +static void SetStoreContext(Parser *P, PyObject *Node); +static void SetDelContext(Parser *P, PyObject *Node); + +static void SetStoreContext(Parser *P, PyObject *Node) { + const char *N = TypeName(Node); + if (strcmp(N, "Attribute") == 0 || strcmp(N, "Subscript") == 0) { + ReplaceCtx(P, Node, "Store"); + } else if (strcmp(N, "Name") == 0) { + ReplaceCtx(P, Node, "Store"); + } else if (strcmp(N, "Tuple") == 0 || strcmp(N, "List") == 0) { + ReplaceCtx(P, Node, "Store"); + PyObject *Elts = PyObject_GetAttrString(Node, "elts"); + if (Elts != NULL) { + Py_ssize_t NElts = PyList_GET_SIZE(Elts); + for (Py_ssize_t I = 0; I < NElts; I++) { + SetStoreContext(P, PyList_GET_ITEM(Elts, I)); + } + Py_DECREF(Elts); + } else { + PyErr_Clear(); + } + } else if (strcmp(N, "Starred") == 0) { + ReplaceCtx(P, Node, "Store"); + PyObject *V = PyObject_GetAttrString(Node, "value"); + if (V != NULL) { + SetStoreContext(P, V); + Py_DECREF(V); + } else { + PyErr_Clear(); + } + } +} + +static void SetDelContext(Parser *P, PyObject *Node) { + const char *N = TypeName(Node); + if (strcmp(N, "Name") == 0 || strcmp(N, "Attribute") == 0 + || strcmp(N, "Subscript") == 0) { + ReplaceCtx(P, Node, "Del"); + } else if (strcmp(N, "Tuple") == 0 || strcmp(N, "List") == 0) { + ReplaceCtx(P, Node, "Del"); + PyObject *Elts = PyObject_GetAttrString(Node, "elts"); + if (Elts != NULL) { + Py_ssize_t NElts = PyList_GET_SIZE(Elts); + for (Py_ssize_t I = 0; I < NElts; I++) { + SetDelContext(P, PyList_GET_ITEM(Elts, I)); + } + Py_DECREF(Elts); + } else { + PyErr_Clear(); + } + } +} + +static int IsNameOrAttrOrSub(PyObject *Node) { + const char *N = TypeName(Node); + return strcmp(N, "Name") == 0 || strcmp(N, "Attribute") == 0 + || strcmp(N, "Subscript") == 0; +} + +static int NodeIsConstant(PyObject *Node) { + return NodeIs(Node, "Constant"); +} + +/* ================= 数字 / 字符串字面量 ================= */ + +/* 解析数字 token 为 Python 数值对象(对齐 vast _ParseNumber) */ +static PyObject *ParseNumberValue(Parser *P, Token *T) { + Py_ssize_t Len = T->End - T->Start; + char *Buf = (char *)PyMem_Malloc((size_t)Len + 1); + if (Buf == NULL) { + return NULL; + } + Py_ssize_t N = 0; + for (Py_ssize_t I = 0; I < Len; I++) { + if (T->Start[I] != '_') { + Buf[N++] = T->Start[I]; + } + } + Buf[N] = '\0'; + PyObject *Result = NULL; + if (N > 0 && (Buf[N - 1] == 'j' || Buf[N - 1] == 'J')) { + /* 复数:去掉 j,实部按 float 解析 */ + Buf[N - 1] = '\0'; + PyObject *RealStr = PyUnicode_FromStringAndSize(Buf, (Py_ssize_t)strlen(Buf)); + if (RealStr != NULL) { + PyObject *Real = PyFloat_FromString(RealStr); + Py_DECREF(RealStr); + if (Real != NULL) { + Result = PyComplex_FromDoubles(PyFloat_AS_DOUBLE(Real), 0.0); + Py_DECREF(Real); + } + } + } else if (N >= 2 && Buf[0] == '0' + && (Buf[1] == 'x' || Buf[1] == 'X' || Buf[1] == 'o' + || Buf[1] == 'O' || Buf[1] == 'b' || Buf[1] == 'B')) { + Result = PyLong_FromString(Buf, NULL, 0); + if (Result == NULL) { + PyErr_Clear(); + } + } else { int IsFloat = 0; - for (Py_ssize_t I = 0; I < Len; I++) { - if (Text[I] == '.') { + for (Py_ssize_t I = 0; I < N; I++) { + if (Buf[I] == '.' || Buf[I] == 'e' || Buf[I] == 'E') { IsFloat = 1; break; } } - PyObject *StrObj = PyUnicode_FromStringAndSize(Text, Len); - if (StrObj == NULL) { - P->Error = 1; - return NULL; - } - PyObject *ValObj; if (IsFloat) { - ValObj = PyFloat_FromString(StrObj); + PyObject *S = PyUnicode_FromStringAndSize(Buf, N); + if (S != NULL) { + Result = PyFloat_FromString(S); + Py_DECREF(S); + } + if (Result == NULL) { + PyErr_Clear(); + } } else { - ValObj = PyLong_FromUnicodeObject(StrObj, 10); + Result = PyLong_FromString(Buf, NULL, 10); + if (Result == NULL) { + PyErr_Clear(); + } } - Py_DECREF(StrObj); - if (ValObj == NULL) { - P->Error = 1; - return NULL; - } - PyObject *Args[2]; - Args[0] = ValObj; - Args[1] = Py_None; - PyObject *Node = NodeMake(P, "Constant", Args, 2); - Py_DECREF(ValObj); - if (Node == NULL) { - return NULL; - } - NodePos(P, Node, Line, Col); - NextToken(P); - return Node; } - if (P->Cur.Kind == TK_NAME) { - const char *Text = P->Cur.Start; - Py_ssize_t Len = P->Cur.Len; - int Line = P->Cur.Line; - int Col = P->Cur.Col; - PyObject *LoadCls = ClassGet(P, "Load"); - if (LoadCls == NULL) { - return NULL; - } - PyObject *Ctx = PyObject_Vectorcall(LoadCls, NULL, 0, NULL); - PyObject *Id = PyUnicode_FromStringAndSize(Text, Len); - if (Ctx == NULL || Id == NULL) { - Py_XDECREF(Ctx); - Py_XDECREF(Id); - return NULL; - } - PyObject *Args[2]; - Args[0] = Id; - Args[1] = Ctx; - PyObject *Node = NodeMake(P, "Name", Args, 2); - Py_DECREF(Ctx); - Py_DECREF(Id); - if (Node == NULL) { - return NULL; - } - NodePos(P, Node, Line, Col); - NextToken(P); - return Node; + if (Result == NULL) { + /* 无法解析:保留原字符串(对齐 vast 的 except ValueError 分支) */ + Result = PyUnicode_FromStringAndSize(T->Start, Len); } - P->Error = 1; - return NULL; + PyMem_Free(Buf); + return Result; } -/* 表达式(阶段 0:仅原子表达式) */ -static PyObject *ParseExpression(Parser *P) { - return ParsePrimary(P); +/* UTF-8 解码一个字符到 UCS4,返回字节数 */ +static int UTF8CharDecode(const char *S, Py_ssize_t N, Py_UCS4 *Out) { + unsigned char C = (unsigned char)S[0]; + if (C < 0x80) { + *Out = C; + return 1; + } + if (C < 0xC0 || N < 2) { + *Out = C; + return 1; + } + if (C < 0xE0) { + *Out = ((C & 0x1F) << 6) | ((unsigned char)S[1] & 0x3F); + return 2; + } + if (C < 0xF0 && N >= 3) { + *Out = ((C & 0x0F) << 12) | (((unsigned char)S[1] & 0x3F) << 6) + | ((unsigned char)S[2] & 0x3F); + return 3; + } + if (N >= 4) { + *Out = ((C & 0x07) << 18) | (((unsigned char)S[1] & 0x3F) << 12) + | (((unsigned char)S[2] & 0x3F) << 6) | ((unsigned char)S[3] & 0x3F); + return 4; + } + *Out = C; + return 1; } -/* 语句:Name '=' expr → Assign;其余表达式 → Expr 语句 */ -static PyObject *ParseStatement(Parser *P) { - int Line = P->Cur.Line; - int Col = P->Cur.Col; - if (P->Cur.Kind == TK_NAME) { - /* 向前看是否为赋值:Name '=' ... */ - const char *Text = P->Cur.Start; - Py_ssize_t Len = P->Cur.Len; - NextToken(P); - if (P->Cur.Kind == TK_EQUAL) { - NextToken(P); - PyObject *Value = ParseExpression(P); - if (Value == NULL) { - return NULL; - } - PyObject *StoreCls = ClassGet(P, "Store"); - if (StoreCls == NULL) { - Py_DECREF(Value); - return NULL; - } - PyObject *Ctx = PyObject_Vectorcall(StoreCls, NULL, 0, NULL); - PyObject *Id = PyUnicode_FromStringAndSize(Text, Len); - if (Ctx == NULL || Id == NULL) { - Py_XDECREF(Ctx); - Py_XDECREF(Id); - Py_DECREF(Value); - return NULL; - } - PyObject *TargetArgs[2]; - TargetArgs[0] = Id; - TargetArgs[1] = Ctx; - PyObject *Target = NodeMake(P, "Name", TargetArgs, 2); - Py_DECREF(Ctx); - Py_DECREF(Id); - if (Target == NULL) { - Py_DECREF(Value); - return NULL; - } - NodePos(P, Target, Line, Col); - PyObject *Targets = PyList_New(1); - PyList_SetItem(Targets, 0, Target); /* 窃取引用 */ - PyObject *AssignArgs[3]; - AssignArgs[0] = Targets; - AssignArgs[1] = Value; - AssignArgs[2] = Py_None; /* type_comment */ - PyObject *Node = NodeMake(P, "Assign", AssignArgs, 3); - Py_DECREF(Value); - if (Node == NULL) { - Py_DECREF(Targets); - return NULL; - } - NodePos(P, Node, Line, Col); - return Node; +static int HexVal(char C) { + if (C >= '0' && C <= '9') return C - '0'; + if (C >= 'a' && C <= 'f') return C - 'a' + 10; + if (C >= 'A' && C <= 'F') return C - 'A' + 10; + return 0; +} + +/* 解码字符串转义序列(对齐 vast _DecodeEscapes) */ +static PyObject *DecodeEscapes(Parser *P, const char *Body, Py_ssize_t Len, int IsRaw) { + if (IsRaw) { + return PyUnicode_DecodeUTF8(Body, Len, NULL); + } + Py_UCS4 *Buf = (Py_UCS4 *)PyMem_Malloc((size_t)(Len * 4 + 4) * sizeof(Py_UCS4)); + if (Buf == NULL) { + return NULL; + } + Py_ssize_t NBuf = 0; + Py_ssize_t I = 0; + while (I < Len) { + unsigned char Ch = (unsigned char)Body[I]; + if (Ch != '\\') { + Py_UCS4 Cp; + int NB = UTF8CharDecode(Body + I, Len - I, &Cp); + Buf[NBuf++] = Cp; + I += NB; + continue; } - /* 非赋值:Cur 已前进,直接用保存的文本构造 Load Name 表达式语句 */ - PyObject *LoadCls = ClassGet(P, "Load"); - if (LoadCls == NULL) { + I++; + if (I >= Len) { + Buf[NBuf++] = '\\'; + break; + } + char Esc = Body[I]; + switch (Esc) { + case '\\': Buf[NBuf++] = '\\'; I++; break; + case '\'': Buf[NBuf++] = '\''; I++; break; + case '"': Buf[NBuf++] = '"'; I++; break; + case 'n': Buf[NBuf++] = '\n'; I++; break; + case 't': Buf[NBuf++] = '\t'; I++; break; + case 'r': Buf[NBuf++] = '\r'; I++; break; + case 'b': Buf[NBuf++] = '\b'; I++; break; + case 'f': Buf[NBuf++] = '\f'; I++; break; + case 'v': Buf[NBuf++] = '\v'; I++; break; + case 'a': Buf[NBuf++] = '\a'; I++; break; + case 'x': { + int V = HexVal(I + 1 < Len ? Body[I + 1] : '0') * 16 + + HexVal(I + 2 < Len ? Body[I + 2] : '0'); + Buf[NBuf++] = V; + I += 3; + break; + } + case 'u': { + int V = 0; + for (int K = 1; K <= 4; K++) { + V = V * 16 + HexVal(I + K < Len ? Body[I + K] : '0'); + } + Buf[NBuf++] = V; + I += 5; + break; + } + case 'U': { + int V = 0; + for (int K = 1; K <= 8; K++) { + V = V * 16 + HexVal(I + K < Len ? Body[I + K] : '0'); + } + Buf[NBuf++] = V; + I += 9; + break; + } + default: + if (Esc >= '0' && Esc <= '7') { + int J = 0; + int V = 0; + while (J < 3 && I + J < Len && Body[I + J] >= '0' && Body[I + J] <= '7') { + V = V * 8 + (Body[I + J] - '0'); + J++; + } + Buf[NBuf++] = V; + I += J; + } else { + Buf[NBuf++] = '\\'; + Buf[NBuf++] = (Py_UCS4)(unsigned char)Esc; + I++; + } + } + } + PyObject *R = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, Buf, NBuf); + PyMem_Free(Buf); + return R; +} + +/* 解码 bytes 字面量(对齐 vast _DecodeBytes) */ +static PyObject *DecodeBytes(Parser *P, const char *Body, Py_ssize_t Len, int IsRaw) { + if (IsRaw) { + return PyBytes_FromStringAndSize(Body, Len); + } + char *Buf = (char *)PyMem_Malloc((size_t)(Len * 4 + 4)); + if (Buf == NULL) { + return NULL; + } + Py_ssize_t NBuf = 0; + Py_ssize_t I = 0; + while (I < Len) { + unsigned char Ch = (unsigned char)Body[I]; + if (Ch != '\\') { + Buf[NBuf++] = Ch; + I++; + continue; + } + I++; + if (I >= Len) { + Buf[NBuf++] = '\\'; + break; + } + char Esc = Body[I]; + switch (Esc) { + case '\\': Buf[NBuf++] = '\\'; I++; break; + case '\'': Buf[NBuf++] = '\''; I++; break; + case '"': Buf[NBuf++] = '"'; I++; break; + case 'n': Buf[NBuf++] = 10; I++; break; + case 't': Buf[NBuf++] = 9; I++; break; + case 'r': Buf[NBuf++] = 13; I++; break; + case 'b': Buf[NBuf++] = 8; I++; break; + case 'f': Buf[NBuf++] = 12; I++; break; + case 'v': Buf[NBuf++] = 11; I++; break; + case 'a': Buf[NBuf++] = 7; I++; break; + case 'x': { + int V = HexVal(I + 1 < Len ? Body[I + 1] : '0') * 16 + + HexVal(I + 2 < Len ? Body[I + 2] : '0'); + Buf[NBuf++] = (char)V; + I += 3; + break; + } + case 'u': { + int V = 0; + for (int K = 1; K <= 4; K++) { + V = V * 16 + HexVal(I + K < Len ? Body[I + K] : '0'); + } + Buf[NBuf++] = (char)V; + I += 5; + break; + } + case 'U': { + int V = 0; + for (int K = 1; K <= 8; K++) { + V = V * 16 + HexVal(I + K < Len ? Body[I + K] : '0'); + } + Buf[NBuf++] = (char)V; + I += 9; + break; + } + default: + if (Esc >= '0' && Esc <= '7') { + int J = 0; + int V = 0; + while (J < 3 && I + J < Len && Body[I + J] >= '0' && Body[I + J] <= '7') { + V = V * 8 + (Body[I + J] - '0'); + J++; + } + Buf[NBuf++] = (char)V; + I += J; + } else { + Buf[NBuf++] = '\\'; + Buf[NBuf++] = (unsigned char)Esc; + I++; + } + } + } + PyObject *R = PyBytes_FromStringAndSize(Buf, NBuf); + PyMem_Free(Buf); + return R; +} + +/* 解码字符串 token 原始文本为实际值(str 或 bytes) */ +static PyObject *DecodeString(Parser *P, Token *T) { + const char *Raw = T->Start; + Py_ssize_t Len = T->End - T->Start; + Py_ssize_t I = 0; + while (I < Len && IsStringPrefixChar((unsigned char)Raw[I])) { + I++; + } + char Quote = Raw[I]; + int Triple = (I + 2 < Len && Raw[I] == Quote && Raw[I + 1] == Quote + && Raw[I + 2] == Quote); + Py_ssize_t BodyStart = I + (Triple ? 3 : 1); + Py_ssize_t BodyEnd = Len - (Triple ? 3 : 1); + int IsRaw = 0; + int IsBytes = 0; + for (Py_ssize_t K = 0; K < I; K++) { + char C = Raw[K]; + if (C == 'r' || C == 'R') IsRaw = 1; + if (C == 'b' || C == 'B') IsBytes = 1; + } + if (IsBytes) { + return DecodeBytes(P, Raw + BodyStart, BodyEnd - BodyStart, IsRaw); + } + return DecodeEscapes(P, Raw + BodyStart, BodyEnd - BodyStart, IsRaw); +} + +/* 拼接字符串字面量值(对齐 vast _JoinConstParts) */ +static PyObject *JoinConstParts(PyObject *Parts) { + Py_ssize_t N = PyList_GET_SIZE(Parts); + if (N == 0) { + return PyUnicode_FromString(""); + } + PyObject *First = PyList_GET_ITEM(Parts, 0); + if (PyBytes_Check(First)) { + PyObject *Acc = PyBytes_FromStringAndSize(NULL, 0); + for (Py_ssize_t I = 0; I < N; I++) { + PyObject *Part = PyList_GET_ITEM(Parts, I); + PyObject *Bytes = PyBytes_Check(Part) ? Part + : PyUnicode_AsUTF8String(Part); + if (Bytes == NULL) { + Py_DECREF(Acc); + return NULL; + } + PyObject *NewAcc = PySequence_Concat(Acc, Bytes); + if (Bytes != Part) { + Py_DECREF(Bytes); + } + Py_DECREF(Acc); + if (NewAcc == NULL) { + return NULL; + } + Acc = NewAcc; + } + return Acc; + } + PyObject *Sep = PyUnicode_FromString(""); + if (Sep == NULL) { + return NULL; + } + PyObject *Joined = PyUnicode_Join(Sep, Parts); + Py_DECREF(Sep); + return Joined; +} + +/* ================= 表达式解析 ================= */ + +static PyObject *ParseExpr(Parser *P); +static PyObject *ParseConditional(Parser *P); +static PyObject *ParseDisjunction(Parser *P); +static PyObject *ParseConjunction(Parser *P); +static PyObject *ParseInversion(Parser *P); +static PyObject *ParseComparison(Parser *P); +static PyObject *ParseBitOr(Parser *P); +static PyObject *ParseBitXor(Parser *P); +static PyObject *ParseBitAnd(Parser *P); +static PyObject *ParseShift(Parser *P); +static PyObject *ParseArith(Parser *P); +static PyObject *ParseTerm(Parser *P); +static PyObject *ParseFactor(Parser *P); +static PyObject *ParsePower(Parser *P); +static PyObject *ParseAwaitPrimary(Parser *P); +static PyObject *ParsePrimary(Parser *P); +static PyObject *ParseAtom(Parser *P); +static PyObject *ParseNumber(Parser *P); +static PyObject *ParseStringConcat(Parser *P); +static PyObject *ParseParen(Parser *P); +static PyObject *ParseList(Parser *P); +static PyObject *ParseDictSet(Parser *P); +static PyObject *ParseYield(Parser *P); +static PyObject *ParseLambda(Parser *P); +static PyObject *ParseArguments(Parser *P, int SkipOpen, int AllowLambda); +static PyObject *ParseArg(Parser *P, int AllowAnnotation); +static PyObject *ParseTarget(Parser *P); +static PyObject *ParseComprehensionTarget(Parser *P); +static PyObject *ParseComprehensionGenerators(Parser *P); + +/* ============ 字符串拼接与 f-string ============ */ + +static PyObject *ParseFStringToken(Parser *P, Token *Tok); +static PyObject *ParseFStringBodyEx(Parser *P, const char *Raw, int RawLen, + Token *OuterTok, int BaseOffset); +static PyObject *ParseFStringValue(Parser *P, const char *Raw, int BodyStart, + int Start, int BodyEnd, Token *OuterTok, int RelBase); +static PyObject *ParseSubExpression(Parser *P, const char *Text, Py_ssize_t TextLen, + int AbsBase); +static int SkipInnerLiteral(const char *Raw, int BodyStart, int Start, int BodyEnd); + +/* f-string 内部字面量常量(仅 lineno/col_offset,对齐 vast) */ +static PyObject *MakeFStrConstant(Parser *P, PyObject *Value, Token *OuterTok) { + PyObject *N = MakeConstant(P, Value); + if (N != NULL) { + NodePosNoEnd(P, N, OuterTok->Line, OuterTok->Col); + } + return N; +} + +static PyObject *ParseStringConcat(Parser *P) { + Token *Start = PeekTok(P, 0); + PyObject *Parts = NewList(); /* JoinedStr 的 values */ + PyObject *ConstParts = NewList(); /* 普通字符串字面量值 */ + int HasFString = 0; + if (Parts == NULL || ConstParts == NULL) { + Py_XDECREF(Parts); + Py_XDECREF(ConstParts); + return NULL; + } + for (;;) { + SkipNL(P); + Token *Tok = PeekTok(P, 0); + if (Tok->Kind == TK_STRING) { + NextTok(P); + PyObject *Val = DecodeString(P, Tok); + if (Val == NULL) { + Py_DECREF(Parts); + Py_DECREF(ConstParts); + return NULL; + } + PyList_Append(ConstParts, Val); + Py_DECREF(Val); + } else if (Tok->Kind == TK_FSTRING) { + NextTok(P); + PyObject *Joined = ParseFStringToken(P, Tok); + if (Joined == NULL) { + Py_DECREF(Parts); + Py_DECREF(ConstParts); + return NULL; + } + HasFString = 1; + if (PyList_GET_SIZE(ConstParts) > 0) { + PyObject *JoinedStr = JoinConstParts(ConstParts); + if (JoinedStr == NULL) { + Py_DECREF(Joined); + Py_DECREF(Parts); + Py_DECREF(ConstParts); + return NULL; + } + PyObject *C = MakeFStrConstant(P, JoinedStr, Start); + Py_DECREF(JoinedStr); + if (C == NULL) { + Py_DECREF(Joined); + Py_DECREF(Parts); + Py_DECREF(ConstParts); + return NULL; + } + PyList_Append(Parts, C); + Py_DECREF(C); + PyList_SetSlice(ConstParts, 0, PyList_GET_SIZE(ConstParts), NULL); + } + PyObject *JVals = PyObject_GetAttrString(Joined, "values"); + if (JVals == NULL) { + Py_DECREF(Joined); + Py_DECREF(Parts); + Py_DECREF(ConstParts); + return NULL; + } + /* 相邻 Constant 合并(f-string 字面量 + 后续常量段) */ + if (PyList_GET_SIZE(Parts) > 0 && NodeIsConstant(PyList_GET_ITEM(Parts, PyList_GET_SIZE(Parts) - 1)) + && PyList_GET_SIZE(JVals) > 0 && NodeIsConstant(PyList_GET_ITEM(JVals, 0))) { + PyObject *Last = PyList_GET_ITEM(Parts, PyList_GET_SIZE(Parts) - 1); + PyObject *LastVal = PyObject_GetAttrString(Last, "value"); + PyObject *FirstVal = PyObject_GetAttrString(PyList_GET_ITEM(JVals, 0), "value"); + if (LastVal == NULL || FirstVal == NULL) { + Py_XDECREF(LastVal); + Py_XDECREF(FirstVal); + Py_DECREF(JVals); + Py_DECREF(Joined); + Py_DECREF(Parts); + Py_DECREF(ConstParts); + return NULL; + } + PyObject *S1 = PyObject_Str(LastVal); + PyObject *S2 = PyObject_Str(FirstVal); + Py_DECREF(LastVal); + Py_DECREF(FirstVal); + PyObject *Merged = NULL; + if (S1 != NULL && S2 != NULL) { + Merged = PyUnicode_Concat(S1, S2); + } + Py_XDECREF(S1); + Py_XDECREF(S2); + if (Merged == NULL) { + Py_DECREF(JVals); + Py_DECREF(Joined); + Py_DECREF(Parts); + Py_DECREF(ConstParts); + return NULL; + } + PyObject *MergedConst = MakeFStrConstant(P, Merged, Start); + Py_DECREF(Merged); + if (MergedConst == NULL) { + Py_DECREF(JVals); + Py_DECREF(Joined); + Py_DECREF(Parts); + Py_DECREF(ConstParts); + return NULL; + } + PyList_SetItem(Parts, PyList_GET_SIZE(Parts) - 1, MergedConst); /* 偷取 */ + PyObject *Val; + for (Py_ssize_t I = 1; I < PyList_GET_SIZE(JVals); I++) { + Val = PyList_GET_ITEM(JVals, I); + PyList_Append(Parts, Val); + } + } else { + PyObject *Val; + for (Py_ssize_t I = 0; I < PyList_GET_SIZE(JVals); I++) { + Val = PyList_GET_ITEM(JVals, I); + PyList_Append(Parts, Val); + } + } + Py_DECREF(JVals); + Py_DECREF(Joined); + } else { + break; + } + } + if (!HasFString) { + PyObject *Val = JoinConstParts(ConstParts); + Py_DECREF(ConstParts); + if (Val == NULL) { + Py_DECREF(Parts); return NULL; } - PyObject *Ctx = PyObject_Vectorcall(LoadCls, NULL, 0, NULL); - PyObject *Id = PyUnicode_FromStringAndSize(Text, Len); - if (Ctx == NULL || Id == NULL) { - Py_XDECREF(Ctx); - Py_XDECREF(Id); - return NULL; - } - PyObject *NameArgs[2]; - NameArgs[0] = Id; - NameArgs[1] = Ctx; - PyObject *NameNode = NodeMake(P, "Name", NameArgs, 2); - Py_DECREF(Ctx); - Py_DECREF(Id); - if (NameNode == NULL) { - return NULL; - } - NodePos(P, NameNode, Line, Col); - PyObject *ExprArgs[1]; - ExprArgs[0] = NameNode; - PyObject *Node = NodeMake(P, "Expr", ExprArgs, 1); - Py_DECREF(NameNode); - if (Node == NULL) { - return NULL; - } - NodePos(P, Node, Line, Col); + PyObject *Node = MakeConstantPos(P, Val, Start); + Py_DECREF(Val); + Py_DECREF(Parts); return Node; } - /* 一般表达式语句 */ - PyObject *ExprVal = ParsePrimary(P); - if (ExprVal == NULL) { - return NULL; + if (PyList_GET_SIZE(ConstParts) > 0) { + if (PyList_GET_SIZE(Parts) > 0 + && NodeIsConstant(PyList_GET_ITEM(Parts, PyList_GET_SIZE(Parts) - 1))) { + PyObject *Last = PyList_GET_ITEM(Parts, PyList_GET_SIZE(Parts) - 1); + PyObject *LastVal = PyObject_GetAttrString(Last, "value"); + PyObject *TailVal = JoinConstParts(ConstParts); + if (LastVal == NULL || TailVal == NULL) { + Py_XDECREF(LastVal); + Py_XDECREF(TailVal); + Py_DECREF(Parts); + Py_DECREF(ConstParts); + return NULL; + } + PyObject *S1 = PyObject_Str(LastVal); + PyObject *S2 = PyObject_Str(TailVal); + Py_DECREF(LastVal); + Py_DECREF(TailVal); + PyObject *Merged = NULL; + if (S1 != NULL && S2 != NULL) { + Merged = PyUnicode_Concat(S1, S2); + } + Py_XDECREF(S1); + Py_XDECREF(S2); + if (Merged == NULL) { + Py_DECREF(Parts); + Py_DECREF(ConstParts); + return NULL; + } + PyObject *MergedConst = MakeFStrConstant(P, Merged, Start); + Py_DECREF(Merged); + if (MergedConst == NULL) { + Py_DECREF(Parts); + Py_DECREF(ConstParts); + return NULL; + } + PyList_SetItem(Parts, PyList_GET_SIZE(Parts) - 1, MergedConst); /* 偷取 */ + } else { + PyObject *TailVal = JoinConstParts(ConstParts); + if (TailVal == NULL) { + Py_DECREF(Parts); + Py_DECREF(ConstParts); + return NULL; + } + PyObject *C = MakeFStrConstant(P, TailVal, Start); + Py_DECREF(TailVal); + if (C == NULL) { + Py_DECREF(Parts); + Py_DECREF(ConstParts); + return NULL; + } + PyList_Append(Parts, C); + Py_DECREF(C); + } } - PyObject *ExprArgs[1]; - ExprArgs[0] = ExprVal; - PyObject *Node = NodeMake(P, "Expr", ExprArgs, 1); - Py_DECREF(ExprVal); - if (Node == NULL) { - return NULL; + Py_DECREF(ConstParts); + PyObject *Argv[1] = { Parts }; + PyObject *Node = NewNode(P, "JoinedStr", Argv, 1); + Py_DECREF(Parts); + if (Node != NULL) { + FixLoc(P, Node, Start); } - NodePos(P, Node, Line, Col); return Node; } -/* 模块:语句序列 */ -static PyObject *ParseModule(Parser *P) { - PyObject *Body = PyList_New(0); +/* 解析 FSTRING token 为 JoinedStr */ +static PyObject *ParseFStringToken(Parser *P, Token *Tok) { + PyObject *Values = ParseFStringBodyEx(P, Tok->Start, (int)(Tok->End - Tok->Start), + Tok, 0); + if (Values == NULL) { + return NULL; + } + PyObject *Argv[1] = { Values }; + PyObject *Node = NewNode(P, "JoinedStr", Argv, 1); + Py_DECREF(Values); + if (Node != NULL) { + NodePos(P, Node, Tok->Line, Tok->Col, Tok->EndLine, Tok->EndCol); + } + return Node; +} + +/* 基于原始文本的 f-string body 解析(完整实现)。 + * 返回 values 列表;BaseOffset 用于嵌套(本实现仅顶层使用)。 */ +static PyObject *ParseFStringBodyEx(Parser *P, const char *Raw, int RawLen, + Token *OuterTok, int BaseOffset); + +/* 字面文本暂存(字节缓冲) */ +typedef struct { + char *Buf; + Py_ssize_t Len; + Py_ssize_t Cap; +} LitBuf; + +static int LitAppend(LitBuf *Lb, const char *S, Py_ssize_t N) { + if (Lb->Len + N > Lb->Cap) { + Py_ssize_t NCap = Lb->Cap ? Lb->Cap * 2 : 64; + while (NCap < Lb->Len + N) { + NCap *= 2; + } + char *NB = (char *)PyMem_Realloc(Lb->Buf, (size_t)NCap); + if (NB == NULL) { + return 0; + } + Lb->Buf = NB; + Lb->Cap = NCap; + } + memcpy(Lb->Buf + Lb->Len, S, (size_t)N); + Lb->Len += N; + return 1; +} + +static PyObject *ParseFStringBodyEx(Parser *P, const char *Raw, int RawLen, + Token *OuterTok, int BaseOffset) { + (void)BaseOffset; + int I = 0; + while (I < RawLen && IsStringPrefixChar((unsigned char)Raw[I])) { + I++; + } + char Quote = Raw[I]; + int Triple = (I + 2 < RawLen && Raw[I] == Quote && Raw[I + 1] == Quote + && Raw[I + 2] == Quote); + int IsRaw = 0; + for (int K = 0; K < I; K++) { + if (Raw[K] == 'r' || Raw[K] == 'R') IsRaw = 1; + } + int BodyStart = I + (Triple ? 3 : 1); + int BodyEnd = RawLen - (Triple ? 3 : 1); + PyObject *Values = NewList(); + if (Values == NULL) { + return NULL; + } + LitBuf Lb; + memset(&Lb, 0, sizeof(Lb)); + int J = 0; + int RelBase = BodyStart; + while (J < BodyEnd - BodyStart) { + char Ch = Raw[BodyStart + J]; + if (Ch == '{') { + if (J + 1 < BodyEnd - BodyStart && Raw[BodyStart + J + 1] == '{') { + if (!LitAppend(&Lb, "{", 1)) { + Py_DECREF(Values); + PyMem_Free(Lb.Buf); + return NULL; + } + J += 2; + continue; + } + if (Lb.Len > 0) { + PyObject *Str = DecodeEscapes(P, Lb.Buf, Lb.Len, IsRaw); + Lb.Len = 0; + if (Str == NULL) { + Py_DECREF(Values); + PyMem_Free(Lb.Buf); + return NULL; + } + PyObject *C = MakeFStrConstant(P, Str, OuterTok); + Py_DECREF(Str); + if (C == NULL) { + Py_DECREF(Values); + PyMem_Free(Lb.Buf); + return NULL; + } + PyList_Append(Values, C); + Py_DECREF(C); + } + PyObject *Fv = ParseFStringValue(P, Raw, BodyStart, J, BodyEnd, OuterTok, RelBase); + if (Fv == NULL) { + Py_DECREF(Values); + PyMem_Free(Lb.Buf); + return NULL; + } + PyList_Append(Values, Fv); + Py_DECREF(Fv); + J = P->FStringExprEnd; + continue; + } else if (Ch == '}') { + if (J + 1 < BodyEnd - BodyStart && Raw[BodyStart + J + 1] == '}') { + if (!LitAppend(&Lb, "}", 1)) { + Py_DECREF(Values); + PyMem_Free(Lb.Buf); + return NULL; + } + J += 2; + continue; + } + /* 单独的 } 宽松处理为字面量 */ + if (!LitAppend(&Lb, "}", 1)) { + Py_DECREF(Values); + PyMem_Free(Lb.Buf); + return NULL; + } + J += 1; + continue; + } else { + Py_UCS4 Cp; + int NB = UTF8CharDecode(Raw + BodyStart + J, BodyEnd - BodyStart - J, &Cp); + char Tmp[4]; + int TN; + if (Cp < 0x80) { + Tmp[0] = (char)Cp; + TN = 1; + } else if (Cp < 0x800) { + Tmp[0] = (char)(0xC0 | (Cp >> 6)); + Tmp[1] = (char)(0x80 | (Cp & 0x3F)); + TN = 2; + } else if (Cp < 0x10000) { + Tmp[0] = (char)(0xE0 | (Cp >> 12)); + Tmp[1] = (char)(0x80 | ((Cp >> 6) & 0x3F)); + Tmp[2] = (char)(0x80 | (Cp & 0x3F)); + TN = 3; + } else { + Tmp[0] = (char)(0xF0 | (Cp >> 18)); + Tmp[1] = (char)(0x80 | ((Cp >> 12) & 0x3F)); + Tmp[2] = (char)(0x80 | ((Cp >> 6) & 0x3F)); + Tmp[3] = (char)(0x80 | (Cp & 0x3F)); + TN = 4; + } + if (!LitAppend(&Lb, Tmp, TN)) { + Py_DECREF(Values); + PyMem_Free(Lb.Buf); + return NULL; + } + J += NB; + continue; + } + } + if (Lb.Len > 0) { + PyObject *Str = DecodeEscapes(P, Lb.Buf, Lb.Len, IsRaw); + if (Str == NULL) { + Py_DECREF(Values); + PyMem_Free(Lb.Buf); + return NULL; + } + PyObject *C = MakeFStrConstant(P, Str, OuterTok); + Py_DECREF(Str); + if (C == NULL) { + Py_DECREF(Values); + PyMem_Free(Lb.Buf); + return NULL; + } + PyList_Append(Values, C); + Py_DECREF(C); + } + PyMem_Free(Lb.Buf); + return Values; +} + +/* 解析单个 {expr[!conv][:spec]} 段,返回 FormattedValue;结束位置存入 P->FStringExprEnd */ +static PyObject *ParseFStringValue(Parser *P, const char *Raw, int BodyStart, + int Start, int BodyEnd, Token *OuterTok, int RelBase) { + int J = Start + 1; /* 跳过 { */ + int Depth = 1; + int ExprStart = J; + int ExprEnd = J; + char StopChar = '}'; + while (J < BodyEnd) { + char Ch = Raw[BodyStart + J]; + if (Ch == '\\') { + J += 2; + continue; + } + if (Ch == '\'' || Ch == '"') { + J = SkipInnerLiteral(Raw, BodyStart, J, BodyEnd); + continue; + } + if (Ch == '(' || Ch == '[' || Ch == '{') { + Depth++; + J++; + continue; + } + if (Ch == ')' || Ch == ']' || Ch == '}') { + if (Depth == 1) { + ExprEnd = J; + J++; + break; + } + Depth--; + J++; + continue; + } + if (Depth == 1 && (Ch == '!' || Ch == ':')) { + ExprEnd = J; + StopChar = Ch; + break; + } + J++; + } + P->FStringExprEnd = J; + int Conversion = -1; + PyObject *FormatSpec = NULL; + int K = J; + if (StopChar == '!') { + K++; + if (K < BodyEnd && (Raw[BodyStart + K] == 'r' || Raw[BodyStart + K] == 's' + || Raw[BodyStart + K] == 'a')) { + Conversion = (unsigned char)Raw[BodyStart + K]; + K++; + } + if (K < BodyEnd && Raw[BodyStart + K] == ':') { + StopChar = ':'; + } else { + if (K < BodyEnd && Raw[BodyStart + K] == '}') { + K++; + } + P->FStringExprEnd = K; + } + } + if (StopChar == ':') { + K++; + int SpecStart = K; + int Depth2 = 1; + while (K < BodyEnd) { + char C2 = Raw[BodyStart + K]; + if (C2 == '{') { + Depth2++; + K++; + } else if (C2 == '}') { + Depth2--; + if (Depth2 == 0) { + break; + } + K++; + } else { + K++; + } + } + int HasNested = 0; + for (int I2 = SpecStart; I2 < K; I2++) { + if (Raw[BodyStart + I2] == '{') { + HasNested = 1; + break; + } + } + if (HasNested) { + /* 嵌套表达式:构造内层 f-string 文本并递归解析 */ + Py_ssize_t SpecLen = K - SpecStart; + char *Inner = (char *)PyMem_Malloc((size_t)(SpecLen * 2 + 4)); + if (Inner == NULL) { + return NULL; + } + Py_ssize_t NI = 0; + Inner[NI++] = 'f'; + Inner[NI++] = '"'; + for (int I2 = SpecStart; I2 < K; I2++) { + char C2 = Raw[BodyStart + I2]; + if (C2 == '"') { + Inner[NI++] = '\\'; + } + Inner[NI++] = C2; + } + Inner[NI++] = '"'; + Token InnerTok; + InnerTok.Kind = TK_FSTRING; + InnerTok.Start = Inner; + InnerTok.End = Inner + NI; + InnerTok.Line = OuterTok->Line; + InnerTok.Col = OuterTok->Col; + InnerTok.EndLine = OuterTok->EndLine; + InnerTok.EndCol = OuterTok->EndCol; + InnerTok.StartOff = OuterTok->StartOff; + InnerTok.EndOff = OuterTok->EndOff; + FormatSpec = ParseFStringToken(P, &InnerTok); + PyMem_Free(Inner); + if (FormatSpec == NULL) { + return NULL; + } + } else { + PyObject *SpecStr = PyUnicode_FromStringAndSize(Raw + BodyStart + SpecStart, + K - SpecStart); + if (SpecStr == NULL) { + return NULL; + } + PyObject *C = MakeFStrConstant(P, SpecStr, OuterTok); + Py_DECREF(SpecStr); + if (C == NULL) { + return NULL; + } + PyObject *JVals = NewList(); + if (JVals == NULL) { + Py_DECREF(C); + return NULL; + } + PyList_Append(JVals, C); + Py_DECREF(C); + PyObject *Argv[1] = { JVals }; + FormatSpec = NewNode(P, "JoinedStr", Argv, 1); + Py_DECREF(JVals); + if (FormatSpec == NULL) { + return NULL; + } + } + if (K < BodyEnd && Raw[BodyStart + K] == '}') { + K++; + } + P->FStringExprEnd = K; + } + /* 解析子表达式并重定位 */ + PyObject *ExprNode = ParseSubExpression(P, Raw + BodyStart + ExprStart, + ExprEnd - ExprStart, + OuterTok->StartOff + RelBase + ExprStart); + if (ExprNode == NULL) { + Py_XDECREF(FormatSpec); + return NULL; + } + PyObject *Argv[3]; + Argv[0] = ExprNode; + Argv[1] = PyLong_FromLong(Conversion); + Argv[2] = FormatSpec ? FormatSpec : Py_None; + PyObject *Fv = NewNode(P, "FormattedValue", Argv, 3); + Py_XDECREF(Argv[1]); + Py_XDECREF(FormatSpec); + Py_DECREF(ExprNode); + if (Fv == NULL) { + return NULL; + } + NodePos(P, Fv, OuterTok->Line, OuterTok->Col, OuterTok->EndLine, OuterTok->EndCol); + return Fv; +} + +/* 跳过 f-string 表达式内部的字符串字面量 */ +static int SkipInnerLiteral(const char *Raw, int BodyStart, int Start, int BodyEnd) { + int J = Start; + while (J < BodyEnd && IsStringPrefixChar((unsigned char)Raw[BodyStart + J])) { + J++; + } + if (J >= BodyEnd || (Raw[BodyStart + J] != '\'' && Raw[BodyStart + J] != '"')) { + return Start + 1; + } + char Quote = Raw[BodyStart + J]; + int Triple = (J + 2 < BodyEnd && Raw[BodyStart + J] == Quote + && Raw[BodyStart + J + 1] == Quote && Raw[BodyStart + J + 2] == Quote); + J += Triple ? 3 : 1; + while (J < BodyEnd) { + char Ch = Raw[BodyStart + J]; + if (Ch == '\\') { + J += 2; + continue; + } + if (Ch == Quote) { + if (Triple) { + if (J + 2 < BodyEnd && Raw[BodyStart + J] == Quote + && Raw[BodyStart + J + 1] == Quote && Raw[BodyStart + J + 2] == Quote) { + J += 3; + return J; + } + J += 1; + } else { + J += 1; + return J; + } + } + J++; + } + return J; +} + +/* 解析 f-string 子表达式文本(重新 tokenize 并重定位到绝对偏移) */ +static PyObject *ParseSubExpression(Parser *P, const char *Text, Py_ssize_t TextLen, + int AbsBase) { + Token *SubToks = NULL; + int NSub = 0; + if (!TokenizeText(Text, TextLen, &SubToks, &NSub, NULL, NULL)) { + return NULL; + } + for (int I = 0; I < NSub; I++) { + Token *T = &SubToks[I]; + T->StartOff += AbsBase; + T->EndOff += AbsBase; + PosInfoLS(P->LineStarts, P->NLineStarts, T->StartOff, &T->Line, &T->Col); + PosInfoLS(P->LineStarts, P->NLineStarts, T->EndOff, &T->EndLine, &T->EndCol); + } + int SavedIdx = P->Idx; + Token *SavedTokens = P->Tokens; + int SavedNTokens = P->NTokens; + P->Tokens = SubToks; + P->NTokens = NSub; + P->Idx = 0; + PyObject *Expr = ParseExpr(P); + P->Tokens = SavedTokens; + P->NTokens = SavedNTokens; + P->Idx = SavedIdx; + PyMem_Free(SubToks); + return Expr; +} + +/* ================= 表达式(优先级链) ================= */ + +/* 完整表达式:lambda 或条件表达式 */ +static PyObject *ParseExpr(Parser *P) { + if (AtName(P, "lambda")) { + return ParseLambda(P); + } + return ParseConditional(P); +} + +/* testlist_star_expr:表达式或逗号分隔元组 */ +static PyObject *ParseExprList(Parser *P) { + Token *Start = PeekTok(P, 0); + PyObject *First = ParseExpr(P); + if (First == NULL) { + return NULL; + } + if (!AtOp(P, ",")) { + return First; + } + PyObject *Elts = NewList(); + if (Elts == NULL) { + Py_DECREF(First); + return NULL; + } + PyList_Append(Elts, First); + while (AtOp(P, ",")) { + NextTok(P); + SkipNL(P); + PyObject *E = ParseExpr(P); + if (E == NULL) { + Py_DECREF(Elts); + Py_DECREF(First); + return NULL; + } + PyList_Append(Elts, E); + Py_DECREF(E); + } + Py_DECREF(First); + PyObject *Load = MakeCtx(P, "Load"); + if (Load == NULL) { + Py_DECREF(Elts); + return NULL; + } + PyObject *Argv[2] = { Elts, Load }; + PyObject *Node = NewNode(P, "Tuple", Argv, 2); + Py_DECREF(Elts); + Py_DECREF(Load); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* 条件表达式:disjunction ('if' disjunction 'else' expression)? */ +static PyObject *ParseConditional(Parser *P) { + Token *Start = PeekTok(P, 0); + PyObject *Body = ParseDisjunction(P); if (Body == NULL) { return NULL; } - NextToken(P); - while (P->Cur.Kind != TK_EOF) { - if (P->Cur.Kind == TK_NEWLINE) { - NextToken(P); + if (!AtName(P, "if")) { + return Body; + } + NextTok(P); + PyObject *Test = ParseDisjunction(P); + if (Test == NULL) { + Py_DECREF(Body); + return NULL; + } + if (!AtName(P, "else")) { + Py_DECREF(Body); + Py_DECREF(Test); + SyntaxErrorAt(P, PeekTok(P, 0), "expected 'else'"); + return NULL; + } + NextTok(P); + PyObject *Orelse = ParseExpr(P); + if (Orelse == NULL) { + Py_DECREF(Body); + Py_DECREF(Test); + return NULL; + } + PyObject *Argv[3] = { Test, Body, Orelse }; + PyObject *Node = NewNode(P, "IfExp", Argv, 3); + Py_DECREF(Test); + Py_DECREF(Body); + Py_DECREF(Orelse); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* disjunction:conjunction ('or' conjunction)* */ +static PyObject *ParseDisjunction(Parser *P) { + Token *Start = PeekTok(P, 0); + PyObject *Values = NewList(); + if (Values == NULL) { + return NULL; + } + PyObject *First = ParseConjunction(P); + if (First == NULL) { + Py_DECREF(Values); + return NULL; + } + PyList_Append(Values, First); + Py_DECREF(First); + for (;;) { + SkipNL(P); + if (!AtName(P, "or")) { + break; + } + NextTok(P); + PyObject *V = ParseConjunction(P); + if (V == NULL) { + Py_DECREF(Values); + return NULL; + } + PyList_Append(Values, V); + Py_DECREF(V); + } + if (PyList_GET_SIZE(Values) == 1) { + PyObject *Only = PyList_GET_ITEM(Values, 0); + Py_INCREF(Only); + Py_DECREF(Values); + return Only; + } + PyObject *Op = MakeSingleton(P, "Or"); + if (Op == NULL) { + Py_DECREF(Values); + return NULL; + } + PyObject *Argv[2] = { Op, Values }; + PyObject *Node = NewNode(P, "BoolOp", Argv, 2); + Py_DECREF(Op); + Py_DECREF(Values); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* conjunction:inversion ('and' inversion)* */ +static PyObject *ParseConjunction(Parser *P) { + Token *Start = PeekTok(P, 0); + PyObject *Values = NewList(); + if (Values == NULL) { + return NULL; + } + PyObject *First = ParseInversion(P); + if (First == NULL) { + Py_DECREF(Values); + return NULL; + } + PyList_Append(Values, First); + Py_DECREF(First); + for (;;) { + SkipNL(P); + if (!AtName(P, "and")) { + break; + } + NextTok(P); + PyObject *V = ParseInversion(P); + if (V == NULL) { + Py_DECREF(Values); + return NULL; + } + PyList_Append(Values, V); + Py_DECREF(V); + } + if (PyList_GET_SIZE(Values) == 1) { + PyObject *Only = PyList_GET_ITEM(Values, 0); + Py_INCREF(Only); + Py_DECREF(Values); + return Only; + } + PyObject *Op = MakeSingleton(P, "And"); + if (Op == NULL) { + Py_DECREF(Values); + return NULL; + } + PyObject *Argv[2] = { Op, Values }; + PyObject *Node = NewNode(P, "BoolOp", Argv, 2); + Py_DECREF(Op); + Py_DECREF(Values); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* inversion:'not' inversion | comparison */ +static PyObject *ParseInversion(Parser *P) { + SkipNL(P); + if (AtName(P, "not")) { + Token *Start = NextTok(P); + PyObject *Operand = ParseInversion(P); + if (Operand == NULL) { + return NULL; + } + PyObject *Op = MakeSingleton(P, "Not"); + if (Op == NULL) { + Py_DECREF(Operand); + return NULL; + } + PyObject *Argv[2] = { Op, Operand }; + PyObject *Node = NewNode(P, "UnaryOp", Argv, 2); + Py_DECREF(Op); + Py_DECREF(Operand); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + return ParseComparison(P); +} + +/* 比较运算符映射 */ +static const char *CompareOpName(const char *S) { + if (strcmp(S, "<") == 0) return "Lt"; + if (strcmp(S, ">") == 0) return "Gt"; + if (strcmp(S, "<=") == 0) return "LtE"; + if (strcmp(S, ">=") == 0) return "GtE"; + if (strcmp(S, "==") == 0) return "Eq"; + if (strcmp(S, "!=") == 0) return "NotEq"; + return NULL; +} + +/* comparison:bit_or (cmpop bit_or)* */ +static PyObject *ParseComparison(Parser *P) { + Token *Start = PeekTok(P, 0); + PyObject *Left = ParseBitOr(P); + if (Left == NULL) { + return NULL; + } + PyObject *Ops = NewList(); + PyObject *Comparators = NewList(); + if (Ops == NULL || Comparators == NULL) { + Py_XDECREF(Ops); + Py_XDECREF(Comparators); + Py_DECREF(Left); + return NULL; + } + for (;;) { + SkipNL(P); + Token *T = PeekTok(P, 0); + const char *OpName = NULL; + if (T->Kind == TK_OP) { + /* 通过 TokIsOp 精确匹配(避免非 NUL 结尾比较),并消费操作符 token */ + if (TokIsOp(T, "<")) { OpName = "Lt"; NextTok(P); } + else if (TokIsOp(T, ">")) { OpName = "Gt"; NextTok(P); } + else if (TokIsOp(T, "<=")) { OpName = "LtE"; NextTok(P); } + else if (TokIsOp(T, ">=")) { OpName = "GtE"; NextTok(P); } + else if (TokIsOp(T, "==")) { OpName = "Eq"; NextTok(P); } + else if (TokIsOp(T, "!=")) { OpName = "NotEq"; NextTok(P); } + } + if (OpName == NULL && T->Kind == TK_NAME && TokIsName(T, "is")) { + NextTok(P); + if (PeekTok(P, 0)->Kind == TK_NAME && TokIsName(PeekTok(P, 0), "not")) { + NextTok(P); + OpName = "IsNot"; + } else { + OpName = "Is"; + } + } else if (OpName == NULL && T->Kind == TK_NAME && TokIsName(T, "in")) { + NextTok(P); + OpName = "In"; + } else if (OpName == NULL && T->Kind == TK_NAME && TokIsName(T, "not") + && PeekTok(P, 1)->Kind == TK_NAME && TokIsName(PeekTok(P, 1), "in")) { + NextTok(P); + NextTok(P); + OpName = "NotIn"; + } + if (OpName == NULL) { + break; + } + PyObject *Op = MakeSingleton(P, OpName); + if (Op == NULL) { + Py_DECREF(Ops); + Py_DECREF(Comparators); + Py_DECREF(Left); + return NULL; + } + PyList_Append(Ops, Op); + Py_DECREF(Op); + PyObject *C = ParseBitOr(P); + if (C == NULL) { + Py_DECREF(Ops); + Py_DECREF(Comparators); + Py_DECREF(Left); + return NULL; + } + PyList_Append(Comparators, C); + Py_DECREF(C); + } + if (PyList_GET_SIZE(Ops) == 0) { + Py_DECREF(Ops); + Py_DECREF(Comparators); + return Left; + } + PyObject *Argv[3] = { Left, Ops, Comparators }; + PyObject *Node = NewNode(P, "Compare", Argv, 3); + Py_DECREF(Left); + Py_DECREF(Ops); + Py_DECREF(Comparators); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* 二元运算通用循环:left = Parse lower; while op: right = Parse lower; BinOp */ +typedef PyObject *(*ParseFunc)(Parser *P); + +static PyObject *ParseBinLoop(Parser *P, ParseFunc Lower, const char *Op1, + const char *Op2, const char *Node1, const char *Node2) { + Token *Start = PeekTok(P, 0); + PyObject *Left = Lower(P); + if (Left == NULL) { + return NULL; + } + for (;;) { + SkipNL(P); + const char *OpName = NULL; + if (Op1 != NULL && AtOp(P, Op1)) { + OpName = Node1; + NextTok(P); + } else if (Op2 != NULL && AtOp(P, Op2)) { + OpName = Node2; + NextTok(P); + } else { + break; + } + PyObject *Right = Lower(P); + if (Right == NULL) { + Py_DECREF(Left); + return NULL; + } + PyObject *Op = MakeSingleton(P, OpName); + if (Op == NULL) { + Py_DECREF(Left); + Py_DECREF(Right); + return NULL; + } + PyObject *Argv[3] = { Left, Op, Right }; + PyObject *Node = NewNode(P, "BinOp", Argv, 3); + Py_DECREF(Op); + Py_DECREF(Right); + Py_DECREF(Left); + if (Node == NULL) { + return NULL; + } + FixLoc(P, Node, Start); + Left = Node; + } + return Left; +} + +static PyObject *ParseBitOr(Parser *P) { + return ParseBinLoop(P, ParseBitXor, "|", NULL, "BitOr", NULL); +} + +static PyObject *ParseBitXor(Parser *P) { + return ParseBinLoop(P, ParseBitAnd, "^", NULL, "BitXor", NULL); +} + +static PyObject *ParseBitAnd(Parser *P) { + return ParseBinLoop(P, ParseShift, "&", NULL, "BitAnd", NULL); +} + +static PyObject *ParseShift(Parser *P) { + return ParseBinLoop(P, ParseArith, "<<", ">>", "LShift", "RShift"); +} + +static PyObject *ParseArith(Parser *P) { + return ParseBinLoop(P, ParseTerm, "+", "-", "Add", "Sub"); +} + +/* term:factor (*|/|//|%|@ factor)* */ +static PyObject *ParseTerm(Parser *P) { + Token *Start = PeekTok(P, 0); + PyObject *Left = ParseFactor(P); + if (Left == NULL) { + return NULL; + } + for (;;) { + SkipNL(P); + Token *T = PeekTok(P, 0); + const char *OpName = NULL; + if (T->Kind == TK_OP) { + if (TokIsOp(T, "*")) OpName = "Mult"; + else if (TokIsOp(T, "/")) OpName = "Div"; + else if (TokIsOp(T, "//")) OpName = "FloorDiv"; + else if (TokIsOp(T, "%")) OpName = "Mod"; + else if (TokIsOp(T, "@")) OpName = "MatMult"; + } + if (OpName == NULL) { + break; + } + NextTok(P); + PyObject *Right = ParseFactor(P); + if (Right == NULL) { + Py_DECREF(Left); + return NULL; + } + PyObject *Op = MakeSingleton(P, OpName); + if (Op == NULL) { + Py_DECREF(Left); + Py_DECREF(Right); + return NULL; + } + PyObject *Argv[3] = { Left, Op, Right }; + PyObject *Node = NewNode(P, "BinOp", Argv, 3); + Py_DECREF(Op); + Py_DECREF(Right); + Py_DECREF(Left); + if (Node == NULL) { + return NULL; + } + FixLoc(P, Node, Start); + Left = Node; + } + return Left; +} + +/* factor:+|-|~ factor | power */ +static PyObject *ParseFactor(Parser *P) { + Token *T = PeekTok(P, 0); + if (T->Kind == TK_OP && (TokIsOp(T, "+") || TokIsOp(T, "-") || TokIsOp(T, "~"))) { + Token *Start = NextTok(P); + PyObject *Operand = ParseFactor(P); + if (Operand == NULL) { + return NULL; + } + const char *OpName = TokIsOp(Start, "+") ? "UAdd" + : (TokIsOp(Start, "-") ? "USub" : "Invert"); + PyObject *Op = MakeSingleton(P, OpName); + if (Op == NULL) { + Py_DECREF(Operand); + return NULL; + } + PyObject *Argv[2] = { Op, Operand }; + PyObject *Node = NewNode(P, "UnaryOp", Argv, 2); + Py_DECREF(Op); + Py_DECREF(Operand); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + return ParsePower(P); +} + +/* power:await_primary ('**' factor)? */ +static PyObject *ParsePower(Parser *P) { + Token *Start = PeekTok(P, 0); + PyObject *Base = ParseAwaitPrimary(P); + if (Base == NULL) { + return NULL; + } + if (!AtOp(P, "**")) { + return Base; + } + NextTok(P); + PyObject *Exp = ParseFactor(P); + if (Exp == NULL) { + Py_DECREF(Base); + return NULL; + } + PyObject *Op = MakeSingleton(P, "Pow"); + if (Op == NULL) { + Py_DECREF(Base); + Py_DECREF(Exp); + return NULL; + } + PyObject *Argv[3] = { Base, Op, Exp }; + PyObject *Node = NewNode(P, "BinOp", Argv, 3); + Py_DECREF(Op); + Py_DECREF(Exp); + Py_DECREF(Base); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* await_primary:'await' primary | primary */ +static PyObject *ParseAwaitPrimary(Parser *P) { + if (AtName(P, "await")) { + Token *Start = NextTok(P); + PyObject *Value = ParsePrimary(P); + if (Value == NULL) { + return NULL; + } + PyObject *Argv[1] = { Value }; + PyObject *Node = NewNode(P, "Await", Argv, 1); + Py_DECREF(Value); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + return ParsePrimary(P); +} + +/* 调用尾部:'(' [args] ')' */ +static PyObject *ParseCallTail(Parser *P, PyObject *Func, Token *Start) { + if (!ExpectOp(P, "(")) { + return NULL; + } + PyObject *Args = NewList(); + PyObject *Keywords = NewList(); + if (Args == NULL || Keywords == NULL) { + Py_XDECREF(Args); + Py_XDECREF(Keywords); + return NULL; + } + SkipNL(P); + if (!AtOp(P, ")")) { + for (;;) { + SkipNL(P); + if (AtOp(P, "**")) { + NextTok(P); + PyObject *Value = ParseExpr(P); + if (Value == NULL) { + Py_DECREF(Args); + Py_DECREF(Keywords); + return NULL; + } + PyObject *Kw = MakeKeyword(P, NULL, Value); + Py_DECREF(Value); + if (Kw == NULL) { + Py_DECREF(Args); + Py_DECREF(Keywords); + return NULL; + } + PyList_Append(Keywords, Kw); + Py_DECREF(Kw); + } else { + if (AtOp(P, "*")) { + NextTok(P); + PyObject *Value = ParseExpr(P); + if (Value == NULL) { + Py_DECREF(Args); + Py_DECREF(Keywords); + return NULL; + } + PyObject *Starred = MakeStarred(P, Value); + Py_DECREF(Value); + if (Starred == NULL) { + Py_DECREF(Args); + Py_DECREF(Keywords); + return NULL; + } + FixLoc(P, Starred, PrevTok(P)); + PyList_Append(Args, Starred); + Py_DECREF(Starred); + } else { + Token *T0 = PeekTok(P, 0); + Token *T1 = PeekTok(P, 1); + if (T0->Kind == TK_NAME && T1->Kind == TK_OP && TokIsOp(T1, "=")) { + Token *NameTok = NextTok(P); + NextTok(P); /* '=' */ + PyObject *Value = ParseExpr(P); + if (Value == NULL) { + Py_DECREF(Args); + Py_DECREF(Keywords); + return NULL; + } + PyObject *Arg = PyUnicode_FromStringAndSize(NameTok->Start, + NameTok->End - NameTok->Start); + PyObject *Kw = MakeKeyword(P, Arg, Value); + Py_XDECREF(Arg); + Py_DECREF(Value); + if (Kw == NULL) { + Py_DECREF(Args); + Py_DECREF(Keywords); + return NULL; + } + PyList_Append(Keywords, Kw); + Py_DECREF(Kw); + } else { + if (PyList_GET_SIZE(Args) == 0 && PyList_GET_SIZE(Keywords) == 0) { + Token *GenStart = PeekTok(P, 0); + PyObject *GenElt = ParseExpr(P); + if (GenElt == NULL) { + Py_DECREF(Args); + Py_DECREF(Keywords); + return NULL; + } + if (AtName(P, "for")) { + PyObject *GenGens = ParseComprehensionGenerators(P); + if (GenGens == NULL) { + Py_DECREF(GenElt); + Py_DECREF(Args); + Py_DECREF(Keywords); + return NULL; + } + PyObject *GArgv[2] = { GenElt, GenGens }; + PyObject *Gen = NewNode(P, "GeneratorExp", GArgv, 2); + Py_DECREF(GenGens); + Py_DECREF(GenElt); + if (Gen == NULL) { + Py_DECREF(Args); + Py_DECREF(Keywords); + return NULL; + } + FixLoc(P, Gen, GenStart); + PyList_Append(Args, Gen); + Py_DECREF(Gen); + } else { + PyList_Append(Args, GenElt); + Py_DECREF(GenElt); + } + } else { + PyObject *E = ParseExpr(P); + if (E == NULL) { + Py_DECREF(Args); + Py_DECREF(Keywords); + return NULL; + } + PyList_Append(Args, E); + Py_DECREF(E); + } + } + } + } + SkipNL(P); + if (AtOp(P, ",")) { + NextTok(P); + SkipNL(P); + if (AtOp(P, ")")) { + break; + } + continue; + } + break; + } + } + if (!ExpectOp(P, ")")) { + Py_DECREF(Args); + Py_DECREF(Keywords); + return NULL; + } + PyObject *Argv[3] = { Func, Args, Keywords }; + PyObject *Node = NewNode(P, "Call", Argv, 3); + Py_DECREF(Args); + Py_DECREF(Keywords); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +static PyObject *ParseSliceList(Parser *P); +static PyObject *ParseSliceItem(Parser *P); + +/* 下标尾部:'[' slice_list ']' */ +static PyObject *ParseSubscriptTail(Parser *P, PyObject *Value, Token *Start) { + if (!ExpectOp(P, "[")) { + return NULL; + } + SkipNL(P); + PyObject *SliceNode = ParseSliceList(P); + if (SliceNode == NULL) { + return NULL; + } + SkipNL(P); + if (!ExpectOp(P, "]")) { + Py_DECREF(SliceNode); + return NULL; + } + PyObject *Load = MakeCtx(P, "Load"); + if (Load == NULL) { + Py_DECREF(SliceNode); + return NULL; + } + PyObject *Argv[3] = { Value, SliceNode, Load }; + PyObject *Node = NewNode(P, "Subscript", Argv, 3); + Py_DECREF(SliceNode); + Py_DECREF(Load); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +static PyObject *ParseSliceList(Parser *P) { + Token *Start = PeekTok(P, 0); + PyObject *Elts = NewList(); + if (Elts == NULL) { + return NULL; + } + PyObject *First = ParseSliceItem(P); + if (First == NULL) { + Py_DECREF(Elts); + return NULL; + } + PyList_Append(Elts, First); + while (AtOp(P, ",")) { + NextTok(P); + SkipNL(P); + PyObject *E = ParseSliceItem(P); + if (E == NULL) { + Py_DECREF(Elts); + Py_DECREF(First); + return NULL; + } + PyList_Append(Elts, E); + Py_DECREF(E); + } + if (PyList_GET_SIZE(Elts) == 1) { + Py_DECREF(Elts); + return First; + } + Py_DECREF(First); + PyObject *Load = MakeCtx(P, "Load"); + if (Load == NULL) { + Py_DECREF(Elts); + return NULL; + } + PyObject *Argv[2] = { Elts, Load }; + PyObject *Node = NewNode(P, "Tuple", Argv, 2); + Py_DECREF(Elts); + Py_DECREF(Load); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +static PyObject *ParseSliceItem(Parser *P) { + Token *Start = PeekTok(P, 0); + if (AtOp(P, ":")) { + NextTok(P); + PyObject *Upper = NULL; + if (!AtOp(P, ":") && !AtOp(P, ",")) { + if (!AtOp(P, "]")) { + Upper = ParseExpr(P); + if (Upper == NULL) { + return NULL; + } + } + } + PyObject *Step = NULL; + if (AtOp(P, ":")) { + NextTok(P); + if (!AtOp(P, "]") && !AtOp(P, ",") && !AtOp(P, ":")) { + Step = ParseExpr(P); + if (Step == NULL) { + Py_XDECREF(Upper); + return NULL; + } + } + } + PyObject *Argv[3] = { Py_None, Upper ? Upper : Py_None, Step ? Step : Py_None }; + PyObject *Node = NewNode(P, "Slice", Argv, 3); + Py_XDECREF(Upper); + Py_XDECREF(Step); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + PyObject *First = ParseExpr(P); + if (First == NULL) { + return NULL; + } + if (!AtOp(P, ":")) { + return First; + } + NextTok(P); + PyObject *Lower = First; + PyObject *Upper = NULL; + PyObject *Step = NULL; + if (!AtOp(P, ":") && !AtOp(P, "]") && !AtOp(P, ",")) { + Upper = ParseExpr(P); + if (Upper == NULL) { + Py_DECREF(Lower); + return NULL; + } + } + if (AtOp(P, ":")) { + NextTok(P); + if (!AtOp(P, "]") && !AtOp(P, ",")) { + Step = ParseExpr(P); + if (Step == NULL) { + Py_DECREF(Lower); + Py_XDECREF(Upper); + return NULL; + } + } + } + PyObject *Argv[3] = { Lower, Upper ? Upper : Py_None, Step ? Step : Py_None }; + PyObject *Node = NewNode(P, "Slice", Argv, 3); + Py_DECREF(Lower); + Py_XDECREF(Upper); + Py_XDECREF(Step); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* 主表达式:原子 + trailer(属性/调用/下标) */ +static PyObject *ParsePrimary(Parser *P) { + Token *Start = PeekTok(P, 0); + PyObject *Node = ParseAtom(P); + if (Node == NULL) { + return NULL; + } + for (;;) { + SkipNL(P); + Token *T = PeekTok(P, 0); + if (T->Kind == TK_OP && TokIsOp(T, ".")) { + NextTok(P); + Token *AttrTok = ExpectName(P, NULL); + if (AttrTok == NULL) { + Py_DECREF(Node); + return NULL; + } + PyObject *NewNode2 = MakeAttrNode(P, Node, AttrTok, Start); + Py_DECREF(Node); + if (NewNode2 == NULL) { + return NULL; + } + Node = NewNode2; + } else if (T->Kind == TK_OP && TokIsOp(T, "(")) { + PyObject *Call = ParseCallTail(P, Node, Start); + Py_DECREF(Node); + if (Call == NULL) { + return NULL; + } + Node = Call; + } else if (T->Kind == TK_OP && TokIsOp(T, "[")) { + PyObject *Sub = ParseSubscriptTail(P, Node, Start); + Py_DECREF(Node); + if (Sub == NULL) { + return NULL; + } + Node = Sub; + } else { + break; + } + } + return Node; +} + +/* 数字常量 */ +static PyObject *ParseNumber(Parser *P) { + Token *T = NextTok(P); + PyObject *Val = ParseNumberValue(P, T); + if (Val == NULL) { + return NULL; + } + PyObject *Node = MakeConstantPos(P, Val, T); + Py_DECREF(Val); + return Node; +} + +/* 原子表达式 */ +static PyObject *ParseAtom(Parser *P) { + SkipNL(P); + Token *T = PeekTok(P, 0); + if (T->Kind == TK_NUMBER) { + return ParseNumber(P); + } + if (T->Kind == TK_STRING || T->Kind == TK_FSTRING) { + return ParseStringConcat(P); + } + if (T->Kind == TK_NAME) { + if (TokIsName(T, "True") || TokIsName(T, "False") || TokIsName(T, "None")) { + NextTok(P); + PyObject *Val = TokIsName(T, "True") ? Py_True + : (TokIsName(T, "False") ? Py_False : Py_None); + PyObject *Node = MakeConstantPos(P, Val, T); + return Node; + } + NextTok(P); /* 必须消费名字 token,否则上层循环死循环(idx 不前) */ + return MakeNameNode(P, T, "Load"); + } + if (T->Kind == TK_OP) { + if (TokIsOp(T, "(")) { + return ParseParen(P); + } + if (TokIsOp(T, "[")) { + return ParseList(P); + } + if (TokIsOp(T, "{")) { + return ParseDictSet(P); + } + if (TokIsOp(T, "...")) { + NextTok(P); + return MakeConstantPos(P, Py_Ellipsis, T); + } + } + SyntaxErrorAt(P, T, "unexpected token"); + return NULL; +} + +/* 括号:空元组 / yield / walrus / 生成器表达式 / 元组 / 普通表达式 */ +static PyObject *ParseParen(Parser *P) { + Token *Start = NextTok(P); /* '(' */ + SkipNL(P); + if (AtOp(P, ")")) { + NextTok(P); + PyObject *Load = MakeCtx(P, "Load"); + if (Load == NULL) { + return NULL; + } + PyObject *Empty = NewList(); + if (Empty == NULL) { + Py_DECREF(Load); + return NULL; + } + PyObject *Argv[2] = { Empty, Load }; + PyObject *Node = NewNode(P, "Tuple", Argv, 2); + Py_DECREF(Empty); + Py_DECREF(Load); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + if (AtName(P, "yield")) { + PyObject *YieldNode = ParseYield(P); + if (YieldNode == NULL) { + return NULL; + } + SkipNL(P); + if (!ExpectOp(P, ")")) { + Py_DECREF(YieldNode); + return NULL; + } + return YieldNode; + } + Token *T0 = PeekTok(P, 0); + Token *T1 = PeekTok(P, 1); + if (T0->Kind == TK_NAME && T1->Kind == TK_OP && TokIsOp(T1, ":=")) { + Token *NameTok = NextTok(P); + NextTok(P); /* ':=' */ + PyObject *Value = ParseExpr(P); + if (Value == NULL) { + return NULL; + } + PyObject *Target = MakeNameNode(P, NameTok, "Store"); + if (Target == NULL) { + Py_DECREF(Value); + return NULL; + } + SkipNL(P); + if (!ExpectOp(P, ")")) { + Py_DECREF(Target); + Py_DECREF(Value); + return NULL; + } + PyObject *Argv[2] = { Target, Value }; + PyObject *Node = NewNode(P, "NamedExpr", Argv, 2); + Py_DECREF(Target); + Py_DECREF(Value); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + PyObject *First = ParseExpr(P); + if (First == NULL) { + return NULL; + } + SkipNL(P); + if (AtName(P, "for")) { + PyObject *Generators = ParseComprehensionGenerators(P); + if (Generators == NULL) { + Py_DECREF(First); + return NULL; + } + SkipNL(P); + if (!ExpectOp(P, ")")) { + Py_DECREF(Generators); + Py_DECREF(First); + return NULL; + } + PyObject *Argv[2] = { First, Generators }; + PyObject *Gen = NewNode(P, "GeneratorExp", Argv, 2); + Py_DECREF(Generators); + Py_DECREF(First); + if (Gen != NULL) { + FixLoc(P, Gen, Start); + } + return Gen; + } + if (AtOp(P, ",")) { + PyObject *Elts = NewList(); + if (Elts == NULL) { + Py_DECREF(First); + return NULL; + } + PyList_Append(Elts, First); + while (AtOp(P, ",")) { + NextTok(P); + SkipNL(P); + if (AtOp(P, ")")) { + break; + } + PyObject *E = ParseExpr(P); + if (E == NULL) { + Py_DECREF(Elts); + Py_DECREF(First); + return NULL; + } + PyList_Append(Elts, E); + Py_DECREF(E); + SkipNL(P); + } + if (!ExpectOp(P, ")")) { + Py_DECREF(Elts); + Py_DECREF(First); + return NULL; + } + Py_DECREF(First); + PyObject *Load = MakeCtx(P, "Load"); + if (Load == NULL) { + Py_DECREF(Elts); + return NULL; + } + PyObject *Argv[2] = { Elts, Load }; + PyObject *Node = NewNode(P, "Tuple", Argv, 2); + Py_DECREF(Elts); + Py_DECREF(Load); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + if (!ExpectOp(P, ")")) { + Py_DECREF(First); + return NULL; + } + return First; +} + +/* 列表字面量 / 列表推导式 */ +static PyObject *ParseList(Parser *P) { + Token *Start = NextTok(P); /* '[' */ + SkipNL(P); + if (AtOp(P, "]")) { + NextTok(P); + PyObject *Load = MakeCtx(P, "Load"); + if (Load == NULL) { + return NULL; + } + PyObject *Empty = NewList(); + if (Empty == NULL) { + Py_DECREF(Load); + return NULL; + } + PyObject *Argv[2] = { Empty, Load }; + PyObject *Node = NewNode(P, "List", Argv, 2); + Py_DECREF(Empty); + Py_DECREF(Load); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + PyObject *First = ParseExpr(P); + if (First == NULL) { + return NULL; + } + SkipNL(P); + if (AtName(P, "for")) { + PyObject *Generators = ParseComprehensionGenerators(P); + if (Generators == NULL) { + Py_DECREF(First); + return NULL; + } + SkipNL(P); + if (!ExpectOp(P, "]")) { + Py_DECREF(Generators); + Py_DECREF(First); + return NULL; + } + PyObject *Argv[2] = { First, Generators }; + PyObject *Lc = NewNode(P, "ListComp", Argv, 2); + Py_DECREF(Generators); + Py_DECREF(First); + if (Lc != NULL) { + FixLoc(P, Lc, Start); + } + return Lc; + } + PyObject *Elts = NewList(); + if (Elts == NULL) { + Py_DECREF(First); + return NULL; + } + PyList_Append(Elts, First); + while (AtOp(P, ",")) { + NextTok(P); + SkipNL(P); + if (AtOp(P, "]")) { + break; + } + PyObject *E = ParseExpr(P); + if (E == NULL) { + Py_DECREF(Elts); + Py_DECREF(First); + return NULL; + } + PyList_Append(Elts, E); + Py_DECREF(E); + SkipNL(P); + } + if (!ExpectOp(P, "]")) { + Py_DECREF(Elts); + Py_DECREF(First); + return NULL; + } + Py_DECREF(First); + PyObject *Load = MakeCtx(P, "Load"); + if (Load == NULL) { + Py_DECREF(Elts); + return NULL; + } + PyObject *Argv[2] = { Elts, Load }; + PyObject *Node = NewNode(P, "List", Argv, 2); + Py_DECREF(Elts); + Py_DECREF(Load); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* 字典 / 集合字面量与推导式 */ +static PyObject *ParseDictSet(Parser *P) { + Token *Start = NextTok(P); /* '{' */ + SkipNL(P); + if (AtOp(P, "}")) { + NextTok(P); + PyObject *Keys = NewList(); + PyObject *Values = NewList(); + if (Keys == NULL || Values == NULL) { + Py_XDECREF(Keys); + Py_XDECREF(Values); + return NULL; + } + PyObject *Argv[2] = { Keys, Values }; + PyObject *Node = NewNode(P, "Dict", Argv, 2); + Py_DECREF(Keys); + Py_DECREF(Values); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + PyObject *Keys = NewList(); + PyObject *Values = NewList(); + if (Keys == NULL || Values == NULL) { + Py_XDECREF(Keys); + Py_XDECREF(Values); + return NULL; + } + for (;;) { + SkipNL(P); + if (AtOp(P, "**")) { + NextTok(P); + PyObject *Val = ParseExpr(P); + if (Val == NULL) { + Py_DECREF(Keys); + Py_DECREF(Values); + return NULL; + } + PyList_Append(Keys, Py_None); + PyList_Append(Values, Val); + Py_DECREF(Val); + } else { + PyObject *Key = ParseExpr(P); + if (Key == NULL) { + Py_DECREF(Keys); + Py_DECREF(Values); + return NULL; + } + SkipNL(P); + if (AtOp(P, ":")) { + NextTok(P); + SkipNL(P); + PyObject *Value = ParseExpr(P); + if (Value == NULL) { + Py_DECREF(Key); + Py_DECREF(Keys); + Py_DECREF(Values); + return NULL; + } + PyList_Append(Keys, Key); + PyList_Append(Values, Value); + Py_DECREF(Value); + Py_DECREF(Key); + } else { + /* 集合元素 */ + PyObject *Elts = NewList(); + if (Elts == NULL) { + Py_DECREF(Key); + Py_DECREF(Keys); + Py_DECREF(Values); + return NULL; + } + PyList_Append(Elts, Key); + Py_DECREF(Key); + while (AtOp(P, ",")) { + NextTok(P); + SkipNL(P); + if (AtOp(P, "}")) { + break; + } + PyObject *E = ParseExpr(P); + if (E == NULL) { + Py_DECREF(Elts); + Py_DECREF(Keys); + Py_DECREF(Values); + return NULL; + } + PyList_Append(Elts, E); + Py_DECREF(E); + SkipNL(P); + } + SkipNL(P); + if (AtName(P, "for")) { + PyObject *Generators = ParseComprehensionGenerators(P); + if (Generators == NULL) { + Py_DECREF(Elts); + Py_DECREF(Keys); + Py_DECREF(Values); + return NULL; + } + SkipNL(P); + if (!ExpectOp(P, "}")) { + Py_DECREF(Generators); + Py_DECREF(Elts); + Py_DECREF(Keys); + Py_DECREF(Values); + return NULL; + } + PyObject *Elt0 = PyList_GET_ITEM(Elts, 0); + Py_INCREF(Elt0); + PyObject *Argv[2] = { Elt0, Generators }; + PyObject *Sc = NewNode(P, "SetComp", Argv, 2); + Py_DECREF(Elt0); + Py_DECREF(Generators); + Py_DECREF(Elts); + Py_DECREF(Keys); + Py_DECREF(Values); + if (Sc != NULL) { + FixLoc(P, Sc, Start); + } + return Sc; + } + if (!ExpectOp(P, "}")) { + Py_DECREF(Elts); + Py_DECREF(Keys); + Py_DECREF(Values); + return NULL; + } + PyObject *Argv[1] = { Elts }; + PyObject *St = NewNode(P, "Set", Argv, 1); + Py_DECREF(Elts); + Py_DECREF(Keys); + Py_DECREF(Values); + if (St != NULL) { + FixLoc(P, St, Start); + } + return St; + } + } + SkipNL(P); + if (AtOp(P, ",")) { + NextTok(P); + SkipNL(P); + if (AtOp(P, "}")) { + break; + } continue; } + break; + } + /* 字典推导式 */ + SkipNL(P); + if (AtName(P, "for")) { + PyObject *Generators = ParseComprehensionGenerators(P); + if (Generators == NULL) { + Py_DECREF(Keys); + Py_DECREF(Values); + return NULL; + } + SkipNL(P); + if (!ExpectOp(P, "}")) { + Py_DECREF(Generators); + Py_DECREF(Keys); + Py_DECREF(Values); + return NULL; + } + PyObject *Key0 = PyList_GET_ITEM(Keys, 0); + PyObject *Val0 = PyList_GET_ITEM(Values, 0); + PyObject *KeyObj; + if (Key0 == Py_None) { + KeyObj = MakeConstant(P, Py_None); + if (KeyObj == NULL) { + Py_DECREF(Generators); + Py_DECREF(Keys); + Py_DECREF(Values); + return NULL; + } + } else { + KeyObj = Key0; + Py_INCREF(KeyObj); + } + Py_INCREF(Val0); + PyObject *Argv[3] = { KeyObj, Val0, Generators }; + PyObject *Dc = NewNode(P, "DictComp", Argv, 3); + Py_DECREF(KeyObj); + Py_DECREF(Val0); + Py_DECREF(Generators); + Py_DECREF(Keys); + Py_DECREF(Values); + if (Dc != NULL) { + FixLoc(P, Dc, Start); + } + return Dc; + } + if (!ExpectOp(P, "}")) { + Py_DECREF(Keys); + Py_DECREF(Values); + return NULL; + } + PyObject *Argv[2] = { Keys, Values }; + PyObject *Node = NewNode(P, "Dict", Argv, 2); + Py_DECREF(Keys); + Py_DECREF(Values); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* 推导式生成器子句:for target in iter [if cond]* [async for ...] */ +static PyObject *ParseComprehensionGenerators(Parser *P) { + PyObject *Generators = NewList(); + if (Generators == NULL) { + return NULL; + } + for (;;) { + int IsAsync = 0; + if (AtName(P, "async") && PeekTok(P, 1)->Kind == TK_NAME + && TokIsName(PeekTok(P, 1), "for")) { + NextTok(P); + IsAsync = 1; + } + if (!AtName(P, "for")) { + break; + } + Token *Start = NextTok(P); + PyObject *Target = ParseComprehensionTarget(P); + if (Target == NULL) { + Py_DECREF(Generators); + return NULL; + } + if (!ExpectName(P, "in")) { + Py_DECREF(Target); + Py_DECREF(Generators); + return NULL; + } + PyObject *Iterable = ParseDisjunction(P); + if (Iterable == NULL) { + Py_DECREF(Target); + Py_DECREF(Generators); + return NULL; + } + PyObject *Ifs = NewList(); + if (Ifs == NULL) { + Py_DECREF(Target); + Py_DECREF(Iterable); + Py_DECREF(Generators); + return NULL; + } + while (AtName(P, "if")) { + NextTok(P); + PyObject *Cond = ParseDisjunction(P); + if (Cond == NULL) { + Py_DECREF(Target); + Py_DECREF(Iterable); + Py_DECREF(Ifs); + Py_DECREF(Generators); + return NULL; + } + PyList_Append(Ifs, Cond); + Py_DECREF(Cond); + } + PyObject *IsAsyncObj = PyLong_FromLong(IsAsync); + PyObject *Argv[4] = { Target, Iterable, Ifs, IsAsyncObj }; + PyObject *Comp = NewNode(P, "comprehension", Argv, 4); + Py_DECREF(IsAsyncObj); + Py_DECREF(Target); + Py_DECREF(Iterable); + Py_DECREF(Ifs); + if (Comp == NULL) { + Py_DECREF(Generators); + return NULL; + } + /* comprehension 无位置属性,不调用 FixLoc */ + (void)Start; + PyList_Append(Generators, Comp); + Py_DECREF(Comp); + if (AtName(P, "for") || (AtName(P, "async") && PeekTok(P, 1)->Kind == TK_NAME + && TokIsName(PeekTok(P, 1), "for"))) { + continue; + } + break; + } + return Generators; +} + +/* 推导式 for 目标(支持裸元组解包) */ +static PyObject *ParseComprehensionTarget(Parser *P) { + Token *Start = PeekTok(P, 0); + PyObject *First = ParseTarget(P); + if (First == NULL) { + return NULL; + } + if (!AtOp(P, ",")) { + return First; + } + PyObject *Elts = NewList(); + if (Elts == NULL) { + Py_DECREF(First); + return NULL; + } + PyList_Append(Elts, First); + while (AtOp(P, ",")) { + NextTok(P); + SkipNL(P); + PyObject *E = ParseTarget(P); + if (E == NULL) { + Py_DECREF(Elts); + Py_DECREF(First); + return NULL; + } + PyList_Append(Elts, E); + Py_DECREF(E); + } + Py_DECREF(First); + PyObject *Store = MakeCtx(P, "Store"); + if (Store == NULL) { + Py_DECREF(Elts); + return NULL; + } + PyObject *Argv[2] = { Elts, Store }; + PyObject *Node = NewNode(P, "Tuple", Argv, 2); + Py_DECREF(Elts); + Py_DECREF(Store); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* 赋值目标(Store 上下文) */ +static PyObject *ParseTarget(Parser *P) { + Token *Start = PeekTok(P, 0); + if (AtOp(P, "(")) { + NextTok(P); + SkipNL(P); + PyObject *Elts = NewList(); + if (Elts == NULL) { + return NULL; + } + if (!AtOp(P, ")")) { + for (;;) { + PyObject *E = ParseTarget(P); + if (E == NULL) { + Py_DECREF(Elts); + return NULL; + } + PyList_Append(Elts, E); + Py_DECREF(E); + SkipNL(P); + if (AtOp(P, ",")) { + NextTok(P); + SkipNL(P); + if (AtOp(P, ")")) { + break; + } + continue; + } + break; + } + } + if (!ExpectOp(P, ")")) { + Py_DECREF(Elts); + return NULL; + } + PyObject *Store = MakeCtx(P, "Store"); + if (Store == NULL) { + Py_DECREF(Elts); + return NULL; + } + PyObject *Argv[2] = { Elts, Store }; + PyObject *Node = NewNode(P, "Tuple", Argv, 2); + Py_DECREF(Elts); + Py_DECREF(Store); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + if (AtOp(P, "[")) { + NextTok(P); + SkipNL(P); + PyObject *Elts = NewList(); + if (Elts == NULL) { + return NULL; + } + if (!AtOp(P, "]")) { + for (;;) { + PyObject *E = ParseTarget(P); + if (E == NULL) { + Py_DECREF(Elts); + return NULL; + } + PyList_Append(Elts, E); + Py_DECREF(E); + SkipNL(P); + if (AtOp(P, ",")) { + NextTok(P); + SkipNL(P); + if (AtOp(P, "]")) { + break; + } + continue; + } + break; + } + } + if (!ExpectOp(P, "]")) { + Py_DECREF(Elts); + return NULL; + } + PyObject *Store = MakeCtx(P, "Store"); + if (Store == NULL) { + Py_DECREF(Elts); + return NULL; + } + PyObject *Argv[2] = { Elts, Store }; + PyObject *Node = NewNode(P, "List", Argv, 2); + Py_DECREF(Elts); + Py_DECREF(Store); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + if (AtOp(P, "*")) { + NextTok(P); + PyObject *Value = ParseTarget(P); + if (Value == NULL) { + return NULL; + } + PyObject *Store = MakeCtx(P, "Store"); + if (Store == NULL) { + Py_DECREF(Value); + return NULL; + } + PyObject *Argv[2] = { Value, Store }; + PyObject *Node = NewNode(P, "Starred", Argv, 2); + Py_DECREF(Value); + Py_DECREF(Store); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + Token *T = PeekTok(P, 0); + if (T->Kind == TK_NAME) { + NextTok(P); /* 必须消费名字 token */ + return MakeNameNode(P, T, "Store"); + } + /* 属性/下标目标:解析 Load 后改 Store */ + PyObject *Node = ParsePrimary(P); + if (Node == NULL) { + return NULL; + } + SetStoreContext(P, Node); + return Node; +} + +/* lambda */ +static PyObject *ParseLambda(Parser *P) { + Token *Start = NextTok(P); /* 'lambda' */ + PyObject *Args; + if (AtOp(P, ":")) { + PyObject *E1 = NewList(); + PyObject *E2 = NewList(); + if (E1 == NULL || E2 == NULL) { + Py_XDECREF(E1); + Py_XDECREF(E2); + return NULL; + } + PyObject *Argv[7]; + Argv[0] = E1; /* posonlyargs */ + Argv[1] = E2; /* args */ + Argv[2] = Py_None; /* vararg */ + Argv[3] = NewList(); /* kwonlyargs */ + Argv[4] = NewList(); /* kw_defaults */ + Argv[5] = Py_None; /* kwarg */ + Argv[6] = NewList(); /* defaults */ + if (Argv[3] == NULL || Argv[4] == NULL || Argv[6] == NULL) { + Py_XDECREF(E1); + Py_XDECREF(E2); + Py_XDECREF(Argv[3]); + Py_XDECREF(Argv[4]); + Py_XDECREF(Argv[6]); + return NULL; + } + Args = NewNode(P, "arguments", Argv, 7); + Py_DECREF(E1); + Py_DECREF(E2); + Py_DECREF(Argv[3]); + Py_DECREF(Argv[4]); + Py_DECREF(Argv[6]); + if (Args == NULL) { + return NULL; + } + } else { + Args = ParseArguments(P, 1, 1); + if (Args == NULL) { + return NULL; + } + } + if (!ExpectOp(P, ":")) { + Py_DECREF(Args); + return NULL; + } + PyObject *Body = ParseExpr(P); + if (Body == NULL) { + Py_DECREF(Args); + return NULL; + } + PyObject *Argv[2] = { Args, Body }; + PyObject *Node = NewNode(P, "Lambda", Argv, 2); + Py_DECREF(Args); + Py_DECREF(Body); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* yield 表达式 */ +static PyObject *ParseYield(Parser *P) { + Token *Start = NextTok(P); /* 'yield' */ + if (AtName(P, "from")) { + NextTok(P); + PyObject *Value = ParseExpr(P); + if (Value == NULL) { + return NULL; + } + PyObject *Argv[1] = { Value }; + PyObject *Node = NewNode(P, "YieldFrom", Argv, 1); + Py_DECREF(Value); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + if (AtOp(P, ")")) { + PyObject *Node = NewNode(P, "Yield", (PyObject *[]){ Py_None }, 1); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + PyObject *Value = ParseExprList(P); + if (Value == NULL) { + return NULL; + } + PyObject *Argv[1] = { Value }; + PyObject *Node = NewNode(P, "Yield", Argv, 1); + Py_DECREF(Value); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* 参数列表(lambda 与函数定义共用) */ +static PyObject *ParseArguments(Parser *P, int SkipOpen, int AllowLambda) { + if (!SkipOpen) { + if (!ExpectOp(P, "(")) { + return NULL; + } + } + SkipNL(P); + PyObject *PosonlyArgs = NewList(); + PyObject *Args = NewList(); + PyObject *KwonlyArgs = NewList(); + PyObject *KwDefaults = NewList(); + PyObject *OrderedArgs = NewList(); + PyObject *OrderedDefaults = NewList(); + PyObject *Vararg = NULL; + PyObject *Kwarg = NULL; + if (PosonlyArgs == NULL || Args == NULL || KwonlyArgs == NULL + || KwDefaults == NULL || OrderedArgs == NULL || OrderedDefaults == NULL) { + Py_XDECREF(PosonlyArgs); + Py_XDECREF(Args); + Py_XDECREF(KwonlyArgs); + Py_XDECREF(KwDefaults); + Py_XDECREF(OrderedArgs); + Py_XDECREF(OrderedDefaults); + return NULL; + } + int SeenStar = 0; + if (!AtOp(P, ")")) { + for (;;) { + SkipNL(P); + Token *T = PeekTok(P, 0); + if (TokIsOp(T, ")")) { + break; + } + if (TokIsOp(T, "*")) { + NextTok(P); + if (AtOp(P, ",")) { + SeenStar = 1; + NextTok(P); + if (AtOp(P, ")") || AtOp(P, ":")) { + break; + } + continue; + } + Vararg = ParseArg(P, !AllowLambda); + if (Vararg == NULL) { + Py_DECREF(PosonlyArgs); + Py_DECREF(Args); + Py_DECREF(KwonlyArgs); + Py_DECREF(KwDefaults); + Py_DECREF(OrderedArgs); + Py_DECREF(OrderedDefaults); + return NULL; + } + SeenStar = 1; + } else if (TokIsOp(T, "**")) { + NextTok(P); + Kwarg = ParseArg(P, !AllowLambda); + if (Kwarg == NULL) { + Py_DECREF(PosonlyArgs); + Py_DECREF(Args); + Py_DECREF(KwonlyArgs); + Py_DECREF(KwDefaults); + Py_DECREF(OrderedArgs); + Py_DECREF(OrderedDefaults); + return NULL; + } + } else { + PyObject *A = ParseArg(P, !AllowLambda); + if (A == NULL) { + Py_DECREF(PosonlyArgs); + Py_DECREF(Args); + Py_DECREF(KwonlyArgs); + Py_DECREF(KwDefaults); + Py_DECREF(OrderedArgs); + Py_DECREF(OrderedDefaults); + return NULL; + } + PyObject *Def = Py_None; + if (AtOp(P, "=")) { + NextTok(P); + Def = ParseExpr(P); + if (Def == NULL) { + Py_DECREF(A); + Py_DECREF(PosonlyArgs); + Py_DECREF(Args); + Py_DECREF(KwonlyArgs); + Py_DECREF(KwDefaults); + Py_DECREF(OrderedArgs); + Py_DECREF(OrderedDefaults); + return NULL; + } + } + if (SeenStar) { + PyList_Append(KwonlyArgs, A); + PyList_Append(KwDefaults, Def); + } else { + PyList_Append(OrderedArgs, A); + PyList_Append(OrderedDefaults, Def); + } + Py_DECREF(A); + Py_XDECREF(Def != Py_None ? Def : NULL); + } + SkipNL(P); + if (AtOp(P, ",")) { + NextTok(P); + if (AtOp(P, ")")) { + break; + } + continue; + } + break; + } + } + Py_ssize_t NOrdered = PyList_GET_SIZE(OrderedArgs); + int LastNone = -1; + for (Py_ssize_t I = 0; I < NOrdered; I++) { + if (PyList_GET_ITEM(OrderedDefaults, I) == Py_None) { + LastNone = (int)I; + } + } + for (Py_ssize_t I = 0; I < NOrdered; I++) { + PyList_Append(Args, PyList_GET_ITEM(OrderedArgs, I)); + } + PyObject *Defaults = NewList(); + if (Defaults == NULL) { + Py_DECREF(PosonlyArgs); + Py_DECREF(Args); + Py_DECREF(KwonlyArgs); + Py_DECREF(KwDefaults); + Py_DECREF(OrderedArgs); + Py_DECREF(OrderedDefaults); + Py_XDECREF(Vararg); + Py_XDECREF(Kwarg); + return NULL; + } + for (int I = LastNone + 1; I < (int)NOrdered; I++) { + PyObject *D = PyList_GET_ITEM(OrderedDefaults, I); + if (D != Py_None) { + PyList_Append(Defaults, D); + } + } + SkipNL(P); + if (!SkipOpen) { + if (!ExpectOp(P, ")")) { + Py_DECREF(PosonlyArgs); + Py_DECREF(Args); + Py_DECREF(KwonlyArgs); + Py_DECREF(KwDefaults); + Py_DECREF(OrderedArgs); + Py_DECREF(OrderedDefaults); + Py_DECREF(Defaults); + Py_XDECREF(Vararg); + Py_XDECREF(Kwarg); + return NULL; + } + } + PyObject *Argv[7]; + Argv[0] = PosonlyArgs; + Argv[1] = Args; + Argv[2] = Vararg ? Vararg : Py_None; + Argv[3] = KwonlyArgs; + Argv[4] = KwDefaults; + Argv[5] = Kwarg ? Kwarg : Py_None; + Argv[6] = Defaults; + PyObject *Node = NewNode(P, "arguments", Argv, 7); + Py_DECREF(PosonlyArgs); + Py_DECREF(Args); + Py_DECREF(KwonlyArgs); + Py_DECREF(KwDefaults); + Py_DECREF(OrderedArgs); + Py_DECREF(OrderedDefaults); + Py_DECREF(Defaults); + Py_XDECREF(Vararg); + Py_XDECREF(Kwarg); + return Node; +} + +/* 单个参数:name [':' annotation] */ +static PyObject *ParseArg(Parser *P, int AllowAnnotation) { + Token *Start = ExpectName(P, NULL); + if (Start == NULL) { + return NULL; + } + PyObject *Annotation = NULL; + if (AllowAnnotation && AtOp(P, ":")) { + NextTok(P); + Annotation = ParseExpr(P); + if (Annotation == NULL) { + return NULL; + } + } + PyObject *Arg = PyUnicode_FromStringAndSize(Start->Start, Start->End - Start->Start); + if (Arg == NULL) { + Py_XDECREF(Annotation); + return NULL; + } + PyObject *Argv[3] = { Arg, Annotation ? Annotation : Py_None, Py_None }; + PyObject *Node = NewNode(P, "arg", Argv, 3); + Py_DECREF(Arg); + Py_XDECREF(Annotation); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* ================= 语句解析 ================= */ + +static PyObject *ParseStatement(Parser *P); +static PyObject *ParseSimpleStmt(Parser *P); +static PyObject *ParseSuite(Parser *P); +static PyObject *ParseIf(Parser *P); +static PyObject *ParseWhile(Parser *P); +static PyObject *ParseFor(Parser *P, int IsAsync); +static PyObject *ParseTry(Parser *P); +static PyObject *ParseWith(Parser *P, int IsAsync); +static PyObject *ParseMatch(Parser *P); +static PyObject *ParseFunctionDef(Parser *P, int IsAsync, PyObject *Decorators); +static PyObject *ParseClassDef(Parser *P, PyObject *Decorators); +static PyObject *ParseExprList(Parser *P); +static PyObject *TryParseAnnAssign(Parser *P); +static PyObject *ParseExprStmt(Parser *P); +static PyObject *ParseImport(Parser *P); +static PyObject *ParseImportFrom(Parser *P); +static PyObject *ParseImportAlias(Parser *P); +static PyObject *ParseGlobal(Parser *P, const char *NodeName); +static PyObject *ParseDelete(Parser *P); +static PyObject *ParseAssert(Parser *P); +static PyObject *ParseDecorated(Parser *P); +static PyObject *ParseTypeParams(Parser *P); +static PyObject *ParseExceptHandler(Parser *P); +static PyObject *ParseTargetList(Parser *P); + +/* 模块:语句序列 */ +static PyObject *ParseModule(Parser *P) { + PyObject *Body = NewList(); + if (Body == NULL) { + return NULL; + } + SkipNEWLINE(P); + while (PeekTok(P, 0)->Kind != TK_ENDMARKER) { PyObject *Stmt = ParseStatement(P); if (Stmt == NULL) { - break; + Py_DECREF(Body); + return NULL; } PyList_Append(Body, Stmt); Py_DECREF(Stmt); + while (AtOp(P, ";")) { + NextTok(P); + if (PeekTok(P, 0)->Kind == TK_NEWLINE) { + break; + } + PyObject *Extra = ParseStatement(P); + if (Extra == NULL) { + Py_DECREF(Body); + return NULL; + } + PyList_Append(Body, Extra); + Py_DECREF(Extra); + } + SkipNEWLINE(P); } - PyObject *Ti = PyList_New(0); + PyObject *Ti = NewList(); if (Ti == NULL) { Py_DECREF(Body); return NULL; } - PyObject *Args[2]; - Args[0] = Body; - Args[1] = Ti; - PyObject *Mod = NodeMake(P, "Module", Args, 2); + PyObject *Argv[2] = { Body, Ti }; + PyObject *Mod = NewNode(P, "Module", Argv, 2); + Py_DECREF(Body); + Py_DECREF(Ti); return Mod; } +/* 语句分派(对齐 vast Parser.ParseStatement) */ +static PyObject *ParseStatement(Parser *P) { + Token *T = PeekTok(P, 0); + if (T->Kind == TK_NAME) { + if (TokIsName(T, "extern")) { + return ParseFunctionDef(P, 0, NULL); /* extern 简化为普通 def(测试未覆盖,宽松) */ + } + if (TokIsName(T, "export")) { + return ParseFunctionDef(P, 0, NULL); + } + if ((TokIsName(T, "fn") || TokIsName(T, "fun") || TokIsName(T, "func") + || TokIsName(T, "function")) + && PeekTok(P, 1)->Kind == TK_NAME) { + return ParseFunctionDef(P, 0, NULL); + } + if (TokIsName(T, "async")) { + Token *N = PeekTok(P, 1); + if (N->Kind == TK_NAME && TokIsName(N, "def")) { + return ParseFunctionDef(P, 1, NULL); + } + if (N->Kind == TK_NAME && TokIsName(N, "for")) { + return ParseFor(P, 1); + } + if (N->Kind == TK_NAME && TokIsName(N, "with")) { + return ParseWith(P, 1); + } + } + if (TokIsName(T, "def")) { + return ParseFunctionDef(P, 0, NULL); + } + if (TokIsName(T, "class")) { + return ParseClassDef(P, NULL); + } + if (TokIsName(T, "if")) { + return ParseIf(P); + } + if (TokIsName(T, "while")) { + return ParseWhile(P); + } + if (TokIsName(T, "for")) { + return ParseFor(P, 0); + } + if (TokIsName(T, "try")) { + return ParseTry(P); + } + if (TokIsName(T, "with")) { + return ParseWith(P, 0); + } + if (TokIsName(T, "match")) { + int SaveIdx = P->Idx; + PyObject *M = ParseMatch(P); + if (M != NULL) { + return M; + } + if (PyErr_Occurred()) { + return NULL; + } + P->Idx = SaveIdx; + P->Error = 0; + /* 回退为表达式语句 */ + } + if (TokIsName(T, "return")) { + Token *Start = NextTok(P); + PyObject *Value = NULL; + if (PeekTok(P, 0)->Kind != TK_NEWLINE && !AtOp(P, ";")) { + Value = ParseExprList(P); + if (Value == NULL) { + return NULL; + } + } + PyObject *Argv[1] = { Value ? Value : Py_None }; + PyObject *Node = NewNode(P, "Return", Argv, 1); + Py_XDECREF(Value); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + if (TokIsName(T, "raise")) { + Token *Start = NextTok(P); + PyObject *Exc = NULL; + PyObject *Cause = NULL; + if (PeekTok(P, 0)->Kind != TK_NEWLINE && !AtOp(P, ";")) { + Exc = ParseExpr(P); + if (Exc == NULL) { + return NULL; + } + if (AtName(P, "from")) { + NextTok(P); + Cause = ParseExpr(P); + if (Cause == NULL) { + Py_DECREF(Exc); + return NULL; + } + } + } + PyObject *Argv[2] = { Exc ? Exc : Py_None, Cause ? Cause : Py_None }; + PyObject *Node = NewNode(P, "Raise", Argv, 2); + Py_XDECREF(Exc); + Py_XDECREF(Cause); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + if (TokIsName(T, "import")) { + return ParseImport(P); + } + if (TokIsName(T, "from")) { + return ParseImportFrom(P); + } + if (TokIsName(T, "global")) { + return ParseGlobal(P, "Global"); + } + if (TokIsName(T, "nonlocal")) { + return ParseGlobal(P, "Nonlocal"); + } + if (TokIsName(T, "del")) { + return ParseDelete(P); + } + if (TokIsName(T, "assert")) { + return ParseAssert(P); + } + if (TokIsName(T, "pass")) { + Token *Start = NextTok(P); + PyObject *Node = NewNode(P, "Pass", NULL, 0); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + if (TokIsName(T, "break")) { + Token *Start = NextTok(P); + PyObject *Node = NewNode(P, "Break", NULL, 0); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + if (TokIsName(T, "continue")) { + Token *Start = NextTok(P); + PyObject *Node = NewNode(P, "Continue", NULL, 0); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + if (TokIsName(T, "yield")) { + return ParseExprStmt(P); + } + } + if (T->Kind == TK_OP && TokIsOp(T, "@")) { + return ParseDecorated(P); + } + return ParseSimpleStmt(P); +} + +/* 简单语句:AnnAssign / Assign / AugAssign / Expr */ +static PyObject *ParseSimpleStmt(Parser *P) { + Token *Start = PeekTok(P, 0); + PyObject *Ann = TryParseAnnAssign(P); + if (Ann != NULL) { + return Ann; + } + if (PyErr_Occurred()) { + return NULL; + } + PyObject *Expr = ParseExprList(P); + if (Expr == NULL) { + return NULL; + } + /* 赋值链 */ + if (AtOp(P, "=")) { + PyObject *Targets = NewList(); + if (Targets == NULL) { + Py_DECREF(Expr); + return NULL; + } + SetStoreContext(P, Expr); + PyList_Append(Targets, Expr); + for (;;) { + NextTok(P); /* '=' */ + PyObject *Value = ParseExprList(P); + if (Value == NULL) { + Py_DECREF(Targets); + Py_DECREF(Expr); + return NULL; + } + if (AtOp(P, "=")) { + SetStoreContext(P, Value); + PyList_Append(Targets, Value); + Py_DECREF(Value); + continue; + } + Py_DECREF(Expr); + PyObject *Argv[3] = { Targets, Value, Py_None }; + PyObject *Node = NewNode(P, "Assign", Argv, 3); + Py_DECREF(Targets); + Py_DECREF(Value); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + } + /* 增强赋值 */ + static const struct { const char *Op; const char *NodeName; } AugOps[] = { + {"+=", "Add"}, {"-=", "Sub"}, {"*=", "Mult"}, {"/=", "Div"}, + {"//=", "FloorDiv"}, {"%=", "Mod"}, {"**=", "Pow"}, {"<<=", "LShift"}, + {">>=", "RShift"}, {"&=", "BitAnd"}, {"|=", "BitOr"}, {"^=", "BitXor"}, + {"@=", "MatMult"}, + }; + for (Py_ssize_t I = 0; I < (Py_ssize_t)(sizeof(AugOps) / sizeof(AugOps[0])); I++) { + if (AtOp(P, AugOps[I].Op)) { + NextTok(P); + PyObject *Value = ParseExpr(P); + if (Value == NULL) { + Py_DECREF(Expr); + return NULL; + } + SetStoreContext(P, Expr); + PyObject *Op = MakeSingleton(P, AugOps[I].NodeName); + if (Op == NULL) { + Py_DECREF(Expr); + Py_DECREF(Value); + return NULL; + } + PyObject *Argv[3] = { Expr, Op, Value }; + PyObject *Node = NewNode(P, "AugAssign", Argv, 3); + Py_DECREF(Op); + Py_DECREF(Value); + Py_DECREF(Expr); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + } + PyObject *Argv[1] = { Expr }; + PyObject *Node = NewNode(P, "Expr", Argv, 1); + Py_DECREF(Expr); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* 尝试解析带注解赋值(对齐 vast _TryParseAnnAssign) */ +static PyObject *TryParseAnnAssign(Parser *P) { + Token *Start = PeekTok(P, 0); + Token *T0 = PeekTok(P, 0); + Token *T1 = PeekTok(P, 1); + if (T0->Kind == TK_NAME && T1->Kind == TK_OP && TokIsOp(T1, ":")) { + Token *NameTok = NextTok(P); + NextTok(P); /* ':' */ + PyObject *Annotation = ParseExpr(P); + if (Annotation == NULL) { + return NULL; + } + PyObject *Value = NULL; + if (AtOp(P, "=")) { + NextTok(P); + Value = ParseExpr(P); + if (Value == NULL) { + Py_DECREF(Annotation); + return NULL; + } + } + PyObject *Target = MakeNameNode(P, NameTok, "Store"); + if (Target == NULL) { + Py_DECREF(Annotation); + Py_XDECREF(Value); + return NULL; + } + PyObject *Argv[4] = { Target, Annotation, Value ? Value : Py_None, PyLong_FromLong(1) }; + PyObject *Node = NewNode(P, "AnnAssign", Argv, 4); + Py_DECREF(Argv[3]); + Py_DECREF(Target); + Py_DECREF(Annotation); + Py_XDECREF(Value); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + if (T0->Kind == TK_NAME + || (T0->Kind == TK_OP && (TokIsOp(T0, "(") || TokIsOp(T0, "[")))) { + int SaveIdx = P->Idx; + int SaveError = P->Error; + PyObject *Expr = ParseExpr(P); + if (Expr == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + P->Idx = SaveIdx; + P->Error = SaveError; + return NULL; + } + if (AtOp(P, ":") && IsNameOrAttrOrSub(Expr)) { + NextTok(P); + PyObject *Annotation = ParseExpr(P); + if (Annotation == NULL) { + Py_DECREF(Expr); + return NULL; + } + PyObject *Value = NULL; + if (AtOp(P, "=")) { + NextTok(P); + Value = ParseExpr(P); + if (Value == NULL) { + Py_DECREF(Annotation); + Py_DECREF(Expr); + return NULL; + } + } + SetStoreContext(P, Expr); + int Simple = NodeIs(Expr, "Name") ? 1 : 0; + PyObject *Argv[4] = { Expr, Annotation, Value ? Value : Py_None, + PyLong_FromLong(Simple) }; + PyObject *Node = NewNode(P, "AnnAssign", Argv, 4); + Py_DECREF(Argv[3]); + Py_DECREF(Annotation); + Py_XDECREF(Value); + Py_DECREF(Expr); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + P->Idx = SaveIdx; + P->Error = SaveError; + Py_DECREF(Expr); + } + return NULL; +} + +/* 表达式语句(含 yield) */ +static PyObject *ParseExprStmt(Parser *P) { + Token *Start = PeekTok(P, 0); + PyObject *Expr; + if (AtName(P, "yield")) { + Expr = ParseYield(P); + } else { + Expr = ParseExprList(P); + } + if (Expr == NULL) { + return NULL; + } + PyObject *Argv[1] = { Expr }; + PyObject *Node = NewNode(P, "Expr", Argv, 1); + Py_DECREF(Expr); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* import a.b.c [as x], ... */ +static PyObject *ParseImport(Parser *P) { + Token *Start = NextTok(P); /* 'import' */ + PyObject *Names = NewList(); + if (Names == NULL) { + return NULL; + } + for (;;) { + Token *NameTok = ExpectName(P, NULL); + if (NameTok == NULL) { + Py_DECREF(Names); + return NULL; + } + Token *EndTok = NameTok; + PyObject *Parts = PyList_New(0); + if (Parts == NULL) { + Py_DECREF(Names); + return NULL; + } + PyObject *FirstPart = PyUnicode_FromStringAndSize(NameTok->Start, + NameTok->End - NameTok->Start); + if (FirstPart == NULL) { + Py_DECREF(Parts); + Py_DECREF(Names); + return NULL; + } + PyList_Append(Parts, FirstPart); + Py_DECREF(FirstPart); + while (AtOp(P, ".")) { + NextTok(P); + Token *PartTok = ExpectName(P, NULL); + if (PartTok == NULL) { + Py_DECREF(Parts); + Py_DECREF(Names); + return NULL; + } + EndTok = PartTok; + PyObject *Part = PyUnicode_FromStringAndSize(PartTok->Start, + PartTok->End - PartTok->Start); + if (Part == NULL) { + Py_DECREF(Parts); + Py_DECREF(Names); + return NULL; + } + PyList_Append(Parts, Part); + Py_DECREF(Part); + } + PyObject *Sep = PyUnicode_FromString("."); + PyObject *Full = PyUnicode_Join(Sep, Parts); + Py_DECREF(Sep); + Py_DECREF(Parts); + if (Full == NULL) { + Py_DECREF(Names); + return NULL; + } + PyObject *Asname = NULL; + if (AtName(P, "as")) { + NextTok(P); + Token *AsTok = ExpectName(P, NULL); + if (AsTok == NULL) { + Py_DECREF(Full); + Py_DECREF(Names); + return NULL; + } + EndTok = AsTok; + Asname = PyUnicode_FromStringAndSize(AsTok->Start, AsTok->End - AsTok->Start); + if (Asname == NULL) { + Py_DECREF(Full); + Py_DECREF(Names); + return NULL; + } + } + PyObject *Argv[2] = { Full, Asname ? Asname : Py_None }; + PyObject *Alias = NewNode(P, "alias", Argv, 2); + Py_DECREF(Full); + Py_XDECREF(Asname); + if (Alias == NULL) { + Py_DECREF(Names); + return NULL; + } + /* alias 无位置属性 */ + (void)NameTok; + (void)EndTok; + PyList_Append(Names, Alias); + Py_DECREF(Alias); + if (AtOp(P, ",")) { + NextTok(P); + continue; + } + break; + } + PyObject *Argv[1] = { Names }; + PyObject *Node = NewNode(P, "Import", Argv, 1); + Py_DECREF(Names); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* from [.]*module import ... */ +static PyObject *ParseImportFrom(Parser *P) { + Token *Start = NextTok(P); /* 'from' */ + int Level = 0; + while (AtOp(P, ".")) { + NextTok(P); + Level++; + } + PyObject *Module = NULL; + if (!AtName(P, "import")) { + Token *NameTok = ExpectName(P, NULL); + if (NameTok == NULL) { + return NULL; + } + PyObject *Parts = PyList_New(0); + if (Parts == NULL) { + return NULL; + } + PyObject *FirstPart = PyUnicode_FromStringAndSize(NameTok->Start, + NameTok->End - NameTok->Start); + if (FirstPart == NULL) { + Py_DECREF(Parts); + return NULL; + } + PyList_Append(Parts, FirstPart); + Py_DECREF(FirstPart); + while (AtOp(P, ".")) { + NextTok(P); + Token *PartTok = ExpectName(P, NULL); + if (PartTok == NULL) { + Py_DECREF(Parts); + return NULL; + } + PyObject *Part = PyUnicode_FromStringAndSize(PartTok->Start, + PartTok->End - PartTok->Start); + if (Part == NULL) { + Py_DECREF(Parts); + return NULL; + } + PyList_Append(Parts, Part); + Py_DECREF(Part); + } + PyObject *Sep = PyUnicode_FromString("."); + Module = PyUnicode_Join(Sep, Parts); + Py_DECREF(Sep); + Py_DECREF(Parts); + if (Module == NULL) { + return NULL; + } + } + if (!ExpectName(P, "import")) { + Py_XDECREF(Module); + return NULL; + } + PyObject *Names = NewList(); + if (Names == NULL) { + Py_XDECREF(Module); + return NULL; + } + if (AtOp(P, "(")) { + NextTok(P); + SkipNL(P); + if (!AtOp(P, ")")) { + for (;;) { + PyObject *Alias = ParseImportAlias(P); + if (Alias == NULL) { + Py_DECREF(Names); + Py_XDECREF(Module); + return NULL; + } + PyList_Append(Names, Alias); + Py_DECREF(Alias); + SkipNL(P); + if (AtOp(P, ",")) { + NextTok(P); + SkipNL(P); + if (AtOp(P, ")")) { + break; + } + continue; + } + break; + } + } + if (!ExpectOp(P, ")")) { + Py_DECREF(Names); + Py_XDECREF(Module); + return NULL; + } + } else { + if (AtOp(P, "*")) { + Token *StarTok = NextTok(P); + PyObject *Argv[2] = { PyUnicode_FromString("*"), Py_None }; + if (Argv[0] == NULL) { + Py_DECREF(Names); + Py_XDECREF(Module); + return NULL; + } + PyObject *Alias = NewNode(P, "alias", Argv, 2); + Py_DECREF(Argv[0]); + if (Alias == NULL) { + Py_DECREF(Names); + Py_XDECREF(Module); + return NULL; + } + (void)StarTok; + PyList_Append(Names, Alias); + Py_DECREF(Alias); + } else { + for (;;) { + PyObject *Alias = ParseImportAlias(P); + if (Alias == NULL) { + Py_DECREF(Names); + Py_XDECREF(Module); + return NULL; + } + PyList_Append(Names, Alias); + Py_DECREF(Alias); + if (AtOp(P, ",")) { + NextTok(P); + continue; + } + break; + } + } + } + PyObject *LevelObj = PyLong_FromLong(Level); + PyObject *Argv[3] = { Module ? Module : Py_None, Names, LevelObj }; + PyObject *Node = NewNode(P, "ImportFrom", Argv, 3); + Py_DECREF(LevelObj); + Py_XDECREF(Module); + Py_DECREF(Names); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* 单个 import 别名:name [as alias] */ +static PyObject *ParseImportAlias(Parser *P) { + Token *NameTok = ExpectName(P, NULL); + if (NameTok == NULL) { + return NULL; + } + PyObject *Asname = NULL; + if (AtName(P, "as")) { + NextTok(P); + Token *AsTok = ExpectName(P, NULL); + if (AsTok == NULL) { + return NULL; + } + Asname = PyUnicode_FromStringAndSize(AsTok->Start, AsTok->End - AsTok->Start); + if (Asname == NULL) { + return NULL; + } + } + PyObject *Name = PyUnicode_FromStringAndSize(NameTok->Start, NameTok->End - NameTok->Start); + if (Name == NULL) { + Py_XDECREF(Asname); + return NULL; + } + PyObject *Argv[2] = { Name, Asname ? Asname : Py_None }; + PyObject *Alias = NewNode(P, "alias", Argv, 2); + Py_DECREF(Name); + Py_XDECREF(Asname); + return Alias; +} + +/* global / nonlocal */ +static PyObject *ParseGlobal(Parser *P, const char *NodeName) { + Token *Start = NextTok(P); + PyObject *Names = NewList(); + if (Names == NULL) { + return NULL; + } + for (;;) { + Token *N = ExpectName(P, NULL); + if (N == NULL) { + Py_DECREF(Names); + return NULL; + } + PyObject *Name = PyUnicode_FromStringAndSize(N->Start, N->End - N->Start); + if (Name == NULL) { + Py_DECREF(Names); + return NULL; + } + PyList_Append(Names, Name); + Py_DECREF(Name); + if (AtOp(P, ",")) { + NextTok(P); + continue; + } + break; + } + PyObject *Argv[1] = { Names }; + PyObject *Node = NewNode(P, NodeName, Argv, 1); + Py_DECREF(Names); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* del target, ... */ +static PyObject *ParseDelete(Parser *P) { + Token *Start = NextTok(P); /* 'del' */ + PyObject *Targets = NewList(); + if (Targets == NULL) { + return NULL; + } + for (;;) { + PyObject *Expr = ParseExpr(P); + if (Expr == NULL) { + Py_DECREF(Targets); + return NULL; + } + SetDelContext(P, Expr); + PyList_Append(Targets, Expr); + Py_DECREF(Expr); + if (AtOp(P, ",")) { + NextTok(P); + continue; + } + break; + } + PyObject *Argv[1] = { Targets }; + PyObject *Node = NewNode(P, "Delete", Argv, 1); + Py_DECREF(Targets); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* assert test [, msg] */ +static PyObject *ParseAssert(Parser *P) { + Token *Start = NextTok(P); /* 'assert' */ + PyObject *Test = ParseExpr(P); + if (Test == NULL) { + return NULL; + } + PyObject *Msg = NULL; + if (AtOp(P, ",")) { + NextTok(P); + Msg = ParseExpr(P); + if (Msg == NULL) { + Py_DECREF(Test); + return NULL; + } + } + PyObject *Argv[2] = { Test, Msg ? Msg : Py_None }; + PyObject *Node = NewNode(P, "Assert", Argv, 2); + Py_DECREF(Test); + Py_XDECREF(Msg); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* 装饰器 + def/class */ +static PyObject *ParseDecorated(Parser *P) { + PyObject *DecoratorList = NewList(); + if (DecoratorList == NULL) { + return NULL; + } + while (AtOp(P, "@")) { + Token *DecoStart = NextTok(P); + PyObject *Deco = ParseExpr(P); + if (Deco == NULL) { + Py_DECREF(DecoratorList); + return NULL; + } + FixLocEnd(P, Deco, DecoStart, NULL); + PyList_Append(DecoratorList, Deco); + Py_DECREF(Deco); + if (!ExpectNEWLINE(P)) { + Py_DECREF(DecoratorList); + return NULL; + } + SkipNEWLINE(P); + } + Token *T = PeekTok(P, 0); + if (T->Kind == TK_NAME && TokIsName(T, "extern")) { + return ParseFunctionDef(P, 0, DecoratorList); + } + if (T->Kind == TK_NAME && TokIsName(T, "export")) { + return ParseFunctionDef(P, 0, DecoratorList); + } + if (T->Kind == TK_NAME && TokIsName(T, "def")) { + return ParseFunctionDef(P, 0, DecoratorList); + } + if (T->Kind == TK_NAME && TokIsName(T, "async") + && PeekTok(P, 1)->Kind == TK_NAME && TokIsName(PeekTok(P, 1), "def")) { + return ParseFunctionDef(P, 1, DecoratorList); + } + if (T->Kind == TK_NAME && TokIsName(T, "class")) { + return ParseClassDef(P, DecoratorList); + } + SyntaxErrorAt(P, T, "decorator must be followed by def/class"); + Py_DECREF(DecoratorList); + return NULL; +} + +/* def / fn / fun / func / function(含 async def) */ +static PyObject *ParseFunctionDef(Parser *P, int IsAsync, PyObject *Decorators) { + Token *Start = NextTok(P); + if (Start->Kind == TK_NAME && TokIsName(Start, "async")) { + Start = NextTok(P); /* def */ + } + Token *NameTok = ExpectName(P, NULL); + if (NameTok == NULL) { + return NULL; + } + PyObject *TypeParams = NewList(); + if (TypeParams == NULL) { + return NULL; + } + if (AtOp(P, "[")) { + Py_DECREF(TypeParams); + TypeParams = ParseTypeParams(P); + if (TypeParams == NULL) { + return NULL; + } + } + PyObject *Args = ParseArguments(P, 0, 0); + if (Args == NULL) { + Py_DECREF(TypeParams); + return NULL; + } + PyObject *Returns = NULL; + if (AtOp(P, "->")) { + NextTok(P); + Returns = ParseExpr(P); + if (Returns == NULL) { + Py_DECREF(Args); + Py_DECREF(TypeParams); + return NULL; + } + } + if (!ExpectOp(P, ":")) { + Py_DECREF(Args); + Py_XDECREF(Returns); + Py_DECREF(TypeParams); + return NULL; + } + PyObject *Body = ParseSuite(P); + if (Body == NULL) { + Py_DECREF(Args); + Py_XDECREF(Returns); + Py_DECREF(TypeParams); + return NULL; + } + PyObject *Name = PyUnicode_FromStringAndSize(NameTok->Start, NameTok->End - NameTok->Start); + if (Name == NULL) { + Py_DECREF(Args); + Py_XDECREF(Returns); + Py_DECREF(TypeParams); + Py_DECREF(Body); + return NULL; + } + PyObject *Argv[7]; + Argv[0] = Name; + Argv[1] = Args; + Argv[2] = Body; + Argv[3] = Decorators ? Decorators : NewList(); + Argv[4] = Returns ? Returns : Py_None; + Argv[5] = Py_None; /* type_comment */ + Argv[6] = TypeParams; + if (Argv[3] == NULL) { + Py_DECREF(Name); + Py_DECREF(Args); + Py_DECREF(Body); + Py_XDECREF(Returns); + Py_DECREF(TypeParams); + return NULL; + } + PyObject *Node = NewNode(P, IsAsync ? "AsyncFunctionDef" : "FunctionDef", Argv, 7); + Py_DECREF(Name); + Py_DECREF(Args); + Py_DECREF(Body); + if (Argv[3] != Decorators) { + Py_DECREF(Argv[3]); + } + Py_XDECREF(Returns); + Py_DECREF(TypeParams); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* PEP 695 类型参数:[T, *Ts, **P, U: Bound] */ +static PyObject *ParseTypeParams(Parser *P) { + if (!ExpectOp(P, "[")) { + return NULL; + } + SkipNL(P); + PyObject *Params = NewList(); + if (Params == NULL) { + return NULL; + } + if (!AtOp(P, "]")) { + for (;;) { + SkipNL(P); + int Star = 0; + if (AtOp(P, "*")) { + NextTok(P); + Star = 1; + } else if (AtOp(P, "**")) { + NextTok(P); + Star = 2; + } + Token *NameTok = ExpectName(P, NULL); + if (NameTok == NULL) { + Py_DECREF(Params); + return NULL; + } + PyObject *Bound = NULL; + if (Star == 0 && AtOp(P, ":")) { + NextTok(P); + Bound = ParseExpr(P); + if (Bound == NULL) { + Py_DECREF(Params); + return NULL; + } + } + PyObject *Name = PyUnicode_FromStringAndSize(NameTok->Start, + NameTok->End - NameTok->Start); + if (Name == NULL) { + Py_XDECREF(Bound); + Py_DECREF(Params); + return NULL; + } + PyObject *Node = NULL; + if (Star == 1) { + PyObject *Argv[1] = { Name }; + Node = NewNode(P, "TypeVarTuple", Argv, 1); + } else if (Star == 2) { + PyObject *Argv[1] = { Name }; + Node = NewNode(P, "ParamSpec", Argv, 1); + } else { + PyObject *Argv[2] = { Name, Bound ? Bound : Py_None }; + Node = NewNode(P, "TypeVar", Argv, 2); + } + Py_DECREF(Name); + Py_XDECREF(Bound); + if (Node == NULL) { + Py_DECREF(Params); + return NULL; + } + FixLoc(P, Node, NameTok); + PyList_Append(Params, Node); + Py_DECREF(Node); + SkipNL(P); + if (AtOp(P, ",")) { + NextTok(P); + continue; + } + break; + } + } + if (!ExpectOp(P, "]")) { + Py_DECREF(Params); + return NULL; + } + return Params; +} + +/* class 定义 */ +static PyObject *ParseClassDef(Parser *P, PyObject *Decorators) { + Token *Start = NextTok(P); /* class */ + Token *NameTok = ExpectName(P, NULL); + if (NameTok == NULL) { + return NULL; + } + PyObject *TypeParams = NewList(); + if (TypeParams == NULL) { + return NULL; + } + if (AtOp(P, "[")) { + Py_DECREF(TypeParams); + TypeParams = ParseTypeParams(P); + if (TypeParams == NULL) { + return NULL; + } + } + PyObject *Bases = NewList(); + PyObject *Keywords = NewList(); + if (Bases == NULL || Keywords == NULL) { + Py_XDECREF(Bases); + Py_XDECREF(Keywords); + Py_DECREF(TypeParams); + return NULL; + } + if (AtOp(P, "(")) { + NextTok(P); + SkipNL(P); + if (!AtOp(P, ")")) { + for (;;) { + SkipNL(P); + if (AtOp(P, "**")) { + NextTok(P); + PyObject *Value = ParseExpr(P); + if (Value == NULL) { + Py_DECREF(Bases); + Py_DECREF(Keywords); + Py_DECREF(TypeParams); + return NULL; + } + PyObject *Kw = MakeKeyword(P, NULL, Value); + Py_DECREF(Value); + if (Kw == NULL) { + Py_DECREF(Bases); + Py_DECREF(Keywords); + Py_DECREF(TypeParams); + return NULL; + } + PyList_Append(Keywords, Kw); + Py_DECREF(Kw); + } else { + Token *A0 = PeekTok(P, 0); + Token *A1 = PeekTok(P, 1); + if (A0->Kind == TK_NAME && A1->Kind == TK_OP && TokIsOp(A1, "=")) { + Token *ArgTok = NextTok(P); + NextTok(P); /* '=' */ + PyObject *Value = ParseExpr(P); + if (Value == NULL) { + Py_DECREF(Bases); + Py_DECREF(Keywords); + Py_DECREF(TypeParams); + return NULL; + } + PyObject *Arg = PyUnicode_FromStringAndSize(ArgTok->Start, + ArgTok->End - ArgTok->Start); + PyObject *Kw = MakeKeyword(P, Arg, Value); + Py_XDECREF(Arg); + Py_DECREF(Value); + if (Kw == NULL) { + Py_DECREF(Bases); + Py_DECREF(Keywords); + Py_DECREF(TypeParams); + return NULL; + } + PyList_Append(Keywords, Kw); + Py_DECREF(Kw); + } else { + PyObject *Base = ParseExpr(P); + if (Base == NULL) { + Py_DECREF(Bases); + Py_DECREF(Keywords); + Py_DECREF(TypeParams); + return NULL; + } + PyList_Append(Bases, Base); + Py_DECREF(Base); + } + } + SkipNL(P); + if (AtOp(P, ",")) { + NextTok(P); + SkipNL(P); + if (AtOp(P, ")")) { + break; + } + continue; + } + break; + } + } + if (!ExpectOp(P, ")")) { + Py_DECREF(Bases); + Py_DECREF(Keywords); + Py_DECREF(TypeParams); + return NULL; + } + } + if (!ExpectOp(P, ":")) { + Py_DECREF(Bases); + Py_DECREF(Keywords); + Py_DECREF(TypeParams); + return NULL; + } + PyObject *Body = ParseSuite(P); + if (Body == NULL) { + Py_DECREF(Bases); + Py_DECREF(Keywords); + Py_DECREF(TypeParams); + return NULL; + } + PyObject *Name = PyUnicode_FromStringAndSize(NameTok->Start, NameTok->End - NameTok->Start); + if (Name == NULL) { + Py_DECREF(Bases); + Py_DECREF(Keywords); + Py_DECREF(TypeParams); + Py_DECREF(Body); + return NULL; + } + PyObject *Argv[6]; + Argv[0] = Name; + Argv[1] = Bases; + Argv[2] = Keywords; + Argv[3] = Body; + Argv[4] = Decorators ? Decorators : NewList(); + Argv[5] = TypeParams; + if (Argv[4] == NULL) { + Py_DECREF(Name); + Py_DECREF(Bases); + Py_DECREF(Keywords); + Py_DECREF(Body); + Py_DECREF(TypeParams); + return NULL; + } + PyObject *Node = NewNode(P, "ClassDef", Argv, 6); + Py_DECREF(Name); + Py_DECREF(Bases); + Py_DECREF(Keywords); + Py_DECREF(Body); + if (Argv[4] != Decorators) { + Py_DECREF(Argv[4]); + } + Py_DECREF(TypeParams); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* 语句块(套件) */ +static PyObject *ParseSuite(Parser *P) { + if (PeekTok(P, 0)->Kind == TK_NEWLINE) { + NextTok(P); + while (PeekTok(P, 0)->Kind == TK_NL) { + NextTok(P); + } + if (PeekTok(P, 0)->Kind == TK_INDENT) { + NextTok(P); + PyObject *Body = NewList(); + if (Body == NULL) { + return NULL; + } + SkipNEWLINE(P); + while (PeekTok(P, 0)->Kind != TK_DEDENT + && PeekTok(P, 0)->Kind != TK_ENDMARKER) { + PyObject *Stmt = ParseStatement(P); + if (Stmt == NULL) { + Py_DECREF(Body); + return NULL; + } + PyList_Append(Body, Stmt); + Py_DECREF(Stmt); + while (AtOp(P, ";")) { + NextTok(P); + if (PeekTok(P, 0)->Kind == TK_NEWLINE) { + break; + } + PyObject *Extra = ParseStatement(P); + if (Extra == NULL) { + Py_DECREF(Body); + return NULL; + } + PyList_Append(Body, Extra); + Py_DECREF(Extra); + } + SkipNEWLINE(P); + } + Token *T = NextTok(P); + if (T->Kind != TK_DEDENT) { + SyntaxErrorAt(P, T, "expected DEDENT"); + Py_DECREF(Body); + return NULL; + } + return Body; + } + /* 空套件(宽松) */ + return NewList(); + } + /* 单行套件 */ + PyObject *Body = NewList(); + if (Body == NULL) { + return NULL; + } + PyObject *Stmt = ParseStatement(P); + if (Stmt == NULL) { + Py_DECREF(Body); + return NULL; + } + PyList_Append(Body, Stmt); + Py_DECREF(Stmt); + while (AtOp(P, ";")) { + NextTok(P); + if (PeekTok(P, 0)->Kind == TK_NEWLINE) { + break; + } + PyObject *Extra = ParseStatement(P); + if (Extra == NULL) { + Py_DECREF(Body); + return NULL; + } + PyList_Append(Body, Extra); + Py_DECREF(Extra); + } + if (!ExpectNEWLINE(P)) { + Py_DECREF(Body); + return NULL; + } + return Body; +} + +/* if / elif / else */ +static PyObject *ParseIf(Parser *P) { + Token *Start = NextTok(P); /* if */ + PyObject *Test = ParseExpr(P); + if (Test == NULL) { + return NULL; + } + if (!ExpectOp(P, ":")) { + Py_DECREF(Test); + return NULL; + } + PyObject *Body = ParseSuite(P); + if (Body == NULL) { + Py_DECREF(Test); + return NULL; + } + PyObject *Orelse = NewList(); + if (Orelse == NULL) { + Py_DECREF(Test); + Py_DECREF(Body); + return NULL; + } + if (AtName(P, "elif")) { + PyObject *Nested = ParseIf(P); + if (Nested == NULL) { + Py_DECREF(Test); + Py_DECREF(Body); + Py_DECREF(Orelse); + return NULL; + } + PyList_Append(Orelse, Nested); + Py_DECREF(Nested); + } else if (AtName(P, "else")) { + NextTok(P); + if (!ExpectOp(P, ":")) { + Py_DECREF(Test); + Py_DECREF(Body); + Py_DECREF(Orelse); + return NULL; + } + PyObject *ElseBody = ParseSuite(P); + if (ElseBody == NULL) { + Py_DECREF(Test); + Py_DECREF(Body); + Py_DECREF(Orelse); + return NULL; + } + Py_DECREF(Orelse); + Orelse = ElseBody; + } + PyObject *Argv[3] = { Test, Body, Orelse }; + PyObject *Node = NewNode(P, "If", Argv, 3); + Py_DECREF(Test); + Py_DECREF(Body); + Py_DECREF(Orelse); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* while / else */ +static PyObject *ParseWhile(Parser *P) { + Token *Start = NextTok(P); /* while */ + PyObject *Test = ParseExpr(P); + if (Test == NULL) { + return NULL; + } + if (!ExpectOp(P, ":")) { + Py_DECREF(Test); + return NULL; + } + PyObject *Body = ParseSuite(P); + if (Body == NULL) { + Py_DECREF(Test); + return NULL; + } + PyObject *Orelse = NewList(); + if (Orelse == NULL) { + Py_DECREF(Test); + Py_DECREF(Body); + return NULL; + } + if (AtName(P, "else")) { + NextTok(P); + if (!ExpectOp(P, ":")) { + Py_DECREF(Test); + Py_DECREF(Body); + Py_DECREF(Orelse); + return NULL; + } + PyObject *ElseBody = ParseSuite(P); + if (ElseBody == NULL) { + Py_DECREF(Test); + Py_DECREF(Body); + Py_DECREF(Orelse); + return NULL; + } + Py_DECREF(Orelse); + Orelse = ElseBody; + } + PyObject *Argv[3] = { Test, Body, Orelse }; + PyObject *Node = NewNode(P, "While", Argv, 3); + Py_DECREF(Test); + Py_DECREF(Body); + Py_DECREF(Orelse); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* for / async for */ +static PyObject *ParseFor(Parser *P, int IsAsync) { + Token *Start = NextTok(P); + if (Start->Kind == TK_NAME && TokIsName(Start, "async")) { + Start = NextTok(P); /* for */ + } + PyObject *Target = ParseTargetList(P); + if (Target == NULL) { + return NULL; + } + if (!ExpectName(P, "in")) { + Py_DECREF(Target); + return NULL; + } + PyObject *Iterable = ParseExpr(P); + if (Iterable == NULL) { + Py_DECREF(Target); + return NULL; + } + if (!ExpectOp(P, ":")) { + Py_DECREF(Target); + Py_DECREF(Iterable); + return NULL; + } + PyObject *Body = ParseSuite(P); + if (Body == NULL) { + Py_DECREF(Target); + Py_DECREF(Iterable); + return NULL; + } + PyObject *Orelse = NewList(); + if (Orelse == NULL) { + Py_DECREF(Target); + Py_DECREF(Iterable); + Py_DECREF(Body); + return NULL; + } + if (AtName(P, "else")) { + NextTok(P); + if (!ExpectOp(P, ":")) { + Py_DECREF(Target); + Py_DECREF(Iterable); + Py_DECREF(Body); + Py_DECREF(Orelse); + return NULL; + } + PyObject *ElseBody = ParseSuite(P); + if (ElseBody == NULL) { + Py_DECREF(Target); + Py_DECREF(Iterable); + Py_DECREF(Body); + Py_DECREF(Orelse); + return NULL; + } + Py_DECREF(Orelse); + Orelse = ElseBody; + } + PyObject *Argv[5] = { Target, Iterable, Body, Orelse, Py_None }; + PyObject *Node = NewNode(P, IsAsync ? "AsyncFor" : "For", Argv, 5); + Py_DECREF(Target); + Py_DECREF(Iterable); + Py_DECREF(Body); + Py_DECREF(Orelse); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* for 目标列表(支持裸元组解包) */ +static PyObject *ParseTargetList(Parser *P) { + Token *Start = PeekTok(P, 0); + PyObject *First = ParseTarget(P); + if (First == NULL) { + return NULL; + } + if (!AtOp(P, ",")) { + return First; + } + PyObject *Elts = NewList(); + if (Elts == NULL) { + Py_DECREF(First); + return NULL; + } + PyList_Append(Elts, First); + while (AtOp(P, ",")) { + NextTok(P); + SkipNL(P); + PyObject *E = ParseTarget(P); + if (E == NULL) { + Py_DECREF(Elts); + Py_DECREF(First); + return NULL; + } + PyList_Append(Elts, E); + Py_DECREF(E); + } + Py_DECREF(First); + PyObject *Store = MakeCtx(P, "Store"); + if (Store == NULL) { + Py_DECREF(Elts); + return NULL; + } + PyObject *Argv[2] = { Elts, Store }; + PyObject *Node = NewNode(P, "Tuple", Argv, 2); + Py_DECREF(Elts); + Py_DECREF(Store); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* try / except* / else / finally */ +static PyObject *ParseTry(Parser *P) { + Token *Start = NextTok(P); /* try */ + if (!ExpectOp(P, ":")) { + return NULL; + } + PyObject *Body = ParseSuite(P); + if (Body == NULL) { + return NULL; + } + PyObject *Handlers = NewList(); + PyObject *Orelse = NewList(); + PyObject *Finalbody = NewList(); + if (Handlers == NULL || Orelse == NULL || Finalbody == NULL) { + Py_XDECREF(Handlers); + Py_XDECREF(Orelse); + Py_XDECREF(Finalbody); + Py_DECREF(Body); + return NULL; + } + int IsStar = 0; + while (AtName(P, "except")) { + NextTok(P); + if (AtOp(P, "*")) { + IsStar = 1; + NextTok(P); + } + PyObject *Handler = ParseExceptHandler(P); + if (Handler == NULL) { + Py_DECREF(Body); + Py_DECREF(Handlers); + Py_DECREF(Orelse); + Py_DECREF(Finalbody); + return NULL; + } + PyList_Append(Handlers, Handler); + Py_DECREF(Handler); + } + if (AtName(P, "else")) { + NextTok(P); + if (!ExpectOp(P, ":")) { + Py_DECREF(Body); + Py_DECREF(Handlers); + Py_DECREF(Orelse); + Py_DECREF(Finalbody); + return NULL; + } + PyObject *ElseBody = ParseSuite(P); + if (ElseBody == NULL) { + Py_DECREF(Body); + Py_DECREF(Handlers); + Py_DECREF(Orelse); + Py_DECREF(Finalbody); + return NULL; + } + Py_DECREF(Orelse); + Orelse = ElseBody; + } + if (AtName(P, "finally")) { + NextTok(P); + if (!ExpectOp(P, ":")) { + Py_DECREF(Body); + Py_DECREF(Handlers); + Py_DECREF(Orelse); + Py_DECREF(Finalbody); + return NULL; + } + PyObject *FinBody = ParseSuite(P); + if (FinBody == NULL) { + Py_DECREF(Body); + Py_DECREF(Handlers); + Py_DECREF(Orelse); + Py_DECREF(Finalbody); + return NULL; + } + Py_DECREF(Finalbody); + Finalbody = FinBody; + } + PyObject *Argv[4] = { Body, Handlers, Orelse, Finalbody }; + PyObject *Node = NewNode(P, IsStar ? "TryStar" : "Try", Argv, 4); + Py_DECREF(Body); + Py_DECREF(Handlers); + Py_DECREF(Orelse); + Py_DECREF(Finalbody); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* except [expr [as name]] ':' suite */ +static PyObject *ParseExceptHandler(Parser *P) { + Token *Start = PeekTok(P, 0); + PyObject *ExcType = NULL; + PyObject *Name = NULL; + if (!AtOp(P, ":")) { + ExcType = ParseExpr(P); + if (ExcType == NULL) { + return NULL; + } + if (AtName(P, "as")) { + NextTok(P); + Token *NameTok = ExpectName(P, NULL); + if (NameTok == NULL) { + Py_DECREF(ExcType); + return NULL; + } + Name = PyUnicode_FromStringAndSize(NameTok->Start, NameTok->End - NameTok->Start); + if (Name == NULL) { + Py_DECREF(ExcType); + return NULL; + } + } + } + if (!ExpectOp(P, ":")) { + Py_XDECREF(ExcType); + Py_XDECREF(Name); + return NULL; + } + PyObject *Body = ParseSuite(P); + if (Body == NULL) { + Py_XDECREF(ExcType); + Py_XDECREF(Name); + return NULL; + } + PyObject *Argv[3] = { ExcType ? ExcType : Py_None, Name ? Name : Py_None, Body }; + PyObject *Node = NewNode(P, "ExceptHandler", Argv, 3); + Py_XDECREF(ExcType); + Py_XDECREF(Name); + Py_DECREF(Body); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* with / async with */ +static PyObject *ParseWith(Parser *P, int IsAsync) { + Token *Start = NextTok(P); + if (Start->Kind == TK_NAME && TokIsName(Start, "async")) { + Start = NextTok(P); /* with */ + } + PyObject *Items = NewList(); + if (Items == NULL) { + return NULL; + } + for (;;) { + PyObject *Ctx = ParseExpr(P); + if (Ctx == NULL) { + Py_DECREF(Items); + return NULL; + } + PyObject *OptionalVars = NULL; + if (AtName(P, "as")) { + NextTok(P); + OptionalVars = ParseTarget(P); + if (OptionalVars == NULL) { + Py_DECREF(Ctx); + Py_DECREF(Items); + return NULL; + } + } + PyObject *Argv[2] = { Ctx, OptionalVars ? OptionalVars : Py_None }; + PyObject *Item = NewNode(P, "withitem", Argv, 2); + Py_DECREF(Ctx); + Py_XDECREF(OptionalVars); + if (Item == NULL) { + Py_DECREF(Items); + return NULL; + } + /* withitem 无位置属性 */ + (void)Start; + PyList_Append(Items, Item); + Py_DECREF(Item); + if (AtOp(P, ",")) { + NextTok(P); + continue; + } + break; + } + if (!ExpectOp(P, ":")) { + Py_DECREF(Items); + return NULL; + } + PyObject *Body = ParseSuite(P); + if (Body == NULL) { + Py_DECREF(Items); + return NULL; + } + PyObject *Argv[3] = { Items, Body, Py_None }; + PyObject *Node = NewNode(P, IsAsync ? "AsyncWith" : "With", Argv, 3); + Py_DECREF(Items); + Py_DECREF(Body); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* ================= match 语句(PEP 634) ================= */ + +static PyObject *ParsePattern(Parser *P); +static PyObject *ParsePatternOr(Parser *P); +static PyObject *ParsePatternAtom(Parser *P); +static PyObject *ParsePatternValue(Parser *P); +static PyObject *ParseMatchValueAttr(Parser *P); +static PyObject *ParseMatchClass(Parser *P); +static PyObject *ParsePatternClassName(Parser *P); +static PyObject *ParseMatchSequence(Parser *P); +static PyObject *ParseMatchMapping(Parser *P); + +static PyObject *ParseMatch(Parser *P) { + Token *Start = NextTok(P); /* match */ + PyObject *Subject = ParseExpr(P); + if (Subject == NULL) { + return NULL; + } + if (!ExpectOp(P, ":")) { + Py_DECREF(Subject); + return NULL; + } + if (!ExpectNEWLINE(P)) { + Py_DECREF(Subject); + return NULL; + } + Token *IndentTok = NextTok(P); + if (IndentTok->Kind != TK_INDENT) { + SyntaxErrorAt(P, IndentTok, "expected INDENT"); + Py_DECREF(Subject); + return NULL; + } + PyObject *Cases = NewList(); + if (Cases == NULL) { + Py_DECREF(Subject); + return NULL; + } + SkipNEWLINE(P); + while (AtName(P, "case")) { + Token *CaseStart = NextTok(P); + PyObject *Pattern = ParsePattern(P); + if (Pattern == NULL) { + Py_DECREF(Cases); + Py_DECREF(Subject); + return NULL; + } + PyObject *Guard = NULL; + if (AtName(P, "if")) { + NextTok(P); + Guard = ParseExpr(P); + if (Guard == NULL) { + Py_DECREF(Pattern); + Py_DECREF(Cases); + Py_DECREF(Subject); + return NULL; + } + } + if (!ExpectOp(P, ":")) { + Py_DECREF(Pattern); + Py_XDECREF(Guard); + Py_DECREF(Cases); + Py_DECREF(Subject); + return NULL; + } + PyObject *CaseBody = ParseSuite(P); + if (CaseBody == NULL) { + Py_DECREF(Pattern); + Py_XDECREF(Guard); + Py_DECREF(Cases); + Py_DECREF(Subject); + return NULL; + } + PyObject *Argv[3] = { Pattern, Guard ? Guard : Py_None, CaseBody }; + PyObject *Mc = NewNode(P, "match_case", Argv, 3); + Py_DECREF(Pattern); + Py_XDECREF(Guard); + Py_DECREF(CaseBody); + if (Mc == NULL) { + Py_DECREF(Cases); + Py_DECREF(Subject); + return NULL; + } + /* match_case 无位置属性 */ + (void)CaseStart; + PyList_Append(Cases, Mc); + Py_DECREF(Mc); + SkipNEWLINE(P); + } + Token *DedentTok = NextTok(P); + if (DedentTok->Kind != TK_DEDENT) { + SyntaxErrorAt(P, DedentTok, "expected DEDENT"); + Py_DECREF(Cases); + Py_DECREF(Subject); + return NULL; + } + PyObject *Argv[2] = { Subject, Cases }; + PyObject *Node = NewNode(P, "Match", Argv, 2); + Py_DECREF(Subject); + Py_DECREF(Cases); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* 模式(顶层,处理 | 或模式) */ +static PyObject *ParsePattern(Parser *P) { + Token *Start = PeekTok(P, 0); + PyObject *Patterns = NewList(); + if (Patterns == NULL) { + return NULL; + } + PyObject *First = ParsePatternOr(P); + if (First == NULL) { + Py_DECREF(Patterns); + return NULL; + } + PyList_Append(Patterns, First); + while (AtOp(P, "|")) { + NextTok(P); + PyObject *Pat = ParsePatternOr(P); + if (Pat == NULL) { + Py_DECREF(Patterns); + Py_DECREF(First); + return NULL; + } + PyList_Append(Patterns, Pat); + Py_DECREF(Pat); + } + if (PyList_GET_SIZE(Patterns) == 1) { + Py_DECREF(Patterns); + return First; + } + Py_DECREF(First); + PyObject *Argv[1] = { Patterns }; + PyObject *Node = NewNode(P, "MatchOr", Argv, 1); + Py_DECREF(Patterns); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* 单模式(含 as 绑定) */ +static PyObject *ParsePatternOr(Parser *P) { + Token *Start = PeekTok(P, 0); + PyObject *Pat = ParsePatternAtom(P); + if (Pat == NULL) { + return NULL; + } + if (AtName(P, "as")) { + NextTok(P); + Token *NameTok = ExpectName(P, NULL); + if (NameTok == NULL) { + Py_DECREF(Pat); + return NULL; + } + PyObject *Name = PyUnicode_FromStringAndSize(NameTok->Start, NameTok->End - NameTok->Start); + if (Name == NULL) { + Py_DECREF(Pat); + return NULL; + } + PyObject *Argv[2] = { Pat, Name }; + PyObject *Node = NewNode(P, "MatchAs", Argv, 2); + Py_DECREF(Pat); + Py_DECREF(Name); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + return Pat; +} + +/* 模式原子 */ +static PyObject *ParsePatternAtom(Parser *P) { + Token *Start = PeekTok(P, 0); + Token *T = PeekTok(P, 0); + if (T->Kind == TK_NAME) { + if (TokIsName(T, "True") || TokIsName(T, "False") || TokIsName(T, "None")) { + NextTok(P); + PyObject *Val = TokIsName(T, "True") ? Py_True + : (TokIsName(T, "False") ? Py_False : Py_None); + PyObject *Argv[1] = { Val }; + PyObject *Node = NewNode(P, "MatchSingleton", Argv, 1); + if (Node != NULL) { + FixLoc(P, Node, T); + } + return Node; + } + /* 类模式:Name(...) 或 a.b.c(...) —— 扫描属性链末尾判断 '(' */ + int PeekI = 1; + while (PeekTok(P, PeekI)->Kind == TK_OP && TokIsOp(PeekTok(P, PeekI), ".") + && PeekTok(P, PeekI + 1)->Kind == TK_NAME) { + PeekI += 2; + } + if (PeekTok(P, PeekI)->Kind == TK_OP && TokIsOp(PeekTok(P, PeekI), "(")) { + return ParseMatchClass(P); + } + /* 属性值模式 a.b(无括号) */ + if (PeekTok(P, 1)->Kind == TK_OP && TokIsOp(PeekTok(P, 1), ".")) { + return ParseMatchValueAttr(P); + } + /* 捕获模式(含 _ 通配符) */ + NextTok(P); + PyObject *Name = NULL; + if (!TokIsName(T, "_")) { + Name = PyUnicode_FromStringAndSize(T->Start, T->End - T->Start); + if (Name == NULL) { + return NULL; + } + } + PyObject *Argv[2] = { Py_None, Name ? Name : Py_None }; + PyObject *Node = NewNode(P, "MatchAs", Argv, 2); + Py_XDECREF(Name); + if (Node != NULL) { + FixLoc(P, Node, T); + } + return Node; + } + if (T->Kind == TK_OP && (TokIsOp(T, "[") || TokIsOp(T, "("))) { + return ParseMatchSequence(P); + } + if (T->Kind == TK_OP && TokIsOp(T, "{")) { + return ParseMatchMapping(P); + } + if (T->Kind == TK_NUMBER || T->Kind == TK_STRING || T->Kind == TK_FSTRING + || (T->Kind == TK_OP && (TokIsOp(T, "-") || TokIsOp(T, "+")))) { + PyObject *Value = ParsePatternValue(P); + if (Value == NULL) { + return NULL; + } + PyObject *Argv[1] = { Value }; + PyObject *Node = NewNode(P, "MatchValue", Argv, 1); + Py_DECREF(Value); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + if (T->Kind == TK_OP && TokIsOp(T, "*")) { + NextTok(P); + PyObject *Name = NULL; + if (PeekTok(P, 0)->Kind != TK_NEWLINE && !AtOp(P, ",")) { + Token *NameTok = ExpectName(P, NULL); + if (NameTok == NULL) { + return NULL; + } + Name = PyUnicode_FromStringAndSize(NameTok->Start, NameTok->End - NameTok->Start); + if (Name == NULL) { + return NULL; + } + } + PyObject *Argv[1] = { Name ? Name : Py_None }; + PyObject *Node = NewNode(P, "MatchStar", Argv, 1); + Py_XDECREF(Name); + if (Node != NULL) { + FixLoc(P, Node, T); + } + return Node; + } + SyntaxErrorAt(P, T, "unexpected pattern token"); + return NULL; +} + +/* 值模式表达式(带可选 +/- 前缀) */ +static PyObject *ParsePatternValue(Parser *P) { + Token *Start = PeekTok(P, 0); + Token *T = PeekTok(P, 0); + if (T->Kind == TK_OP && (TokIsOp(T, "-") || TokIsOp(T, "+"))) { + Token *OpTok = NextTok(P); + PyObject *Inner; + if (PeekTok(P, 0)->Kind == TK_NUMBER) { + Inner = ParseNumber(P); + } else if (PeekTok(P, 0)->Kind == TK_STRING || PeekTok(P, 0)->Kind == TK_FSTRING) { + Inner = ParseStringConcat(P); + } else { + SyntaxErrorAt(P, PeekTok(P, 0), "expected literal after sign in pattern"); + return NULL; + } + if (Inner == NULL) { + return NULL; + } + const char *OpName = TokIsOp(OpTok, "-") ? "USub" : "UAdd"; + PyObject *Op = MakeSingleton(P, OpName); + if (Op == NULL) { + Py_DECREF(Inner); + return NULL; + } + PyObject *Argv[2] = { Op, Inner }; + PyObject *Node = NewNode(P, "UnaryOp", Argv, 2); + Py_DECREF(Op); + Py_DECREF(Inner); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; + } + if (T->Kind == TK_NUMBER) { + return ParseNumber(P); + } + return ParseStringConcat(P); +} + +/* 值模式属性访问:a.b.c(无括号) */ +static PyObject *ParseMatchValueAttr(Parser *P) { + Token *Start = PeekTok(P, 0); + PyObject *Expr = ParseExpr(P); + if (Expr == NULL) { + return NULL; + } + PyObject *Argv[1] = { Expr }; + PyObject *Node = NewNode(P, "MatchValue", Argv, 1); + Py_DECREF(Expr); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* 类模式:Name(pos, kw=v, ...) */ +static PyObject *ParseMatchClass(Parser *P) { + Token *Start = PeekTok(P, 0); + PyObject *ClsName = ParsePatternClassName(P); + if (ClsName == NULL) { + return NULL; + } + if (!ExpectOp(P, "(")) { + Py_DECREF(ClsName); + return NULL; + } + SkipNL(P); + PyObject *Patterns = NewList(); + PyObject *KwdAttrs = NewList(); + PyObject *KwdPatterns = NewList(); + if (Patterns == NULL || KwdAttrs == NULL || KwdPatterns == NULL) { + Py_XDECREF(Patterns); + Py_XDECREF(KwdAttrs); + Py_XDECREF(KwdPatterns); + Py_DECREF(ClsName); + return NULL; + } + if (!AtOp(P, ")")) { + for (;;) { + SkipNL(P); + if (AtOp(P, "*")) { + NextTok(P); + PyObject *Pat = ParsePattern(P); + if (Pat == NULL) { + Py_DECREF(Patterns); + Py_DECREF(KwdAttrs); + Py_DECREF(KwdPatterns); + Py_DECREF(ClsName); + return NULL; + } + PyList_Append(Patterns, Pat); + Py_DECREF(Pat); + } else { + Token *A0 = PeekTok(P, 0); + Token *A1 = PeekTok(P, 1); + if (A0->Kind == TK_NAME && A1->Kind == TK_OP && TokIsOp(A1, "=")) { + Token *AttrTok = NextTok(P); + NextTok(P); /* '=' */ + PyObject *Attr = PyUnicode_FromStringAndSize(AttrTok->Start, + AttrTok->End - AttrTok->Start); + PyObject *Pat = ParsePattern(P); + if (Attr == NULL || Pat == NULL) { + Py_XDECREF(Attr); + Py_XDECREF(Pat); + Py_DECREF(Patterns); + Py_DECREF(KwdAttrs); + Py_DECREF(KwdPatterns); + Py_DECREF(ClsName); + return NULL; + } + PyList_Append(KwdAttrs, Attr); + PyList_Append(KwdPatterns, Pat); + Py_DECREF(Attr); + Py_DECREF(Pat); + } else { + PyObject *Pat = ParsePattern(P); + if (Pat == NULL) { + Py_DECREF(Patterns); + Py_DECREF(KwdAttrs); + Py_DECREF(KwdPatterns); + Py_DECREF(ClsName); + return NULL; + } + PyList_Append(Patterns, Pat); + Py_DECREF(Pat); + } + } + SkipNL(P); + if (AtOp(P, ",")) { + NextTok(P); + if (AtOp(P, ")")) { + break; + } + continue; + } + break; + } + } + if (!ExpectOp(P, ")")) { + Py_DECREF(Patterns); + Py_DECREF(KwdAttrs); + Py_DECREF(KwdPatterns); + Py_DECREF(ClsName); + return NULL; + } + PyObject *Argv[4] = { ClsName, Patterns, KwdAttrs, KwdPatterns }; + PyObject *Node = NewNode(P, "MatchClass", Argv, 4); + Py_DECREF(ClsName); + Py_DECREF(Patterns); + Py_DECREF(KwdAttrs); + Py_DECREF(KwdPatterns); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* 类模式的类名(Name / a.b.c 属性链) */ +static PyObject *ParsePatternClassName(Parser *P) { + Token *Start = PeekTok(P, 0); + Token *NameTok = ExpectName(P, NULL); + if (NameTok == NULL) { + return NULL; + } + PyObject *Node = MakeNameNode(P, NameTok, "Load"); + if (Node == NULL) { + return NULL; + } + while (AtOp(P, ".")) { + NextTok(P); + Token *AttrTok = ExpectName(P, NULL); + if (AttrTok == NULL) { + Py_DECREF(Node); + return NULL; + } + PyObject *AttrNode = MakeAttrNode(P, Node, AttrTok, Start); + Py_DECREF(Node); + if (AttrNode == NULL) { + return NULL; + } + Node = AttrNode; + } + return Node; +} + +/* 序列模式 [a, b] / (a, b) */ +static PyObject *ParseMatchSequence(Parser *P) { + Token *Start = NextTok(P); /* '[' 或 '(' */ + const char *Close = TokIsOp(Start, "[") ? "]" : ")"; + PyObject *Patterns = NewList(); + if (Patterns == NULL) { + return NULL; + } + SkipNL(P); + if (!AtOp(P, Close)) { + for (;;) { + SkipNL(P); + PyObject *Pat = ParsePattern(P); + if (Pat == NULL) { + Py_DECREF(Patterns); + return NULL; + } + PyList_Append(Patterns, Pat); + Py_DECREF(Pat); + SkipNL(P); + if (AtOp(P, ",")) { + NextTok(P); + if (AtOp(P, Close)) { + break; + } + continue; + } + break; + } + } + if (!ExpectOp(P, Close)) { + Py_DECREF(Patterns); + return NULL; + } + PyObject *Argv[1] = { Patterns }; + PyObject *Node = NewNode(P, "MatchSequence", Argv, 1); + Py_DECREF(Patterns); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + +/* 映射模式 {k: v, **rest} */ +static PyObject *ParseMatchMapping(Parser *P) { + Token *Start = NextTok(P); /* '{' */ + PyObject *Keys = NewList(); + PyObject *Patterns = NewList(); + PyObject *Rest = NULL; + if (Keys == NULL || Patterns == NULL) { + Py_XDECREF(Keys); + Py_XDECREF(Patterns); + return NULL; + } + SkipNL(P); + if (!AtOp(P, "}")) { + for (;;) { + SkipNL(P); + if (AtOp(P, "**")) { + NextTok(P); + Token *NameTok = ExpectName(P, NULL); + if (NameTok == NULL) { + Py_DECREF(Keys); + Py_DECREF(Patterns); + return NULL; + } + Rest = PyUnicode_FromStringAndSize(NameTok->Start, NameTok->End - NameTok->Start); + if (Rest == NULL) { + Py_DECREF(Keys); + Py_DECREF(Patterns); + return NULL; + } + } else { + PyObject *Key = ParseExpr(P); + if (Key == NULL) { + Py_DECREF(Keys); + Py_DECREF(Patterns); + Py_XDECREF(Rest); + return NULL; + } + if (!ExpectOp(P, ":")) { + Py_DECREF(Key); + Py_DECREF(Keys); + Py_DECREF(Patterns); + Py_XDECREF(Rest); + return NULL; + } + PyObject *Pat = ParsePattern(P); + if (Pat == NULL) { + Py_DECREF(Key); + Py_DECREF(Keys); + Py_DECREF(Patterns); + Py_XDECREF(Rest); + return NULL; + } + PyList_Append(Keys, Key); + PyList_Append(Patterns, Pat); + Py_DECREF(Key); + Py_DECREF(Pat); + } + SkipNL(P); + if (AtOp(P, ",")) { + NextTok(P); + if (AtOp(P, "}")) { + break; + } + continue; + } + break; + } + } + if (!ExpectOp(P, "}")) { + Py_DECREF(Keys); + Py_DECREF(Patterns); + Py_XDECREF(Rest); + return NULL; + } + PyObject *Argv[3] = { Keys, Patterns, Rest ? Rest : Py_None }; + PyObject *Node = NewNode(P, "MatchMapping", Argv, 3); + Py_DECREF(Keys); + Py_DECREF(Patterns); + Py_XDECREF(Rest); + if (Node != NULL) { + FixLoc(P, Node, Start); + } + return Node; +} + /* ================= 模块入口 ================= */ static PyObject *VastC_Parse(PyObject *Self, PyObject *Args, PyObject *Kwargs) { @@ -433,17 +6163,32 @@ static PyObject *VastC_Parse(PyObject *Self, PyObject *Args, PyObject *Kwargs) { if (!PyArg_ParseTupleAndKeywords(Args, Kwargs, "s#|s:parse", KwList, &Src, &Len, &Mode)) { return NULL; } + Token *Tokens = NULL; + int NTokens = 0; + int *LineStarts = NULL; + int NLineStarts = 0; + if (!TokenizeText(Src, Len, &Tokens, &NTokens, &LineStarts, &NLineStarts)) { + PyErr_SetString(PyExc_MemoryError, "vast_c: out of memory during tokenize"); + return NULL; + } Parser P; P.Src = Src; P.Len = Len; - P.Pos = 0; - P.Line = 1; - P.Col = 0; + P.Tokens = Tokens; + P.NTokens = NTokens; + P.Idx = 0; + P.LineStarts = LineStarts; + P.NLineStarts = NLineStarts; P.Error = 0; + P.ErrMsg = NULL; + P.ErrLine = 0; + P.FStringExprEnd = 0; PyObject *Tree = ParseModule(&P); + PyMem_Free(Tokens); + PyMem_Free(LineStarts); if (Tree == NULL) { if (P.Error) { - PyErr_SetString(PyExc_SyntaxError, "vast_c: 语法错误(阶段 0 仅支持赋值/名字/数字)"); + PyErr_SetString(PyExc_SyntaxError, P.ErrMsg ? P.ErrMsg : "vast_c: syntax error"); } return NULL; } @@ -451,12 +6196,12 @@ static PyObject *VastC_Parse(PyObject *Self, PyObject *Args, PyObject *Kwargs) { return Tree; } if (strcmp(Mode, "eval") == 0) { - /* 复用 vast.parse 的 eval 语义:Expose Expr 的 value 为 Expression 节点 */ + /* 复用 vast.parse 的 eval 语义:暴露 Expr 的 value 为 Expression 节点 */ PyObject *BodyAttr = PyObject_GetAttrString(Tree, "body"); Py_DECREF(Tree); if (BodyAttr == NULL || PyList_GET_SIZE(BodyAttr) != 1) { Py_XDECREF(BodyAttr); - PyErr_SetString(PyExc_SyntaxError, "vast_c: eval 模式需要单一表达式"); + PyErr_SetString(PyExc_SyntaxError, "vast_c: eval mode requires single expression"); return NULL; } PyObject *Stmt = PyList_GetItem(BodyAttr, 0); /* borrowed */ @@ -465,20 +6210,32 @@ static PyObject *VastC_Parse(PyObject *Self, PyObject *Args, PyObject *Kwargs) { if (Value == NULL) { return NULL; } - PyObject *Args2[1]; - Args2[0] = Value; - PyObject *ExprNode = NodeMake(&P, "Expression", Args2, 1); + PyObject *Argv2[1]; + Argv2[0] = Value; + PyObject *ExprNode = NewNode(&P, "Expression", Argv2, 1); Py_DECREF(Value); return ExprNode; } + if (strcmp(Mode, "single") == 0) { + PyObject *BodyAttr = PyObject_GetAttrString(Tree, "body"); + Py_DECREF(Tree); + if (BodyAttr == NULL) { + return NULL; + } + PyObject *Argv2[1]; + Argv2[0] = BodyAttr; + PyObject *IntNode = NewNode(&P, "Interactive", Argv2, 1); + Py_DECREF(BodyAttr); + return IntNode; + } Py_DECREF(Tree); - PyErr_SetString(PyExc_ValueError, "vast_c: 不支持的 mode"); + PyErr_SetString(PyExc_ValueError, "vast_c: unsupported mode"); return NULL; } static PyMethodDef VastC_Methods[] = { {"parse", (PyCFunction)VastC_Parse, METH_VARARGS | METH_KEYWORDS, - "parse(source, mode='exec') -> vast.nodes 树(阶段 0 子集)"}, + "parse(source, mode='exec') -> vast.nodes 树(完整 Python 3.12 + Viper 软关键字)"}, {NULL, NULL, 0, NULL}, };