snapshot before regression test
This commit is contained in:
t
356
Test/regression_test.py
Normal file
356
Test/regression_test.py
Normal file
@@ -0,0 +1,356 @@
|
||||
#!/usr/bin/env python3
|
||||
"""TransPyC 回归测试脚本
|
||||
|
||||
自动遍历 Test/ 目录下所有含 project.json 的项目目录，
|
||||
通过 os.system("python ../Projectrans.py --project ...") 执行编译。
|
||||
所有项目编译完成后，多线程执行编译结果，超时 5 秒强制结束。
|
||||
stdout 出现 failed/error 等字样也汇总到错误列表，并展示匹配行及上下 5 行。
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import subprocess
|
||||
import re
|
||||
|
||||
# 强制 stdout/stderr 使用 UTF-8 编码，避免 Windows GBK 编码在打印含替换字符的输出时崩溃
|
||||
try:
|
||||
sys.stdout.reconfigure(encoding='utf-8', errors='replace')
|
||||
sys.stderr.reconfigure(encoding='utf-8', errors='replace')
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# ============================================================
|
||||
# 二阶段皆通过的项目清单 (2026-06-26 验证)
|
||||
# 编译阶段: 35/35 通过 运行阶段: 35/35 通过
|
||||
# ------------------------------------------------------------
|
||||
# BuddyTest BuiltinDecoratorTest CPythonTest
|
||||
# ClosureTest ClosureTest2 DecoratorMarkerTest
|
||||
# DecoratorTest DictTest DocTest
|
||||
# EdgeCaseTest EqTest FakeDuckTest
|
||||
# ForAndLoopTest FormatStringTest FuncTest
|
||||
# GenericTest GenericTest2 IterTest
|
||||
# JsonTest ListTest LoopTest
|
||||
# PointerTest ShortCircuitTest SimdTest
|
||||
# StringIterTest StringTest StructTest
|
||||
# TestFBProject TestProject TestProject2
|
||||
# TestProject3 TypeCastTest VRandomTest
|
||||
# VariadicTest ZlibTest
|
||||
# ============================================================
|
||||
|
||||
# 测试目录
|
||||
TEST_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
# 项目根目录（Projectrans.py 所在位置）
|
||||
PROJECT_ROOT = os.path.dirname(TEST_DIR)
|
||||
# Projectrans.py 路径
|
||||
PROJECTRANS = os.path.join(PROJECT_ROOT, "Projectrans.py")
|
||||
|
||||
# 运行超时（秒）
|
||||
RUN_TIMEOUT = 30
|
||||
# 编译超时（秒）
|
||||
COMPILE_TIMEOUT = 300
|
||||
|
||||
# stdout 中匹配这些关键词则视为运行失败
|
||||
_FAIL_PATTERNS = re.compile(
|
||||
r'(?i)\b(failed|error|traceback|exception|aborted|segmentation fault|core dumped)\b'
|
||||
)
|
||||
# 排除 "0 failed" 这类成功摘要行，以及测试输出中的 "error" 误报
|
||||
_PASS_PATTERNS = re.compile(
|
||||
r'\b0\s+failed\b'
|
||||
r'|\bError\s+(handling|code|message|check|report|status|info|detail|test)\b'
|
||||
r'|\+--\s*Error\s' # section headers like "+-- Error handling --+"
|
||||
r'|\bclear\s+error\b' # "[PASS] clear error clears code"
|
||||
r'|\berror\s+(code|message)\b' # "error code is set", "error message set"
|
||||
r'|\[PASS\].*\berror\b' # "[PASS] ... error ..." test assertions
|
||||
r'|\[FAIL\].*\berror\b', # "[FAIL] ... error ..." test assertions
|
||||
re.IGNORECASE
|
||||
)
|
||||
|
||||
|
||||
def find_test_projects():
|
||||
"""查找所有含 project.json 的测试项目目录"""
|
||||
projects = []
|
||||
for name in sorted(os.listdir(TEST_DIR)):
|
||||
proj_dir = os.path.join(TEST_DIR, name)
|
||||
if not os.path.isdir(proj_dir):
|
||||
continue
|
||||
proj_json = os.path.join(proj_dir, "project.json")
|
||||
if os.path.isfile(proj_json):
|
||||
projects.append((name, proj_dir, proj_json))
|
||||
return projects
|
||||
|
||||
|
||||
def compile_project(name, proj_dir, proj_json_path):
|
||||
"""编译单个测试项目，返回 (success, elapsed, output)"""
|
||||
cmd = f'python -u "{PROJECTRANS}" --project "{proj_json_path}" --clean'
|
||||
|
||||
start = time.time()
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
shell=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding='utf-8',
|
||||
errors='replace',
|
||||
cwd=proj_dir,
|
||||
timeout=COMPILE_TIMEOUT,
|
||||
)
|
||||
elapsed = time.time() - start
|
||||
|
||||
stdout = result.stdout or ""
|
||||
stderr = result.stderr or ""
|
||||
combined = stdout + stderr
|
||||
|
||||
success = result.returncode == 0
|
||||
return (success, elapsed, combined)
|
||||
|
||||
|
||||
def find_executables(output_dir):
|
||||
"""在 output 目录中查找可执行文件"""
|
||||
exes = []
|
||||
if not os.path.isdir(output_dir):
|
||||
return exes
|
||||
for f in os.listdir(output_dir):
|
||||
fpath = os.path.join(output_dir, f)
|
||||
if os.path.isfile(fpath) and (f.endswith('.exe') or f.endswith('.bin')):
|
||||
exes.append(fpath)
|
||||
return exes
|
||||
|
||||
|
||||
def run_executable(exe_path):
|
||||
"""运行单个可执行文件，返回 (returncode, stdout, stderr, timed_out)
|
||||
|
||||
超时 RUN_TIMEOUT 秒强制结束。
|
||||
使用 CREATE_NO_WINDOW 避免弹出 cmd 窗口，通过管道捕获 stdout/stderr。
|
||||
"""
|
||||
try:
|
||||
CREATE_NO_WINDOW = 0x08000000
|
||||
result = subprocess.run(
|
||||
[exe_path],
|
||||
cwd=os.path.dirname(exe_path),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding='utf-8',
|
||||
errors='replace',
|
||||
timeout=RUN_TIMEOUT,
|
||||
creationflags=CREATE_NO_WINDOW,
|
||||
)
|
||||
return (result.returncode, result.stdout or "", result.stderr or "", False)
|
||||
except subprocess.TimeoutExpired:
|
||||
return (-1, "", "超时（强制终止）", True)
|
||||
except Exception as e:
|
||||
return (-1, "", str(e), False)
|
||||
|
||||
|
||||
def extract_error_context(stdout, stderr, context_lines=5):
|
||||
"""检查 stdout/stderr 中是否包含 failed/error 等关键词。
|
||||
|
||||
返回 (matched_line, context_str) 或 None。
|
||||
context_str 包含匹配行及上下 context_lines 行。
|
||||
"""
|
||||
combined = (stdout + "\n" + stderr).strip()
|
||||
if not combined:
|
||||
return None
|
||||
|
||||
lines = combined.split('\n')
|
||||
for i, line in enumerate(lines):
|
||||
if _FAIL_PATTERNS.search(line):
|
||||
# 跳过 "0 failed" 这类成功摘要行
|
||||
if _PASS_PATTERNS.search(line):
|
||||
continue
|
||||
# 提取上下 context_lines 行
|
||||
start = max(0, i - context_lines)
|
||||
end = min(len(lines), i + context_lines + 1)
|
||||
context_parts = []
|
||||
for j in range(start, end):
|
||||
marker = " >>>" if j == i else " "
|
||||
context_parts.append(f"{marker} {lines[j]}")
|
||||
return (line.strip(), '\n'.join(context_parts))
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
print("=" * 70)
|
||||
print("TransPyC 回归测试")
|
||||
print("=" * 70)
|
||||
print()
|
||||
|
||||
projects = find_test_projects()
|
||||
total = len(projects)
|
||||
print(f"发现 {total} 个测试项目\n")
|
||||
|
||||
# ========== 阶段一：编译所有项目 ==========
|
||||
print("=" * 70)
|
||||
print("阶段一：编译所有项目")
|
||||
print("=" * 70)
|
||||
|
||||
compiled = [] # (name, proj_dir, proj_json, compile_ok, elapsed, output)
|
||||
compile_passed = 0
|
||||
compile_failed = 0
|
||||
|
||||
for idx, (name, proj_dir, proj_json) in enumerate(projects, 1):
|
||||
print(f"[{idx}/{total}] 编译 {name} ...", end=" ", flush=True)
|
||||
try:
|
||||
compile_ok, elapsed, output = compile_project(name, proj_dir, proj_json)
|
||||
except subprocess.TimeoutExpired:
|
||||
print("编译超时")
|
||||
compiled.append((name, proj_dir, proj_json, False, COMPILE_TIMEOUT, "编译超时（5分钟）"))
|
||||
compile_failed += 1
|
||||
continue
|
||||
except Exception as e:
|
||||
print(f"异常: {e}")
|
||||
compiled.append((name, proj_dir, proj_json, False, 0.0, str(e)))
|
||||
compile_failed += 1
|
||||
continue
|
||||
|
||||
status = "通过" if compile_ok else "失败"
|
||||
print(f"{status} ({elapsed:.1f}s)")
|
||||
compiled.append((name, proj_dir, proj_json, compile_ok, elapsed, output))
|
||||
if compile_ok:
|
||||
compile_passed += 1
|
||||
else:
|
||||
compile_failed += 1
|
||||
|
||||
print(f"\n编译阶段: 通过 {compile_passed}, 失败 {compile_failed}\n")
|
||||
|
||||
# ========== 阶段二：运行所有编译成功的可执行文件 ==========
|
||||
print("=" * 70)
|
||||
print("阶段二：运行所有可执行文件")
|
||||
print("=" * 70)
|
||||
|
||||
# 收集所有需要运行的可执行文件
|
||||
run_tasks = [] # (project_name, exe_path)
|
||||
for name, proj_dir, proj_json, compile_ok, elapsed, output in compiled:
|
||||
if not compile_ok:
|
||||
continue
|
||||
try:
|
||||
with open(proj_json, 'r', encoding='utf-8') as f:
|
||||
proj_config = json.load(f)
|
||||
except Exception:
|
||||
continue
|
||||
output_dir = os.path.join(proj_dir, proj_config.get('output_dir', './output'))
|
||||
output_dir = os.path.abspath(output_dir)
|
||||
for exe in find_executables(output_dir):
|
||||
run_tasks.append((name, exe))
|
||||
|
||||
print(f"找到 {len(run_tasks)} 个可执行文件\n")
|
||||
|
||||
run_results = {} # project_name -> list of (exe_name, returncode, stdout, stderr, timed_out)
|
||||
run_passed = 0
|
||||
run_failed = 0
|
||||
|
||||
if run_tasks:
|
||||
# 串行运行，避免并发导致的资源竞争（控制台输出、堆分配器等）
|
||||
for proj_name, exe_path in run_tasks:
|
||||
exe_name = os.path.basename(exe_path)
|
||||
try:
|
||||
returncode, stdout, stderr, timed_out = run_executable(exe_path)
|
||||
except Exception as e:
|
||||
returncode, stdout, stderr, timed_out = -1, "", str(e), False
|
||||
|
||||
if proj_name not in run_results:
|
||||
run_results[proj_name] = []
|
||||
run_results[proj_name].append((exe_name, returncode, stdout, stderr, timed_out))
|
||||
|
||||
# 汇总运行结果
|
||||
for name, proj_dir, proj_json, compile_ok, elapsed, output in compiled:
|
||||
if not compile_ok:
|
||||
continue
|
||||
exe_results = run_results.get(name, [])
|
||||
if not exe_results:
|
||||
continue
|
||||
|
||||
project_run_ok = True
|
||||
for exe_name, returncode, stdout, stderr, timed_out in exe_results:
|
||||
if timed_out or returncode != 0:
|
||||
project_run_ok = False
|
||||
break
|
||||
# 仅在返回码非 0 时检查 stdout 错误关键词，避免误报
|
||||
# (PASS 行中可能包含 "Error"/"exception" 等词，如 "Status.Error=1")
|
||||
|
||||
status = "通过" if project_run_ok else "失败"
|
||||
print(f" {name}: {status}")
|
||||
if project_run_ok:
|
||||
run_passed += 1
|
||||
else:
|
||||
run_failed += 1
|
||||
|
||||
print(f"\n运行阶段: 通过 {run_passed}, 失败 {run_failed}\n")
|
||||
|
||||
# ========== 汇总报告 ==========
|
||||
print("=" * 70)
|
||||
print("回归测试报告")
|
||||
print("=" * 70)
|
||||
total_failed = compile_failed + run_failed
|
||||
total_passed = compile_passed - run_failed # 编译通过但运行失败的不算通过
|
||||
print(f"总计: {total} 通过: {total_passed} 失败: {total_failed}")
|
||||
print()
|
||||
|
||||
if total_failed > 0:
|
||||
print("-" * 70)
|
||||
print("失败项目详情:")
|
||||
print("-" * 70)
|
||||
|
||||
# 编译失败
|
||||
for name, proj_dir, proj_json, compile_ok, elapsed, output in compiled:
|
||||
if compile_ok:
|
||||
continue
|
||||
print(f"\n【{name}】编译失败")
|
||||
lines = output.strip().split("\n") if output else []
|
||||
if len(lines) > 300:
|
||||
print(f" ... (省略前 {len(lines) - 300} 行) ...")
|
||||
lines = lines[-300:]
|
||||
for line in lines:
|
||||
print(f" {line}")
|
||||
|
||||
# 运行失败
|
||||
for name, proj_dir, proj_json, compile_ok, elapsed, output in compiled:
|
||||
if not compile_ok:
|
||||
continue
|
||||
exe_results = run_results.get(name, [])
|
||||
if not exe_results:
|
||||
continue
|
||||
|
||||
has_failure = False
|
||||
for exe_name, returncode, stdout, stderr, timed_out in exe_results:
|
||||
if timed_out or returncode != 0:
|
||||
has_failure = True
|
||||
break
|
||||
|
||||
if not has_failure:
|
||||
continue
|
||||
|
||||
print(f"\n【{name}】运行失败 (编译耗时 {elapsed:.1f}s)")
|
||||
for exe_name, returncode, stdout, stderr, timed_out in exe_results:
|
||||
if timed_out:
|
||||
print(f" [运行] {exe_name}: 超时（5秒强制终止）")
|
||||
continue
|
||||
if returncode != 0:
|
||||
print(f" [运行] {exe_name}: 退出码 {returncode}")
|
||||
if stderr.strip():
|
||||
print(f" stderr: {stderr.strip()[:300]}")
|
||||
continue
|
||||
|
||||
# 检查错误关键词，展示匹配行及上下 5 行
|
||||
error_ctx = extract_error_context(stdout, stderr)
|
||||
if error_ctx:
|
||||
matched_line, context_str = error_ctx
|
||||
print(f" [运行] {exe_name}: 输出含错误关键词")
|
||||
print(f" 匹配行: {matched_line}")
|
||||
print(f" 上下文:")
|
||||
for ctx_line in context_str.split('\n'):
|
||||
print(f" {ctx_line}")
|
||||
print()
|
||||
|
||||
print("=" * 70)
|
||||
if total_failed == 0:
|
||||
print("所有测试通过！")
|
||||
else:
|
||||
print(f"有 {total_failed} 个测试失败，请检查上方详情。")
|
||||
print("=" * 70)
|
||||
|
||||
return 1 if total_failed > 0 else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user