diff --git a/includes/vipersimd.vp b/includes/vipersimd.vp index e567495..85f281d 100644 --- a/includes/vipersimd.vp +++ b/includes/vipersimd.vp @@ -184,15 +184,62 @@ movups [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], xmm0""", # ============================================================ def simd_add_array(a: t.CPtr, b: t.CPtr, out: t.CPtr, n: t.CSizeT) -> t.CVoid: - """AVX2 batch: out[i:i+4] = a[i:i+4] + b[i:i+4], i=0,4,8,...""" + """AVX2 batch: out[i:i+4] = a[i:i+4] + b[i:i+4], i=0,4,8,... + ×8 双缓冲(32 doubles/大迭代):a(ymm0-7)+b(ymm8-15) 16 路 load 全部在飞后 + 做 8 条独立 add→store 链,最大化内存级并行、压榨访存口。 + 尾巴(不足 8 个 ymm 的块)回退单向量循环,语义与原版一致(floor(n/4) 向量)。 + 注:ymm0-15 clobber 需 TPV c.Asm 支持(CallGenCBuiltin ASM_MAX_CLOBBERS=32)。""" c.Asm(f"""mov rcx, {c.AsmInp(n, t.ASM_DESCR.REG_ANY)} shr rcx, 2 mov r8, {c.AsmInp(a, t.ASM_DESCR.REG_ANY)} mov r9, {c.AsmInp(b, t.ASM_DESCR.REG_ANY)} mov r10, {c.AsmInp(out, t.ASM_DESCR.REG_ANY)} -test rcx, rcx -jz .Ladd_end -.Ladd_loop: +mov r11, rcx +shr r11, 3 +test r11, r11 +jz .La_add_tail +.La_add_big: +vmovupd ymm0, [r8] +vmovupd ymm1, [r8 + 32] +vmovupd ymm2, [r8 + 64] +vmovupd ymm3, [r8 + 96] +vmovupd ymm4, [r8 + 128] +vmovupd ymm5, [r8 + 160] +vmovupd ymm6, [r8 + 192] +vmovupd ymm7, [r8 + 224] +vmovupd ymm8, [r9] +vmovupd ymm9, [r9 + 32] +vmovupd ymm10, [r9 + 64] +vmovupd ymm11, [r9 + 96] +vmovupd ymm12, [r9 + 128] +vmovupd ymm13, [r9 + 160] +vmovupd ymm14, [r9 + 192] +vmovupd ymm15, [r9 + 224] +vaddpd ymm0, ymm0, ymm8 +vaddpd ymm1, ymm1, ymm9 +vaddpd ymm2, ymm2, ymm10 +vaddpd ymm3, ymm3, ymm11 +vaddpd ymm4, ymm4, ymm12 +vaddpd ymm5, ymm5, ymm13 +vaddpd ymm6, ymm6, ymm14 +vaddpd ymm7, ymm7, ymm15 +vmovupd [r10], ymm0 +vmovupd [r10 + 32], ymm1 +vmovupd [r10 + 64], ymm2 +vmovupd [r10 + 96], ymm3 +vmovupd [r10 + 128], ymm4 +vmovupd [r10 + 160], ymm5 +vmovupd [r10 + 192], ymm6 +vmovupd [r10 + 224], ymm7 +add r8, 256 +add r9, 256 +add r10, 256 +dec r11 +jnz .La_add_big +.La_add_tail: +and rcx, 7 +jz .La_add_end +.La_add_tloop: vmovupd ymm0, [r8] vmovupd ymm1, [r9] vaddpd ymm0, ymm0, ymm1 @@ -201,10 +248,12 @@ add r8, 32 add r9, 32 add r10, 32 dec rcx -jnz .Ladd_loop -.Ladd_end: +jnz .La_add_tloop +.La_add_end: vzeroupper""", - op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'rcx', 'r8', 'r9', 'r10']) + op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'ymm2', 'ymm3', + 'ymm4', 'ymm5', 'ymm6', 'ymm7', 'ymm8', 'ymm9', 'ymm10', 'ymm11', + 'ymm12', 'ymm13', 'ymm14', 'ymm15', 'rcx', 'r8', 'r9', 'r10', 'r11']) def simd_sub_array(a: t.CPtr, b: t.CPtr, out: t.CPtr, n: t.CSizeT) -> t.CVoid: @@ -232,15 +281,59 @@ vzeroupper""", def simd_mul_array(a: t.CPtr, b: t.CPtr, out: t.CPtr, n: t.CSizeT) -> t.CVoid: - """AVX2 batch: out[i:i+4] = a[i:i+4] * b[i:i+4]""" + """AVX2 batch: out[i:i+4] = a[i:i+4] * b[i:i+4] + ×8 双缓冲(同 simd_add_array)。""" c.Asm(f"""mov rcx, {c.AsmInp(n, t.ASM_DESCR.REG_ANY)} shr rcx, 2 mov r8, {c.AsmInp(a, t.ASM_DESCR.REG_ANY)} mov r9, {c.AsmInp(b, t.ASM_DESCR.REG_ANY)} mov r10, {c.AsmInp(out, t.ASM_DESCR.REG_ANY)} -test rcx, rcx -jz .Lmul_end -.Lmul_loop: +mov r11, rcx +shr r11, 3 +test r11, r11 +jz .Lm_mul_tail +.Lm_mul_big: +vmovupd ymm0, [r8] +vmovupd ymm1, [r8 + 32] +vmovupd ymm2, [r8 + 64] +vmovupd ymm3, [r8 + 96] +vmovupd ymm4, [r8 + 128] +vmovupd ymm5, [r8 + 160] +vmovupd ymm6, [r8 + 192] +vmovupd ymm7, [r8 + 224] +vmovupd ymm8, [r9] +vmovupd ymm9, [r9 + 32] +vmovupd ymm10, [r9 + 64] +vmovupd ymm11, [r9 + 96] +vmovupd ymm12, [r9 + 128] +vmovupd ymm13, [r9 + 160] +vmovupd ymm14, [r9 + 192] +vmovupd ymm15, [r9 + 224] +vmulpd ymm0, ymm0, ymm8 +vmulpd ymm1, ymm1, ymm9 +vmulpd ymm2, ymm2, ymm10 +vmulpd ymm3, ymm3, ymm11 +vmulpd ymm4, ymm4, ymm12 +vmulpd ymm5, ymm5, ymm13 +vmulpd ymm6, ymm6, ymm14 +vmulpd ymm7, ymm7, ymm15 +vmovupd [r10], ymm0 +vmovupd [r10 + 32], ymm1 +vmovupd [r10 + 64], ymm2 +vmovupd [r10 + 96], ymm3 +vmovupd [r10 + 128], ymm4 +vmovupd [r10 + 160], ymm5 +vmovupd [r10 + 192], ymm6 +vmovupd [r10 + 224], ymm7 +add r8, 256 +add r9, 256 +add r10, 256 +dec r11 +jnz .Lm_mul_big +.Lm_mul_tail: +and rcx, 7 +jz .Lm_mul_end +.Lm_mul_tloop: vmovupd ymm0, [r8] vmovupd ymm1, [r9] vmulpd ymm0, ymm0, ymm1 @@ -249,10 +342,12 @@ add r8, 32 add r9, 32 add r10, 32 dec rcx -jnz .Lmul_loop -.Lmul_end: +jnz .Lm_mul_tloop +.Lm_mul_end: vzeroupper""", - op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'rcx', 'r8', 'r9', 'r10']) + op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'ymm2', 'ymm3', + 'ymm4', 'ymm5', 'ymm6', 'ymm7', 'ymm8', 'ymm9', 'ymm10', 'ymm11', + 'ymm12', 'ymm13', 'ymm14', 'ymm15', 'rcx', 'r8', 'r9', 'r10', 'r11']) def simd_div_array(a: t.CPtr, b: t.CPtr, out: t.CPtr, n: t.CSizeT) -> t.CVoid: @@ -280,24 +375,48 @@ vzeroupper""", def simd_sqrt_array(a: t.CPtr, out: t.CPtr, n: t.CSizeT) -> t.CVoid: - """AVX2 batch: out[i:i+4] = sqrt(a[i:i+4])""" + """AVX2 batch: out[i:i+4] = sqrt(a[i:i+4]) + ×4 展开:4 路独立 vsqrtpd 链隐藏平方根长延迟(吞吐受限场景显著提速)。""" c.Asm(f"""mov rcx, {c.AsmInp(n, t.ASM_DESCR.REG_ANY)} shr rcx, 2 mov r8, {c.AsmInp(a, t.ASM_DESCR.REG_ANY)} mov r10, {c.AsmInp(out, t.ASM_DESCR.REG_ANY)} -test rcx, rcx -jz .Lsqrt_end -.Lsqrt_loop: +mov r11, rcx +shr r11, 2 +test r11, r11 +jz .Ls_sqrt_tail +.Ls_sqrt_big: +vmovupd ymm0, [r8] +vmovupd ymm1, [r8 + 32] +vmovupd ymm2, [r8 + 64] +vmovupd ymm3, [r8 + 96] +vsqrtpd ymm0, ymm0 +vsqrtpd ymm1, ymm1 +vsqrtpd ymm2, ymm2 +vsqrtpd ymm3, ymm3 +vmovupd [r10], ymm0 +vmovupd [r10 + 32], ymm1 +vmovupd [r10 + 64], ymm2 +vmovupd [r10 + 96], ymm3 +add r8, 128 +add r10, 128 +dec r11 +jnz .Ls_sqrt_big +.Ls_sqrt_tail: +and rcx, 3 +jz .Ls_sqrt_end +.Ls_sqrt_tloop: vmovupd ymm0, [r8] vsqrtpd ymm0, ymm0 vmovupd [r10], ymm0 add r8, 32 add r10, 32 dec rcx -jnz .Lsqrt_loop -.Lsqrt_end: +jnz .Ls_sqrt_tloop +.Ls_sqrt_end: vzeroupper""", - op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'rcx', 'r8', 'r10']) + op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'ymm2', 'ymm3', + 'rcx', 'r8', 'r10', 'r11']) def simd_abs_array(a: t.CPtr, out: t.CPtr, n: t.CSizeT) -> t.CVoid: