将 vipersimd.vp 的注解描述得更准确（t.CPtr -> double*）
This commit is contained in:
TermiNexus committed
1 parent
4fed54aa8b
commit
4c6877a87f
1 file changed
+211
-214
+211
-214
@@ -1,199 +1,196 @@
|
||||
import t, c
|
||||
|
||||
|
||||
# ============================================================
|
||||
# AVX2 double precision vector operations (4 x double, 256-bit YMM)
|
||||
# ============================================================
|
||||
|
||||
def simd_add4d(a: t.CPtr, b: t.CPtr, out: t.CPtr):
|
||||
def simd_add4d(a: double*, b: double*, out: double*):
|
||||
"""AVX2: out[0:4] = a[0:4] + b[0:4]"""
|
||||
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
||||
vmovupd ymm1, [{c.AsmInp(b, t.ASM_DESCR.REG_ANY)}]
|
||||
__asm__(f"""vmovupd ymm0, [{asminp(a, ASM_DESCR.REG_ANY)}]
|
||||
vmovupd ymm1, [{asminp(b, ASM_DESCR.REG_ANY)}]
|
||||
vaddpd ymm0, ymm0, ymm1
|
||||
vmovupd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], ymm0""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
||||
vmovupd [{asminp(out, ASM_DESCR.REG_ANY)}], ymm0""",
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
||||
|
||||
|
||||
def simd_sub4d(a: t.CPtr, b: t.CPtr, out: t.CPtr):
|
||||
def simd_sub4d(a: double*, b: double*, out: double*):
|
||||
"""AVX2: out[0:4] = a[0:4] - b[0:4]"""
|
||||
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
||||
vmovupd ymm1, [{c.AsmInp(b, t.ASM_DESCR.REG_ANY)}]
|
||||
__asm__(f"""vmovupd ymm0, [{asminp(a, ASM_DESCR.REG_ANY)}]
|
||||
vmovupd ymm1, [{asminp(b, ASM_DESCR.REG_ANY)}]
|
||||
vsubpd ymm0, ymm0, ymm1
|
||||
vmovupd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], ymm0""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
||||
vmovupd [{asminp(out, ASM_DESCR.REG_ANY)}], ymm0""",
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
||||
|
||||
|
||||
def simd_mul4d(a: t.CPtr, b: t.CPtr, out: t.CPtr):
|
||||
def simd_mul4d(a: double*, b: double*, out: double*):
|
||||
"""AVX2: out[0:4] = a[0:4] * b[0:4]"""
|
||||
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
||||
vmovupd ymm1, [{c.AsmInp(b, t.ASM_DESCR.REG_ANY)}]
|
||||
__asm__(f"""vmovupd ymm0, [{asminp(a, ASM_DESCR.REG_ANY)}]
|
||||
vmovupd ymm1, [{asminp(b, ASM_DESCR.REG_ANY)}]
|
||||
vmulpd ymm0, ymm0, ymm1
|
||||
vmovupd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], ymm0""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
||||
vmovupd [{asminp(out, ASM_DESCR.REG_ANY)}], ymm0""",
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
||||
|
||||
|
||||
def simd_div4d(a: t.CPtr, b: t.CPtr, out: t.CPtr):
|
||||
def simd_div4d(a: double*, b: double*, out: double*):
|
||||
"""AVX2: out[0:4] = a[0:4] / b[0:4]"""
|
||||
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
||||
vmovupd ymm1, [{c.AsmInp(b, t.ASM_DESCR.REG_ANY)}]
|
||||
__asm__(f"""vmovupd ymm0, [{asminp(a, ASM_DESCR.REG_ANY)}]
|
||||
vmovupd ymm1, [{asminp(b, ASM_DESCR.REG_ANY)}]
|
||||
vdivpd ymm0, ymm0, ymm1
|
||||
vmovupd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], ymm0""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
||||
vmovupd [{asminp(out, ASM_DESCR.REG_ANY)}], ymm0""",
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
||||
|
||||
|
||||
def simd_sqrt4d(a: t.CPtr, out: t.CPtr):
|
||||
def simd_sqrt4d(a: double*, out: double*):
|
||||
"""AVX2: out[0:4] = sqrt(a[0:4])"""
|
||||
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
||||
__asm__(f"""vmovupd ymm0, [{asminp(a, ASM_DESCR.REG_ANY)}]
|
||||
vsqrtpd ymm0, ymm0
|
||||
vmovupd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], ymm0""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0'])
|
||||
vmovupd [{asminp(out, ASM_DESCR.REG_ANY)}], ymm0""",
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'ymm0'])
|
||||
|
||||
|
||||
def simd_neg4d(a: t.CPtr, out: t.CPtr):
|
||||
def simd_neg4d(a: double*, out: double*):
|
||||
"""AVX2: out[0:4] = -a[0:4] (flip sign bit via XOR)"""
|
||||
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
||||
__asm__(f"""vmovupd ymm0, [{asminp(a, ASM_DESCR.REG_ANY)}]
|
||||
vpcmpeqd ymm1, ymm1, ymm1
|
||||
vpsllq ymm1, ymm1, 63
|
||||
vxorpd ymm0, ymm0, ymm1
|
||||
vmovupd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], ymm0""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
||||
vmovupd [{asminp(out, ASM_DESCR.REG_ANY)}], ymm0""",
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
||||
|
||||
|
||||
def simd_abs4d(a: t.CPtr, out: t.CPtr):
|
||||
def simd_abs4d(a: double*, out: double*):
|
||||
"""AVX2: out[0:4] = |a[0:4]| (clear sign bit via ANDN)"""
|
||||
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
||||
__asm__(f"""vmovupd ymm0, [{asminp(a, ASM_DESCR.REG_ANY)}]
|
||||
vpcmpeqd ymm1, ymm1, ymm1
|
||||
vpsllq ymm1, ymm1, 63
|
||||
vandnpd ymm0, ymm1, ymm0
|
||||
vmovupd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], ymm0""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
||||
vmovupd [{asminp(out, ASM_DESCR.REG_ANY)}], ymm0""",
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
||||
|
||||
|
||||
def simd_max4d(a: t.CPtr, b: t.CPtr, out: t.CPtr):
|
||||
def simd_max4d(a: double*, b: double*, out: double*):
|
||||
"""AVX2: out[0:4] = max(a[0:4], b[0:4])"""
|
||||
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
||||
vmovupd ymm1, [{c.AsmInp(b, t.ASM_DESCR.REG_ANY)}]
|
||||
__asm__(f"""vmovupd ymm0, [{asminp(a, ASM_DESCR.REG_ANY)}]
|
||||
vmovupd ymm1, [{asminp(b, ASM_DESCR.REG_ANY)}]
|
||||
vmaxpd ymm0, ymm0, ymm1
|
||||
vmovupd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], ymm0""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
||||
vmovupd [{asminp(out, ASM_DESCR.REG_ANY)}], ymm0""",
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
||||
|
||||
|
||||
def simd_min4d(a: t.CPtr, b: t.CPtr, out: t.CPtr):
|
||||
def simd_min4d(a: double*, b: double*, out: double*):
|
||||
"""AVX2: out[0:4] = min(a[0:4], b[0:4])"""
|
||||
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
||||
vmovupd ymm1, [{c.AsmInp(b, t.ASM_DESCR.REG_ANY)}]
|
||||
__asm__(f"""vmovupd ymm0, [{asminp(a, ASM_DESCR.REG_ANY)}]
|
||||
vmovupd ymm1, [{asminp(b, ASM_DESCR.REG_ANY)}]
|
||||
vminpd ymm0, ymm0, ymm1
|
||||
vmovupd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], ymm0""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
||||
vmovupd [{asminp(out, ASM_DESCR.REG_ANY)}], ymm0""",
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
||||
|
||||
|
||||
def simd_mul_scalar4d(a: t.CPtr, scalar: t.CPtr, out: t.CPtr):
|
||||
def simd_mul_scalar4d(a: double*, scalar: double*, out: double*):
|
||||
"""AVX2: out[0:4] = a[0:4] * scalar (broadcast)"""
|
||||
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
||||
vbroadcastsd ymm1, [{c.AsmInp(scalar, t.ASM_DESCR.REG_ANY)}]
|
||||
__asm__(f"""vmovupd ymm0, [{asminp(a, ASM_DESCR.REG_ANY)}]
|
||||
vbroadcastsd ymm1, [{asminp(scalar, ASM_DESCR.REG_ANY)}]
|
||||
vmulpd ymm0, ymm0, ymm1
|
||||
vmovupd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], ymm0""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
||||
vmovupd [{asminp(out, ASM_DESCR.REG_ANY)}], ymm0""",
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
||||
|
||||
|
||||
def simd_add_scalar4d(a: t.CPtr, scalar: t.CPtr, out: t.CPtr):
|
||||
def simd_add_scalar4d(a: double*, scalar: double*, out: double*):
|
||||
"""AVX2: out[0:4] = a[0:4] + scalar (broadcast)"""
|
||||
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
||||
vbroadcastsd ymm1, [{c.AsmInp(scalar, t.ASM_DESCR.REG_ANY)}]
|
||||
__asm__(f"""vmovupd ymm0, [{asminp(a, ASM_DESCR.REG_ANY)}]
|
||||
vbroadcastsd ymm1, [{asminp(scalar, ASM_DESCR.REG_ANY)}]
|
||||
vaddpd ymm0, ymm0, ymm1
|
||||
vmovupd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], ymm0""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
||||
vmovupd [{asminp(out, ASM_DESCR.REG_ANY)}], ymm0""",
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
||||
|
||||
|
||||
def simd_hsum4d(a: t.CPtr, out: t.CPtr):
|
||||
def simd_hsum4d(a: double*, out: double*):
|
||||
"""AVX2: out[0] = a[0]+a[1]+a[2]+a[3] (horizontal sum)"""
|
||||
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
||||
__asm__(f"""vmovupd ymm0, [{asminp(a, ASM_DESCR.REG_ANY)}]
|
||||
vextractf128 xmm1, ymm0, 1
|
||||
vaddpd xmm0, xmm0, xmm1
|
||||
vhaddpd xmm0, xmm0, xmm0
|
||||
vmovsd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], xmm0""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
||||
vmovsd [{asminp(out, ASM_DESCR.REG_ANY)}], xmm0""",
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
||||
|
||||
|
||||
def simd_dot4d(a: t.CPtr, b: t.CPtr, out: t.CPtr):
|
||||
def simd_dot4d(a: double*, b: double*, out: double*):
|
||||
"""AVX2: out[0] = dot(a[0:4], b[0:4])"""
|
||||
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
||||
vmovupd ymm1, [{c.AsmInp(b, t.ASM_DESCR.REG_ANY)}]
|
||||
__asm__(f"""vmovupd ymm0, [{asminp(a, ASM_DESCR.REG_ANY)}]
|
||||
vmovupd ymm1, [{asminp(b, ASM_DESCR.REG_ANY)}]
|
||||
vmulpd ymm0, ymm0, ymm1
|
||||
vextractf128 xmm1, ymm0, 1
|
||||
vaddpd xmm0, xmm0, xmm1
|
||||
vhaddpd xmm0, xmm0, xmm0
|
||||
vmovsd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], xmm0""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
||||
vmovsd [{asminp(out, ASM_DESCR.REG_ANY)}], xmm0""",
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
||||
|
||||
|
||||
def simd_fma4d(a: t.CPtr, b: t.CPtr, c_val: t.CPtr, out: t.CPtr):
|
||||
def simd_fma4d(a: double*, b: double*, c_val: double*, out: double*):
|
||||
"""AVX2: out[0:4] = a[0:4]*b[0:4] + c[0:4] (fused multiply-add)
|
||||
Uses vfmadd213pd: ymm0 = ymm1 * ymm0 + ymm2
|
||||
Load order: b->ymm0, a->ymm1, c->ymm2"""
|
||||
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(b, t.ASM_DESCR.REG_ANY)}]
|
||||
vmovupd ymm1, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
||||
vmovupd ymm2, [{c.AsmInp(c_val, t.ASM_DESCR.REG_ANY)}]
|
||||
__asm__(f"""vmovupd ymm0, [{asminp(b, ASM_DESCR.REG_ANY)}]
|
||||
vmovupd ymm1, [{asminp(a, ASM_DESCR.REG_ANY)}]
|
||||
vmovupd ymm2, [{asminp(c_val, ASM_DESCR.REG_ANY)}]
|
||||
vfmadd213pd ymm0, ymm1, ymm2
|
||||
vmovupd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], ymm0""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'ymm2'])
|
||||
vmovupd [{asminp(out, ASM_DESCR.REG_ANY)}], ymm0""",
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'ymm2'])
|
||||
|
||||
|
||||
# ============================================================
|
||||
# SSE2 single precision vector operations (4 x float, 128-bit XMM)
|
||||
# ============================================================
|
||||
|
||||
def simd_add4f(a: t.CPtr, b: t.CPtr, out: t.CPtr):
|
||||
def simd_add4f(a: double*, b: double*, out: double*):
|
||||
"""SSE2: out[0:4] = a[0:4] + b[0:4] (float)"""
|
||||
c.Asm(f"""movups xmm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
||||
movups xmm1, [{c.AsmInp(b, t.ASM_DESCR.REG_ANY)}]
|
||||
__asm__(f"""movups xmm0, [{asminp(a, ASM_DESCR.REG_ANY)}]
|
||||
movups xmm1, [{asminp(b, ASM_DESCR.REG_ANY)}]
|
||||
addps xmm0, xmm1
|
||||
movups [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], xmm0""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'xmm0', 'xmm1'])
|
||||
movups [{asminp(out, ASM_DESCR.REG_ANY)}], xmm0""",
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'xmm0', 'xmm1'])
|
||||
|
||||
|
||||
def simd_sub4f(a: t.CPtr, b: t.CPtr, out: t.CPtr):
|
||||
def simd_sub4f(a: double*, b: double*, out: double*):
|
||||
"""SSE2: out[0:4] = a[0:4] - b[0:4] (float)"""
|
||||
c.Asm(f"""movups xmm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
||||
movups xmm1, [{c.AsmInp(b, t.ASM_DESCR.REG_ANY)}]
|
||||
__asm__(f"""movups xmm0, [{asminp(a, ASM_DESCR.REG_ANY)}]
|
||||
movups xmm1, [{asminp(b, ASM_DESCR.REG_ANY)}]
|
||||
subps xmm0, xmm1
|
||||
movups [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], xmm0""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'xmm0', 'xmm1'])
|
||||
movups [{asminp(out, ASM_DESCR.REG_ANY)}], xmm0""",
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'xmm0', 'xmm1'])
|
||||
|
||||
|
||||
def simd_mul4f(a: t.CPtr, b: t.CPtr, out: t.CPtr):
|
||||
def simd_mul4f(a: double*, b: double*, out: double*):
|
||||
"""SSE2: out[0:4] = a[0:4] * b[0:4] (float)"""
|
||||
c.Asm(f"""movups xmm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
||||
movups xmm1, [{c.AsmInp(b, t.ASM_DESCR.REG_ANY)}]
|
||||
__asm__(f"""movups xmm0, [{asminp(a, ASM_DESCR.REG_ANY)}]
|
||||
movups xmm1, [{asminp(b, ASM_DESCR.REG_ANY)}]
|
||||
mulps xmm0, xmm1
|
||||
movups [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], xmm0""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'xmm0', 'xmm1'])
|
||||
movups [{asminp(out, ASM_DESCR.REG_ANY)}], xmm0""",
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'xmm0', 'xmm1'])
|
||||
|
||||
|
||||
def simd_div4f(a: t.CPtr, b: t.CPtr, out: t.CPtr):
|
||||
def simd_div4f(a: double*, b: double*, out: double*):
|
||||
"""SSE2: out[0:4] = a[0:4] / b[0:4] (float)"""
|
||||
c.Asm(f"""movups xmm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
||||
movups xmm1, [{c.AsmInp(b, t.ASM_DESCR.REG_ANY)}]
|
||||
__asm__(f"""movups xmm0, [{asminp(a, ASM_DESCR.REG_ANY)}]
|
||||
movups xmm1, [{asminp(b, ASM_DESCR.REG_ANY)}]
|
||||
divps xmm0, xmm1
|
||||
movups [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], xmm0""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'xmm0', 'xmm1'])
|
||||
movups [{asminp(out, ASM_DESCR.REG_ANY)}], xmm0""",
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'xmm0', 'xmm1'])
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Batch array operations (AVX2 loop, process N double values)
|
||||
# ============================================================
|
||||
|
||||
def simd_add_array(a: t.CPtr, b: t.CPtr, out: t.CPtr, n: sizet):
|
||||
def simd_add_array(a: double*, b: double*, out: double*, n: sizet):
|
||||
"""AVX2 batch: out[i:i+4] = a[i:i+4] + b[i:i+4], i=0,4,8,...
|
||||
×8 双缓冲（32 doubles/大迭代）：a(ymm0-7)+b(ymm8-15) 16 路 load 全部在飞后
|
||||
做 8 条独立 add→store 链，最大化内存级并行、压榨访存口。
|
||||
尾巴（不足 8 个 ymm 的块）回退单向量循环，语义与原版一致（floor(n/4) 向量）。
|
||||
注：ymm0-15 clobber 需 TPV c.Asm 支持（CallGenCBuiltin ASM_MAX_CLOBBERS=32）。"""
|
||||
c.Asm(f"""mov rcx, {c.AsmInp(n, t.ASM_DESCR.REG_ANY)}
|
||||
注：ymm0-15 clobber 需 TPV __asm__ 支持（CallGenCBuiltin ASM_MAX_CLOBBERS=32）。"""
|
||||
__asm__(f"""mov rcx, {asminp(n, ASM_DESCR.REG_ANY)}
|
||||
shr rcx, 2
|
||||
mov r8, {c.AsmInp(a, t.ASM_DESCR.REG_ANY)}
|
||||
mov r9, {c.AsmInp(b, t.ASM_DESCR.REG_ANY)}
|
||||
mov r10, {c.AsmInp(out, t.ASM_DESCR.REG_ANY)}
|
||||
mov r8, {asminp(a, ASM_DESCR.REG_ANY)}
|
||||
mov r9, {asminp(b, ASM_DESCR.REG_ANY)}
|
||||
mov r10, {asminp(out, ASM_DESCR.REG_ANY)}
|
||||
mov r11, rcx
|
||||
shr r11, 3
|
||||
test r11, r11
|
||||
@@ -251,18 +248,18 @@ dec rcx
|
||||
jnz .La_add_tloop
|
||||
.La_add_end:
|
||||
vzeroupper""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'ymm2', 'ymm3',
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'ymm2', 'ymm3',
|
||||
'ymm4', 'ymm5', 'ymm6', 'ymm7', 'ymm8', 'ymm9', 'ymm10', 'ymm11',
|
||||
'ymm12', 'ymm13', 'ymm14', 'ymm15', 'rcx', 'r8', 'r9', 'r10', 'r11'])
|
||||
|
||||
|
||||
def simd_sub_array(a: t.CPtr, b: t.CPtr, out: t.CPtr, n: sizet):
|
||||
def simd_sub_array(a: double*, b: double*, out: double*, n: sizet):
|
||||
"""AVX2 batch: out[i:i+4] = a[i:i+4] - b[i:i+4]"""
|
||||
c.Asm(f"""mov rcx, {c.AsmInp(n, t.ASM_DESCR.REG_ANY)}
|
||||
__asm__(f"""mov rcx, {asminp(n, ASM_DESCR.REG_ANY)}
|
||||
shr rcx, 2
|
||||
mov r8, {c.AsmInp(a, t.ASM_DESCR.REG_ANY)}
|
||||
mov r9, {c.AsmInp(b, t.ASM_DESCR.REG_ANY)}
|
||||
mov r10, {c.AsmInp(out, t.ASM_DESCR.REG_ANY)}
|
||||
mov r8, {asminp(a, ASM_DESCR.REG_ANY)}
|
||||
mov r9, {asminp(b, ASM_DESCR.REG_ANY)}
|
||||
mov r10, {asminp(out, ASM_DESCR.REG_ANY)}
|
||||
test rcx, rcx
|
||||
jz .Lsub_end
|
||||
.Lsub_loop:
|
||||
@@ -277,17 +274,17 @@ dec rcx
|
||||
jnz .Lsub_loop
|
||||
.Lsub_end:
|
||||
vzeroupper""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'rcx', 'r8', 'r9', 'r10'])
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'rcx', 'r8', 'r9', 'r10'])
|
||||
|
||||
|
||||
def simd_mul_array(a: t.CPtr, b: t.CPtr, out: t.CPtr, n: sizet):
|
||||
def simd_mul_array(a: double*, b: double*, out: double*, n: sizet):
|
||||
"""AVX2 batch: out[i:i+4] = a[i:i+4] * b[i:i+4]
|
||||
×8 双缓冲（同 simd_add_array）。"""
|
||||
c.Asm(f"""mov rcx, {c.AsmInp(n, t.ASM_DESCR.REG_ANY)}
|
||||
__asm__(f"""mov rcx, {asminp(n, ASM_DESCR.REG_ANY)}
|
||||
shr rcx, 2
|
||||
mov r8, {c.AsmInp(a, t.ASM_DESCR.REG_ANY)}
|
||||
mov r9, {c.AsmInp(b, t.ASM_DESCR.REG_ANY)}
|
||||
mov r10, {c.AsmInp(out, t.ASM_DESCR.REG_ANY)}
|
||||
mov r8, {asminp(a, ASM_DESCR.REG_ANY)}
|
||||
mov r9, {asminp(b, ASM_DESCR.REG_ANY)}
|
||||
mov r10, {asminp(out, ASM_DESCR.REG_ANY)}
|
||||
mov r11, rcx
|
||||
shr r11, 3
|
||||
test r11, r11
|
||||
@@ -345,12 +342,12 @@ dec rcx
|
||||
jnz .Lm_mul_tloop
|
||||
.Lm_mul_end:
|
||||
vzeroupper""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'ymm2', 'ymm3',
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'ymm2', 'ymm3',
|
||||
'ymm4', 'ymm5', 'ymm6', 'ymm7', 'ymm8', 'ymm9', 'ymm10', 'ymm11',
|
||||
'ymm12', 'ymm13', 'ymm14', 'ymm15', 'rcx', 'r8', 'r9', 'r10', 'r11'])
|
||||
|
||||
|
||||
def simd_fma_array(a: t.CPtr, b: t.CPtr, c_val: t.CPtr, out: t.CPtr, n: sizet):
|
||||
def simd_fma_array(a: double*, b: double*, c_val: double*, out: double*, n: sizet):
|
||||
"""AVX2 batch (单趟融合乘加): out[i:i+4] = a[i:i+4]*b[i:i+4] + c_val[i:i+4]
|
||||
×8 双缓冲（同 simd_mul_array）：a(ymm0-7)+b(ymm8-15) 16 路 load 在飞后，
|
||||
用 8 条 vfmadd213pd 以 c_val 内存操作数为加数一次完成 FMA 并落 out，
|
||||
@@ -358,12 +355,12 @@ def simd_fma_array(a: t.CPtr, b: t.CPtr, c_val: t.CPtr, out: t.CPtr, n: sizet):
|
||||
语义：vfmadd213pd ymmX, ymmY, mem => ymmX = ymmY*ymmX + mem
|
||||
ymm8..15 = a(big lane)*b(big lane) + c(big lane)，单次舍入。
|
||||
尾巴（不足 8 个 ymm 的块）回退单向量循环。n 须为 4 的倍数（floor(n/4) 向量）。"""
|
||||
c.Asm(f"""mov rcx, {c.AsmInp(n, t.ASM_DESCR.REG_ANY)}
|
||||
__asm__(f"""mov rcx, {asminp(n, ASM_DESCR.REG_ANY)}
|
||||
shr rcx, 2
|
||||
mov r8, {c.AsmInp(a, t.ASM_DESCR.REG_ANY)}
|
||||
mov r9, {c.AsmInp(b, t.ASM_DESCR.REG_ANY)}
|
||||
mov rdx, {c.AsmInp(c_val, t.ASM_DESCR.REG_ANY)}
|
||||
mov r10, {c.AsmInp(out, t.ASM_DESCR.REG_ANY)}
|
||||
mov r8, {asminp(a, ASM_DESCR.REG_ANY)}
|
||||
mov r9, {asminp(b, ASM_DESCR.REG_ANY)}
|
||||
mov rdx, {asminp(c_val, ASM_DESCR.REG_ANY)}
|
||||
mov r10, {asminp(out, ASM_DESCR.REG_ANY)}
|
||||
mov r11, rcx
|
||||
shr r11, 3
|
||||
test r11, r11
|
||||
@@ -423,12 +420,12 @@ dec rcx
|
||||
jnz .Lfma_tloop
|
||||
.Lfma_end:
|
||||
vzeroupper""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'ymm2', 'ymm3',
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'ymm2', 'ymm3',
|
||||
'ymm4', 'ymm5', 'ymm6', 'ymm7', 'ymm8', 'ymm9', 'ymm10', 'ymm11',
|
||||
'ymm12', 'ymm13', 'ymm14', 'ymm15', 'rcx', 'rdx', 'r8', 'r9', 'r10', 'r11'])
|
||||
|
||||
|
||||
def scalar_fma_array(a: t.CPtr, b: t.CPtr, c_val: t.CPtr, out: t.CPtr, n: sizet):
|
||||
def scalar_fma_array(a: double*, b: double*, c_val: double*, out: double*, n: sizet):
|
||||
"""标量(逐元素)融合乘加: out[i] = a[i]*b[i] + c_val[i], 一次只算 1 个 double。
|
||||
|
||||
与 simd_fma_array 是同一数学内核的"未向量化"对照实现:
|
||||
@@ -438,11 +435,11 @@ def scalar_fma_array(a: t.CPtr, b: t.CPtr, c_val: t.CPtr, out: t.CPtr, n: sizet)
|
||||
* 每次循环 3 读 1 写、8 字节粒度, 相比 ×4 的 ymm 少 4 倍数据并行度,
|
||||
用来量化"向量化本身"带来的收益(SIMD 加速比)。
|
||||
n 须 >= 1; n == 0 直接跳过。"""
|
||||
c.Asm(f"""mov rcx, {c.AsmInp(n, t.ASM_DESCR.REG_ANY)}
|
||||
mov r8, {c.AsmInp(a, t.ASM_DESCR.REG_ANY)}
|
||||
mov r9, {c.AsmInp(b, t.ASM_DESCR.REG_ANY)}
|
||||
mov rdx, {c.AsmInp(c_val, t.ASM_DESCR.REG_ANY)}
|
||||
mov r10, {c.AsmInp(out, t.ASM_DESCR.REG_ANY)}
|
||||
__asm__(f"""mov rcx, {asminp(n, ASM_DESCR.REG_ANY)}
|
||||
mov r8, {asminp(a, ASM_DESCR.REG_ANY)}
|
||||
mov r9, {asminp(b, ASM_DESCR.REG_ANY)}
|
||||
mov rdx, {asminp(c_val, ASM_DESCR.REG_ANY)}
|
||||
mov r10, {asminp(out, ASM_DESCR.REG_ANY)}
|
||||
test rcx, rcx
|
||||
jz .Lsca_end
|
||||
.Lsca_loop:
|
||||
@@ -457,19 +454,19 @@ add r10, 8
|
||||
dec rcx
|
||||
jnz .Lsca_loop
|
||||
.Lsca_end:""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'xmm0', 'xmm1',
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'xmm0', 'xmm1',
|
||||
'rcx', 'rdx', 'r8', 'r9', 'r10'])
|
||||
|
||||
|
||||
def scalar_add_array(a: t.CPtr, b: t.CPtr, out: t.CPtr, n: sizet):
|
||||
def scalar_add_array(a: double*, b: double*, out: double*, n: sizet):
|
||||
"""标量(逐元素)加法: out[i] = a[i] + b[i], 一次 1 个 double。
|
||||
|
||||
`simd_add_array` 的未向量化对照: 标量 `vaddsd` vs SIMD `vaddpd`,
|
||||
同为 IEEE 加法 -> 结果逐位一致。"""
|
||||
c.Asm(f"""mov rcx, {c.AsmInp(n, t.ASM_DESCR.REG_ANY)}
|
||||
mov r8, {c.AsmInp(a, t.ASM_DESCR.REG_ANY)}
|
||||
mov r9, {c.AsmInp(b, t.ASM_DESCR.REG_ANY)}
|
||||
mov r10, {c.AsmInp(out, t.ASM_DESCR.REG_ANY)}
|
||||
__asm__(f"""mov rcx, {asminp(n, ASM_DESCR.REG_ANY)}
|
||||
mov r8, {asminp(a, ASM_DESCR.REG_ANY)}
|
||||
mov r9, {asminp(b, ASM_DESCR.REG_ANY)}
|
||||
mov r10, {asminp(out, ASM_DESCR.REG_ANY)}
|
||||
test rcx, rcx
|
||||
jz .Lsca_add_end
|
||||
.Lsca_add_loop:
|
||||
@@ -482,18 +479,18 @@ add r10, 8
|
||||
dec rcx
|
||||
jnz .Lsca_add_loop
|
||||
.Lsca_add_end:""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'xmm0', 'rcx', 'r8', 'r9', 'r10'])
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'xmm0', 'rcx', 'r8', 'r9', 'r10'])
|
||||
|
||||
|
||||
def scalar_mul_array(a: t.CPtr, b: t.CPtr, out: t.CPtr, n: sizet):
|
||||
def scalar_mul_array(a: double*, b: double*, out: double*, n: sizet):
|
||||
"""标量(逐元素)乘法: out[i] = a[i] * b[i], 一次 1 个 double。
|
||||
|
||||
`simd_mul_array` 的未向量化对照: 标量 `vmulsd` vs SIMD `vmulpd`,
|
||||
同为 IEEE 乘法 -> 结果逐位一致。"""
|
||||
c.Asm(f"""mov rcx, {c.AsmInp(n, t.ASM_DESCR.REG_ANY)}
|
||||
mov r8, {c.AsmInp(a, t.ASM_DESCR.REG_ANY)}
|
||||
mov r9, {c.AsmInp(b, t.ASM_DESCR.REG_ANY)}
|
||||
mov r10, {c.AsmInp(out, t.ASM_DESCR.REG_ANY)}
|
||||
__asm__(f"""mov rcx, {asminp(n, ASM_DESCR.REG_ANY)}
|
||||
mov r8, {asminp(a, ASM_DESCR.REG_ANY)}
|
||||
mov r9, {asminp(b, ASM_DESCR.REG_ANY)}
|
||||
mov r10, {asminp(out, ASM_DESCR.REG_ANY)}
|
||||
test rcx, rcx
|
||||
jz .Lsca_mul_end
|
||||
.Lsca_mul_loop:
|
||||
@@ -506,17 +503,17 @@ add r10, 8
|
||||
dec rcx
|
||||
jnz .Lsca_mul_loop
|
||||
.Lsca_mul_end:""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'xmm0', 'rcx', 'r8', 'r9', 'r10'])
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'xmm0', 'rcx', 'r8', 'r9', 'r10'])
|
||||
|
||||
|
||||
def scalar_sqrt_array(a: t.CPtr, out: t.CPtr, n: sizet):
|
||||
def scalar_sqrt_array(a: double*, out: double*, n: sizet):
|
||||
"""标量(逐元素)平方根: out[i] = sqrt(a[i]), 一次 1 个 double。
|
||||
|
||||
`simd_sqrt_array` 的未向量化对照: 标量 `vsqrtsd` vs SIMD `vsqrtpd`,
|
||||
两者都是 IEEE 正确舍入的平方根 -> 结果逐位一致。"""
|
||||
c.Asm(f"""mov rcx, {c.AsmInp(n, t.ASM_DESCR.REG_ANY)}
|
||||
mov r8, {c.AsmInp(a, t.ASM_DESCR.REG_ANY)}
|
||||
mov r10, {c.AsmInp(out, t.ASM_DESCR.REG_ANY)}
|
||||
__asm__(f"""mov rcx, {asminp(n, ASM_DESCR.REG_ANY)}
|
||||
mov r8, {asminp(a, ASM_DESCR.REG_ANY)}
|
||||
mov r10, {asminp(out, ASM_DESCR.REG_ANY)}
|
||||
test rcx, rcx
|
||||
jz .Lsca_sqrt_end
|
||||
.Lsca_sqrt_loop:
|
||||
@@ -528,16 +525,16 @@ add r10, 8
|
||||
dec rcx
|
||||
jnz .Lsca_sqrt_loop
|
||||
.Lsca_sqrt_end:""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'xmm0', 'rcx', 'r8', 'r10'])
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'xmm0', 'rcx', 'r8', 'r10'])
|
||||
|
||||
|
||||
def simd_div_array(a: t.CPtr, b: t.CPtr, out: t.CPtr, n: sizet):
|
||||
def simd_div_array(a: double*, b: double*, out: double*, n: sizet):
|
||||
"""AVX2 batch: out[i:i+4] = a[i:i+4] / b[i:i+4]"""
|
||||
c.Asm(f"""mov rcx, {c.AsmInp(n, t.ASM_DESCR.REG_ANY)}
|
||||
__asm__(f"""mov rcx, {asminp(n, ASM_DESCR.REG_ANY)}
|
||||
shr rcx, 2
|
||||
mov r8, {c.AsmInp(a, t.ASM_DESCR.REG_ANY)}
|
||||
mov r9, {c.AsmInp(b, t.ASM_DESCR.REG_ANY)}
|
||||
mov r10, {c.AsmInp(out, t.ASM_DESCR.REG_ANY)}
|
||||
mov r8, {asminp(a, ASM_DESCR.REG_ANY)}
|
||||
mov r9, {asminp(b, ASM_DESCR.REG_ANY)}
|
||||
mov r10, {asminp(out, ASM_DESCR.REG_ANY)}
|
||||
test rcx, rcx
|
||||
jz .Ldiv_end
|
||||
.Ldiv_loop:
|
||||
@@ -552,16 +549,16 @@ dec rcx
|
||||
jnz .Ldiv_loop
|
||||
.Ldiv_end:
|
||||
vzeroupper""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'rcx', 'r8', 'r9', 'r10'])
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'rcx', 'r8', 'r9', 'r10'])
|
||||
|
||||
|
||||
def simd_sqrt_array(a: t.CPtr, out: t.CPtr, n: sizet):
|
||||
def simd_sqrt_array(a: double*, out: double*, n: sizet):
|
||||
"""AVX2 batch: out[i:i+4] = sqrt(a[i:i+4])
|
||||
×4 展开：4 路独立 vsqrtpd 链隐藏平方根长延迟（吞吐受限场景显著提速）。"""
|
||||
c.Asm(f"""mov rcx, {c.AsmInp(n, t.ASM_DESCR.REG_ANY)}
|
||||
__asm__(f"""mov rcx, {asminp(n, ASM_DESCR.REG_ANY)}
|
||||
shr rcx, 2
|
||||
mov r8, {c.AsmInp(a, t.ASM_DESCR.REG_ANY)}
|
||||
mov r10, {c.AsmInp(out, t.ASM_DESCR.REG_ANY)}
|
||||
mov r8, {asminp(a, ASM_DESCR.REG_ANY)}
|
||||
mov r10, {asminp(out, ASM_DESCR.REG_ANY)}
|
||||
mov r11, rcx
|
||||
shr r11, 2
|
||||
test r11, r11
|
||||
@@ -596,16 +593,16 @@ dec rcx
|
||||
jnz .Ls_sqrt_tloop
|
||||
.Ls_sqrt_end:
|
||||
vzeroupper""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'ymm2', 'ymm3',
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'ymm2', 'ymm3',
|
||||
'rcx', 'r8', 'r10', 'r11'])
|
||||
|
||||
|
||||
def simd_abs_array(a: t.CPtr, out: t.CPtr, n: sizet):
|
||||
def simd_abs_array(a: double*, out: double*, n: sizet):
|
||||
"""AVX2 batch: out[i:i+4] = |a[i:i+4]|"""
|
||||
c.Asm(f"""mov rcx, {c.AsmInp(n, t.ASM_DESCR.REG_ANY)}
|
||||
__asm__(f"""mov rcx, {asminp(n, ASM_DESCR.REG_ANY)}
|
||||
shr rcx, 2
|
||||
mov r8, {c.AsmInp(a, t.ASM_DESCR.REG_ANY)}
|
||||
mov r10, {c.AsmInp(out, t.ASM_DESCR.REG_ANY)}
|
||||
mov r8, {asminp(a, ASM_DESCR.REG_ANY)}
|
||||
mov r10, {asminp(out, ASM_DESCR.REG_ANY)}
|
||||
vpcmpeqd ymm2, ymm2, ymm2
|
||||
vpsllq ymm2, ymm2, 63
|
||||
test rcx, rcx
|
||||
@@ -620,15 +617,15 @@ dec rcx
|
||||
jnz .Labs_loop
|
||||
.Labs_end:
|
||||
vzeroupper""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm2', 'rcx', 'r8', 'r10'])
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm2', 'rcx', 'r8', 'r10'])
|
||||
|
||||
|
||||
def simd_neg_array(a: t.CPtr, out: t.CPtr, n: sizet):
|
||||
def simd_neg_array(a: double*, out: double*, n: sizet):
|
||||
"""AVX2 batch: out[i:i+4] = -a[i:i+4] (flip sign bit via XOR)"""
|
||||
c.Asm(f"""mov rcx, {c.AsmInp(n, t.ASM_DESCR.REG_ANY)}
|
||||
__asm__(f"""mov rcx, {asminp(n, ASM_DESCR.REG_ANY)}
|
||||
shr rcx, 2
|
||||
mov r8, {c.AsmInp(a, t.ASM_DESCR.REG_ANY)}
|
||||
mov r10, {c.AsmInp(out, t.ASM_DESCR.REG_ANY)}
|
||||
mov r8, {asminp(a, ASM_DESCR.REG_ANY)}
|
||||
mov r10, {asminp(out, ASM_DESCR.REG_ANY)}
|
||||
vpcmpeqd ymm2, ymm2, ymm2
|
||||
vpsllq ymm2, ymm2, 63
|
||||
test rcx, rcx
|
||||
@@ -643,15 +640,15 @@ dec rcx
|
||||
jnz .Lneg_loop
|
||||
.Lneg_end:
|
||||
vzeroupper""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm2', 'rcx', 'r8', 'r10'])
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm2', 'rcx', 'r8', 'r10'])
|
||||
|
||||
|
||||
def simd_dot_array(a: t.CPtr, b: t.CPtr, n: sizet, out: t.CPtr):
|
||||
"""AVX2 batch: out[0] = dot(a[0:n], b[0:n]) using FMA accumulation"""
|
||||
c.Asm(f"""mov rcx, {c.AsmInp(n, t.ASM_DESCR.REG_ANY)}
|
||||
__asm__(f"""mov rcx, {asminp(n, ASM_DESCR.REG_ANY)}
|
||||
shr rcx, 2
|
||||
mov r8, {c.AsmInp(a, t.ASM_DESCR.REG_ANY)}
|
||||
mov r9, {c.AsmInp(b, t.ASM_DESCR.REG_ANY)}
|
||||
mov r8, {asminp(a, ASM_DESCR.REG_ANY)}
|
||||
mov r9, {asminp(b, ASM_DESCR.REG_ANY)}
|
||||
vxorpd ymm2, ymm2, ymm2
|
||||
test rcx, rcx
|
||||
jz .Ldot_end
|
||||
@@ -667,126 +664,126 @@ jnz .Ldot_loop
|
||||
vextractf128 xmm1, ymm2, 1
|
||||
vaddpd xmm2, xmm2, xmm1
|
||||
vhaddpd xmm2, xmm2, xmm2
|
||||
vmovsd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], xmm2
|
||||
vmovsd [{asminp(out, ASM_DESCR.REG_ANY)}], xmm2
|
||||
vzeroupper""",
|
||||
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'ymm2', 'rcx', 'r8', 'r9'])
|
||||
op=[ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'ymm2', 'rcx', 'r8', 'r9'])
|
||||
|
||||
|
||||
# ============================================================
|
||||
# LLVMIR precise scalar operations (LLVM intrinsic functions)
|
||||
# ============================================================
|
||||
|
||||
def simd_sqrt(x: t.CDouble) -> t.CDouble:
|
||||
def simd_sqrt(x: double) -> double:
|
||||
"""LLVM intrinsic: sqrt(x)"""
|
||||
out: t.CDouble
|
||||
c.LLVMIR(f"{c.LOut(out)} = call double @llvm.sqrt.f64(double {c.LInp(x)})")
|
||||
out: double
|
||||
__llvmir__(f"{irout(out)} = call double @llvm.sqrt.f64(double {irinp(x)})")
|
||||
return out
|
||||
|
||||
|
||||
def simd_fabs(x: t.CDouble) -> t.CDouble:
|
||||
def simd_fabs(x: double) -> double:
|
||||
"""LLVM intrinsic: |x|"""
|
||||
out: t.CDouble
|
||||
c.LLVMIR(f"{c.LOut(out)} = call double @llvm.fabs.f64(double {c.LInp(x)})")
|
||||
out: double
|
||||
__llvmir__(f"{irout(out)} = call double @llvm.fabs.f64(double {irinp(x)})")
|
||||
return out
|
||||
|
||||
|
||||
def simd_floor(x: t.CDouble) -> t.CDouble:
|
||||
def simd_floor(x: double) -> double:
|
||||
"""LLVM intrinsic: floor(x)"""
|
||||
out: t.CDouble
|
||||
c.LLVMIR(f"{c.LOut(out)} = call double @llvm.floor.f64(double {c.LInp(x)})")
|
||||
out: double
|
||||
__llvmir__(f"{irout(out)} = call double @llvm.floor.f64(double {irinp(x)})")
|
||||
return out
|
||||
|
||||
|
||||
def simd_ceil(x: t.CDouble) -> t.CDouble:
|
||||
def simd_ceil(x: double) -> double:
|
||||
"""LLVM intrinsic: ceil(x)"""
|
||||
out: t.CDouble
|
||||
c.LLVMIR(f"{c.LOut(out)} = call double @llvm.ceil.f64(double {c.LInp(x)})")
|
||||
out: double
|
||||
__llvmir__(f"{irout(out)} = call double @llvm.ceil.f64(double {irinp(x)})")
|
||||
return out
|
||||
|
||||
|
||||
def simd_round(x: t.CDouble) -> t.CDouble:
|
||||
def simd_round(x: double) -> double:
|
||||
"""LLVM intrinsic: round(x)"""
|
||||
out: t.CDouble
|
||||
c.LLVMIR(f"{c.LOut(out)} = call double @llvm.round.f64(double {c.LInp(x)})")
|
||||
out: double
|
||||
__llvmir__(f"{irout(out)} = call double @llvm.round.f64(double {irinp(x)})")
|
||||
return out
|
||||
|
||||
|
||||
def simd_trunc(x: t.CDouble) -> t.CDouble:
|
||||
def simd_trunc(x: double) -> double:
|
||||
"""LLVM intrinsic: trunc(x)"""
|
||||
out: t.CDouble
|
||||
c.LLVMIR(f"{c.LOut(out)} = call double @llvm.trunc.f64(double {c.LInp(x)})")
|
||||
out: double
|
||||
__llvmir__(f"{irout(out)} = call double @llvm.trunc.f64(double {irinp(x)})")
|
||||
return out
|
||||
|
||||
|
||||
def simd_fma(a: t.CDouble, b: t.CDouble, c_val: t.CDouble) -> t.CDouble:
|
||||
def simd_fma(a: double, b: double, c_val: double) -> double:
|
||||
"""LLVM intrinsic: a*b + c (single rounding)"""
|
||||
out: t.CDouble
|
||||
c.LLVMIR(f"{c.LOut(out)} = call double @llvm.fma.f64(double {c.LInp(a)}, double {c.LInp(b)}, double {c.LInp(c_val)})")
|
||||
out: double
|
||||
__llvmir__(f"{irout(out)} = call double @llvm.fma.f64(double {irinp(a)}, double {irinp(b)}, double {irinp(c_val)})")
|
||||
return out
|
||||
|
||||
|
||||
def simd_copysign(mag: t.CDouble, sign: t.CDouble) -> t.CDouble:
|
||||
def simd_copysign(mag: double, sign: double) -> double:
|
||||
"""LLVM intrinsic: copysign(mag, sign)"""
|
||||
out: t.CDouble
|
||||
c.LLVMIR(f"{c.LOut(out)} = call double @llvm.copysign.f64(double {c.LInp(mag)}, double {c.LInp(sign)})")
|
||||
out: double
|
||||
__llvmir__(f"{irout(out)} = call double @llvm.copysign.f64(double {irinp(mag)}, double {irinp(sign)})")
|
||||
return out
|
||||
|
||||
|
||||
def simd_neg(x: t.CDouble) -> t.CDouble:
|
||||
def simd_neg(x: double) -> double:
|
||||
"""LLVM intrinsic: -x (via copysign, correct for negative/-0.0/NaN inputs)
|
||||
|
||||
copysign(x, -1.0) is wrong for x < 0 (returns -|x| instead of +|x|).
|
||||
copysign(x, -x) = |x| * sign(-x) = -x is correct for all IEEE 754 values.
|
||||
"""
|
||||
out: t.CDouble
|
||||
c.LLVMIR(f"{c.LOut(out)} = call double @llvm.copysign.f64(double {c.LInp(x)}, double {c.LInp(t.CDouble(0.0) - x)})")
|
||||
out: double
|
||||
__llvmir__(f"{irout(out)} = call double @llvm.copysign.f64(double {irinp(x)}, double {irinp(double(0.0) - x)})")
|
||||
return out
|
||||
|
||||
|
||||
def simd_minnum(a: t.CDouble, b: t.CDouble) -> t.CDouble:
|
||||
def simd_minnum(a: double, b: double) -> double:
|
||||
"""LLVM intrinsic: min(a, b) (NaN-aware)"""
|
||||
out: t.CDouble
|
||||
c.LLVMIR(f"{c.LOut(out)} = call double @llvm.minnum.f64(double {c.LInp(a)}, double {c.LInp(b)})")
|
||||
out: double
|
||||
__llvmir__(f"{irout(out)} = call double @llvm.minnum.f64(double {irinp(a)}, double {irinp(b)})")
|
||||
return out
|
||||
|
||||
|
||||
def simd_maxnum(a: t.CDouble, b: t.CDouble) -> t.CDouble:
|
||||
def simd_maxnum(a: double, b: double) -> double:
|
||||
"""LLVM intrinsic: max(a, b) (NaN-aware)"""
|
||||
out: t.CDouble
|
||||
c.LLVMIR(f"{c.LOut(out)} = call double @llvm.maxnum.f64(double {c.LInp(a)}, double {c.LInp(b)})")
|
||||
out: double
|
||||
__llvmir__(f"{irout(out)} = call double @llvm.maxnum.f64(double {irinp(a)}, double {irinp(b)})")
|
||||
return out
|
||||
|
||||
|
||||
def simd_exp(x: t.CDouble) -> t.CDouble:
|
||||
def simd_exp(x: double) -> double:
|
||||
"""LLVM intrinsic: e^x"""
|
||||
out: t.CDouble
|
||||
c.LLVMIR(f"{c.LOut(out)} = call double @llvm.exp.f64(double {c.LInp(x)})")
|
||||
out: double
|
||||
__llvmir__(f"{irout(out)} = call double @llvm.exp.f64(double {irinp(x)})")
|
||||
return out
|
||||
|
||||
|
||||
def simd_log(x: t.CDouble) -> t.CDouble:
|
||||
def simd_log(x: double) -> double:
|
||||
"""LLVM intrinsic: ln(x)"""
|
||||
out: t.CDouble
|
||||
c.LLVMIR(f"{c.LOut(out)} = call double @llvm.log.f64(double {c.LInp(x)})")
|
||||
out: double
|
||||
__llvmir__(f"{irout(out)} = call double @llvm.log.f64(double {irinp(x)})")
|
||||
return out
|
||||
|
||||
|
||||
def simd_sin(x: t.CDouble) -> t.CDouble:
|
||||
def simd_sin(x: double) -> double:
|
||||
"""LLVM intrinsic: sin(x)"""
|
||||
out: t.CDouble
|
||||
c.LLVMIR(f"{c.LOut(out)} = call double @llvm.sin.f64(double {c.LInp(x)})")
|
||||
out: double
|
||||
__llvmir__(f"{irout(out)} = call double @llvm.sin.f64(double {irinp(x)})")
|
||||
return out
|
||||
|
||||
|
||||
def simd_cos(x: t.CDouble) -> t.CDouble:
|
||||
def simd_cos(x: double) -> double:
|
||||
"""LLVM intrinsic: cos(x)"""
|
||||
out: t.CDouble
|
||||
c.LLVMIR(f"{c.LOut(out)} = call double @llvm.cos.f64(double {c.LInp(x)})")
|
||||
out: double
|
||||
__llvmir__(f"{irout(out)} = call double @llvm.cos.f64(double {irinp(x)})")
|
||||
return out
|
||||
|
||||
|
||||
def simd_pow(x: t.CDouble, y: t.CDouble) -> t.CDouble:
|
||||
def simd_pow(x: double, y: double) -> double:
|
||||
"""LLVM intrinsic: x^y"""
|
||||
out: t.CDouble
|
||||
c.LLVMIR(f"{c.LOut(out)} = call double @llvm.pow.f64(double {c.LInp(x)}, double {c.LInp(y)})")
|
||||
out: double
|
||||
__llvmir__(f"{irout(out)} = call double @llvm.pow.f64(double {irinp(x)}, double {irinp(y)})")
|
||||
return out
|
||||
Reference in new issue
Block a user