461 lines
15 KiB
Python
461 lines
15 KiB
Python
import t, c
|
|
|
|
|
|
# ============================================================
|
|
# AVX2 双精度向量运算 (4 x double, 256-bit YMM)
|
|
# ============================================================
|
|
|
|
def simd_add4d(a: t.CPtr, b: t.CPtr, out: t.CPtr) -> t.CVoid:
|
|
"""AVX2: out[0:4] = a[0:4] + b[0:4]"""
|
|
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
|
vmovupd ymm1, [{c.AsmInp(b, t.ASM_DESCR.REG_ANY)}]
|
|
vaddpd ymm0, ymm0, ymm1
|
|
vmovupd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], ymm0""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
|
|
|
|
|
def simd_sub4d(a: t.CPtr, b: t.CPtr, out: t.CPtr) -> t.CVoid:
|
|
"""AVX2: out[0:4] = a[0:4] - b[0:4]"""
|
|
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
|
vmovupd ymm1, [{c.AsmInp(b, t.ASM_DESCR.REG_ANY)}]
|
|
vsubpd ymm0, ymm0, ymm1
|
|
vmovupd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], ymm0""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
|
|
|
|
|
def simd_mul4d(a: t.CPtr, b: t.CPtr, out: t.CPtr) -> t.CVoid:
|
|
"""AVX2: out[0:4] = a[0:4] * b[0:4]"""
|
|
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
|
vmovupd ymm1, [{c.AsmInp(b, t.ASM_DESCR.REG_ANY)}]
|
|
vmulpd ymm0, ymm0, ymm1
|
|
vmovupd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], ymm0""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
|
|
|
|
|
def simd_div4d(a: t.CPtr, b: t.CPtr, out: t.CPtr) -> t.CVoid:
|
|
"""AVX2: out[0:4] = a[0:4] / b[0:4]"""
|
|
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
|
vmovupd ymm1, [{c.AsmInp(b, t.ASM_DESCR.REG_ANY)}]
|
|
vdivpd ymm0, ymm0, ymm1
|
|
vmovupd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], ymm0""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
|
|
|
|
|
def simd_sqrt4d(a: t.CPtr, out: t.CPtr) -> t.CVoid:
|
|
"""AVX2: out[0:4] = sqrt(a[0:4])"""
|
|
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
|
vsqrtpd ymm0, ymm0
|
|
vmovupd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], ymm0""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0'])
|
|
|
|
|
|
def simd_neg4d(a: t.CPtr, out: t.CPtr) -> t.CVoid:
|
|
"""AVX2: out[0:4] = -a[0:4] (flip sign bit via XOR)"""
|
|
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
|
vpcmpeqd ymm1, ymm1, ymm1
|
|
vpsllq ymm1, ymm1, 63
|
|
vxorpd ymm0, ymm0, ymm1
|
|
vmovupd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], ymm0""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
|
|
|
|
|
def simd_abs4d(a: t.CPtr, out: t.CPtr) -> t.CVoid:
|
|
"""AVX2: out[0:4] = |a[0:4]| (clear sign bit via ANDN)"""
|
|
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
|
vpcmpeqd ymm1, ymm1, ymm1
|
|
vpsllq ymm1, ymm1, 63
|
|
vandnpd ymm0, ymm1, ymm0
|
|
vmovupd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], ymm0""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
|
|
|
|
|
def simd_max4d(a: t.CPtr, b: t.CPtr, out: t.CPtr) -> t.CVoid:
|
|
"""AVX2: out[0:4] = max(a[0:4], b[0:4])"""
|
|
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
|
vmovupd ymm1, [{c.AsmInp(b, t.ASM_DESCR.REG_ANY)}]
|
|
vmaxpd ymm0, ymm0, ymm1
|
|
vmovupd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], ymm0""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
|
|
|
|
|
def simd_min4d(a: t.CPtr, b: t.CPtr, out: t.CPtr) -> t.CVoid:
|
|
"""AVX2: out[0:4] = min(a[0:4], b[0:4])"""
|
|
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
|
vmovupd ymm1, [{c.AsmInp(b, t.ASM_DESCR.REG_ANY)}]
|
|
vminpd ymm0, ymm0, ymm1
|
|
vmovupd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], ymm0""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
|
|
|
|
|
def simd_mul_scalar4d(a: t.CPtr, scalar: t.CPtr, out: t.CPtr) -> t.CVoid:
|
|
"""AVX2: out[0:4] = a[0:4] * scalar (broadcast)"""
|
|
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
|
vbroadcastsd ymm1, [{c.AsmInp(scalar, t.ASM_DESCR.REG_ANY)}]
|
|
vmulpd ymm0, ymm0, ymm1
|
|
vmovupd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], ymm0""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
|
|
|
|
|
def simd_add_scalar4d(a: t.CPtr, scalar: t.CPtr, out: t.CPtr) -> t.CVoid:
|
|
"""AVX2: out[0:4] = a[0:4] + scalar (broadcast)"""
|
|
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
|
vbroadcastsd ymm1, [{c.AsmInp(scalar, t.ASM_DESCR.REG_ANY)}]
|
|
vaddpd ymm0, ymm0, ymm1
|
|
vmovupd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], ymm0""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
|
|
|
|
|
def simd_hsum4d(a: t.CPtr, out: t.CPtr) -> t.CVoid:
|
|
"""AVX2: out[0] = a[0]+a[1]+a[2]+a[3] (horizontal sum)"""
|
|
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
|
vextractf128 xmm1, ymm0, 1
|
|
vaddpd xmm0, xmm0, xmm1
|
|
vhaddpd xmm0, xmm0, xmm0
|
|
vmovsd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], xmm0""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
|
|
|
|
|
def simd_dot4d(a: t.CPtr, b: t.CPtr, out: t.CPtr) -> t.CVoid:
|
|
"""AVX2: out[0] = dot(a[0:4], b[0:4])"""
|
|
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
|
vmovupd ymm1, [{c.AsmInp(b, t.ASM_DESCR.REG_ANY)}]
|
|
vmulpd ymm0, ymm0, ymm1
|
|
vextractf128 xmm1, ymm0, 1
|
|
vaddpd xmm0, xmm0, xmm1
|
|
vhaddpd xmm0, xmm0, xmm0
|
|
vmovsd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], xmm0""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1'])
|
|
|
|
|
|
def simd_fma4d(a: t.CPtr, b: t.CPtr, c_val: t.CPtr, out: t.CPtr) -> t.CVoid:
|
|
"""AVX2: out[0:4] = a[0:4]*b[0:4] + c[0:4] (fused multiply-add)
|
|
Uses vfmadd213pd: ymm0 = ymm1 * ymm0 + ymm2
|
|
Load order: b->ymm0, a->ymm1, c->ymm2"""
|
|
c.Asm(f"""vmovupd ymm0, [{c.AsmInp(b, t.ASM_DESCR.REG_ANY)}]
|
|
vmovupd ymm1, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
|
vmovupd ymm2, [{c.AsmInp(c_val, t.ASM_DESCR.REG_ANY)}]
|
|
vfmadd213pd ymm0, ymm1, ymm2
|
|
vmovupd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], ymm0""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'ymm2'])
|
|
|
|
|
|
# ============================================================
|
|
# SSE2 单精度向量运算 (4 x float, 128-bit XMM)
|
|
# ============================================================
|
|
|
|
def simd_add4f(a: t.CPtr, b: t.CPtr, out: t.CPtr) -> t.CVoid:
|
|
"""SSE2: out[0:4] = a[0:4] + b[0:4] (float)"""
|
|
c.Asm(f"""movups xmm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
|
movups xmm1, [{c.AsmInp(b, t.ASM_DESCR.REG_ANY)}]
|
|
addps xmm0, xmm1
|
|
movups [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], xmm0""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'xmm0', 'xmm1'])
|
|
|
|
|
|
def simd_sub4f(a: t.CPtr, b: t.CPtr, out: t.CPtr) -> t.CVoid:
|
|
"""SSE2: out[0:4] = a[0:4] - b[0:4] (float)"""
|
|
c.Asm(f"""movups xmm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
|
movups xmm1, [{c.AsmInp(b, t.ASM_DESCR.REG_ANY)}]
|
|
subps xmm0, xmm1
|
|
movups [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], xmm0""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'xmm0', 'xmm1'])
|
|
|
|
|
|
def simd_mul4f(a: t.CPtr, b: t.CPtr, out: t.CPtr) -> t.CVoid:
|
|
"""SSE2: out[0:4] = a[0:4] * b[0:4] (float)"""
|
|
c.Asm(f"""movups xmm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
|
movups xmm1, [{c.AsmInp(b, t.ASM_DESCR.REG_ANY)}]
|
|
mulps xmm0, xmm1
|
|
movups [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], xmm0""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'xmm0', 'xmm1'])
|
|
|
|
|
|
def simd_div4f(a: t.CPtr, b: t.CPtr, out: t.CPtr) -> t.CVoid:
|
|
"""SSE2: out[0:4] = a[0:4] / b[0:4] (float)"""
|
|
c.Asm(f"""movups xmm0, [{c.AsmInp(a, t.ASM_DESCR.REG_ANY)}]
|
|
movups xmm1, [{c.AsmInp(b, t.ASM_DESCR.REG_ANY)}]
|
|
divps xmm0, xmm1
|
|
movups [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], xmm0""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'xmm0', 'xmm1'])
|
|
|
|
|
|
# ============================================================
|
|
# 批量数组运算 (AVX2 循环, 处理 N 个 double)
|
|
# ============================================================
|
|
|
|
def simd_add_array(a: t.CPtr, b: t.CPtr, out: t.CPtr, n: t.CSizeT) -> t.CVoid:
|
|
"""AVX2 batch: out[i:i+4] = a[i:i+4] + b[i:i+4], i=0,4,8,..."""
|
|
c.Asm(f"""mov rcx, {c.AsmInp(n, t.ASM_DESCR.REG_ANY)}
|
|
shr rcx, 2
|
|
mov r8, {c.AsmInp(a, t.ASM_DESCR.REG_ANY)}
|
|
mov r9, {c.AsmInp(b, t.ASM_DESCR.REG_ANY)}
|
|
mov r10, {c.AsmInp(out, t.ASM_DESCR.REG_ANY)}
|
|
test rcx, rcx
|
|
jz .Ladd_end
|
|
.Ladd_loop:
|
|
vmovupd ymm0, [r8]
|
|
vmovupd ymm1, [r9]
|
|
vaddpd ymm0, ymm0, ymm1
|
|
vmovupd [r10], ymm0
|
|
add r8, 32
|
|
add r9, 32
|
|
add r10, 32
|
|
dec rcx
|
|
jnz .Ladd_loop
|
|
.Ladd_end:
|
|
vzeroupper""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'rcx', 'r8', 'r9', 'r10'])
|
|
|
|
|
|
def simd_sub_array(a: t.CPtr, b: t.CPtr, out: t.CPtr, n: t.CSizeT) -> t.CVoid:
|
|
"""AVX2 batch: out[i:i+4] = a[i:i+4] - b[i:i+4]"""
|
|
c.Asm(f"""mov rcx, {c.AsmInp(n, t.ASM_DESCR.REG_ANY)}
|
|
shr rcx, 2
|
|
mov r8, {c.AsmInp(a, t.ASM_DESCR.REG_ANY)}
|
|
mov r9, {c.AsmInp(b, t.ASM_DESCR.REG_ANY)}
|
|
mov r10, {c.AsmInp(out, t.ASM_DESCR.REG_ANY)}
|
|
test rcx, rcx
|
|
jz .Lsub_end
|
|
.Lsub_loop:
|
|
vmovupd ymm0, [r8]
|
|
vmovupd ymm1, [r9]
|
|
vsubpd ymm0, ymm0, ymm1
|
|
vmovupd [r10], ymm0
|
|
add r8, 32
|
|
add r9, 32
|
|
add r10, 32
|
|
dec rcx
|
|
jnz .Lsub_loop
|
|
.Lsub_end:
|
|
vzeroupper""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'rcx', 'r8', 'r9', 'r10'])
|
|
|
|
|
|
def simd_mul_array(a: t.CPtr, b: t.CPtr, out: t.CPtr, n: t.CSizeT) -> t.CVoid:
|
|
"""AVX2 batch: out[i:i+4] = a[i:i+4] * b[i:i+4]"""
|
|
c.Asm(f"""mov rcx, {c.AsmInp(n, t.ASM_DESCR.REG_ANY)}
|
|
shr rcx, 2
|
|
mov r8, {c.AsmInp(a, t.ASM_DESCR.REG_ANY)}
|
|
mov r9, {c.AsmInp(b, t.ASM_DESCR.REG_ANY)}
|
|
mov r10, {c.AsmInp(out, t.ASM_DESCR.REG_ANY)}
|
|
test rcx, rcx
|
|
jz .Lmul_end
|
|
.Lmul_loop:
|
|
vmovupd ymm0, [r8]
|
|
vmovupd ymm1, [r9]
|
|
vmulpd ymm0, ymm0, ymm1
|
|
vmovupd [r10], ymm0
|
|
add r8, 32
|
|
add r9, 32
|
|
add r10, 32
|
|
dec rcx
|
|
jnz .Lmul_loop
|
|
.Lmul_end:
|
|
vzeroupper""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'rcx', 'r8', 'r9', 'r10'])
|
|
|
|
|
|
def simd_div_array(a: t.CPtr, b: t.CPtr, out: t.CPtr, n: t.CSizeT) -> t.CVoid:
|
|
"""AVX2 batch: out[i:i+4] = a[i:i+4] / b[i:i+4]"""
|
|
c.Asm(f"""mov rcx, {c.AsmInp(n, t.ASM_DESCR.REG_ANY)}
|
|
shr rcx, 2
|
|
mov r8, {c.AsmInp(a, t.ASM_DESCR.REG_ANY)}
|
|
mov r9, {c.AsmInp(b, t.ASM_DESCR.REG_ANY)}
|
|
mov r10, {c.AsmInp(out, t.ASM_DESCR.REG_ANY)}
|
|
test rcx, rcx
|
|
jz .Ldiv_end
|
|
.Ldiv_loop:
|
|
vmovupd ymm0, [r8]
|
|
vmovupd ymm1, [r9]
|
|
vdivpd ymm0, ymm0, ymm1
|
|
vmovupd [r10], ymm0
|
|
add r8, 32
|
|
add r9, 32
|
|
add r10, 32
|
|
dec rcx
|
|
jnz .Ldiv_loop
|
|
.Ldiv_end:
|
|
vzeroupper""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'rcx', 'r8', 'r9', 'r10'])
|
|
|
|
|
|
def simd_sqrt_array(a: t.CPtr, out: t.CPtr, n: t.CSizeT) -> t.CVoid:
|
|
"""AVX2 batch: out[i:i+4] = sqrt(a[i:i+4])"""
|
|
c.Asm(f"""mov rcx, {c.AsmInp(n, t.ASM_DESCR.REG_ANY)}
|
|
shr rcx, 2
|
|
mov r8, {c.AsmInp(a, t.ASM_DESCR.REG_ANY)}
|
|
mov r10, {c.AsmInp(out, t.ASM_DESCR.REG_ANY)}
|
|
test rcx, rcx
|
|
jz .Lsqrt_end
|
|
.Lsqrt_loop:
|
|
vmovupd ymm0, [r8]
|
|
vsqrtpd ymm0, ymm0
|
|
vmovupd [r10], ymm0
|
|
add r8, 32
|
|
add r10, 32
|
|
dec rcx
|
|
jnz .Lsqrt_loop
|
|
.Lsqrt_end:
|
|
vzeroupper""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'rcx', 'r8', 'r10'])
|
|
|
|
|
|
def simd_abs_array(a: t.CPtr, out: t.CPtr, n: t.CSizeT) -> t.CVoid:
|
|
"""AVX2 batch: out[i:i+4] = |a[i:i+4]|"""
|
|
c.Asm(f"""mov rcx, {c.AsmInp(n, t.ASM_DESCR.REG_ANY)}
|
|
shr rcx, 2
|
|
mov r8, {c.AsmInp(a, t.ASM_DESCR.REG_ANY)}
|
|
mov r10, {c.AsmInp(out, t.ASM_DESCR.REG_ANY)}
|
|
vpcmpeqd ymm2, ymm2, ymm2
|
|
vpsllq ymm2, ymm2, 63
|
|
test rcx, rcx
|
|
jz .Labs_end
|
|
.Labs_loop:
|
|
vmovupd ymm0, [r8]
|
|
vandnpd ymm0, ymm2, ymm0
|
|
vmovupd [r10], ymm0
|
|
add r8, 32
|
|
add r10, 32
|
|
dec rcx
|
|
jnz .Labs_loop
|
|
.Labs_end:
|
|
vzeroupper""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm2', 'rcx', 'r8', 'r10'])
|
|
|
|
|
|
def simd_neg_array(a: t.CPtr, out: t.CPtr, n: t.CSizeT) -> t.CVoid:
|
|
"""AVX2 batch: out[i:i+4] = -a[i:i+4] (flip sign bit via XOR)"""
|
|
c.Asm(f"""mov rcx, {c.AsmInp(n, t.ASM_DESCR.REG_ANY)}
|
|
shr rcx, 2
|
|
mov r8, {c.AsmInp(a, t.ASM_DESCR.REG_ANY)}
|
|
mov r10, {c.AsmInp(out, t.ASM_DESCR.REG_ANY)}
|
|
vpcmpeqd ymm2, ymm2, ymm2
|
|
vpsllq ymm2, ymm2, 63
|
|
test rcx, rcx
|
|
jz .Lneg_end
|
|
.Lneg_loop:
|
|
vmovupd ymm0, [r8]
|
|
vxorpd ymm0, ymm0, ymm2
|
|
vmovupd [r10], ymm0
|
|
add r8, 32
|
|
add r10, 32
|
|
dec rcx
|
|
jnz .Lneg_loop
|
|
.Lneg_end:
|
|
vzeroupper""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm2', 'rcx', 'r8', 'r10'])
|
|
|
|
|
|
def simd_dot_array(a: t.CPtr, b: t.CPtr, n: t.CSizeT, out: t.CPtr) -> t.CVoid:
|
|
"""AVX2 batch: out[0] = dot(a[0:n], b[0:n]) using FMA accumulation"""
|
|
c.Asm(f"""mov rcx, {c.AsmInp(n, t.ASM_DESCR.REG_ANY)}
|
|
shr rcx, 2
|
|
mov r8, {c.AsmInp(a, t.ASM_DESCR.REG_ANY)}
|
|
mov r9, {c.AsmInp(b, t.ASM_DESCR.REG_ANY)}
|
|
vxorpd ymm2, ymm2, ymm2
|
|
test rcx, rcx
|
|
jz .Ldot_end
|
|
.Ldot_loop:
|
|
vmovupd ymm0, [r8]
|
|
vmovupd ymm1, [r9]
|
|
vfmadd231pd ymm2, ymm0, ymm1
|
|
add r8, 32
|
|
add r9, 32
|
|
dec rcx
|
|
jnz .Ldot_loop
|
|
.Ldot_end:
|
|
vextractf128 xmm1, ymm2, 1
|
|
vaddpd xmm2, xmm2, xmm1
|
|
vhaddpd xmm2, xmm2, xmm2
|
|
vmovsd [{c.AsmInp(out, t.ASM_DESCR.REG_ANY)}], xmm2
|
|
vzeroupper""",
|
|
op=[t.ASM_DESCR.CLOBBER_MEMORY, 'ymm0', 'ymm1', 'ymm2', 'rcx', 'r8', 'r9'])
|
|
|
|
|
|
# ============================================================
|
|
# LLVMIR 精确标量运算 (LLVM 内置函数)
|
|
# ============================================================
|
|
|
|
def simd_sqrt(x: t.CDouble) -> t.CDouble:
|
|
"""LLVM intrinsic: sqrt(x)"""
|
|
return c.LLVMIR(f"call double @llvm.sqrt.f64(double {c.LInp(x)})", t.CDouble)
|
|
|
|
|
|
def simd_fabs(x: t.CDouble) -> t.CDouble:
|
|
"""LLVM intrinsic: |x|"""
|
|
return c.LLVMIR(f"call double @llvm.fabs.f64(double {c.LInp(x)})", t.CDouble)
|
|
|
|
|
|
def simd_floor(x: t.CDouble) -> t.CDouble:
|
|
"""LLVM intrinsic: floor(x)"""
|
|
return c.LLVMIR(f"call double @llvm.floor.f64(double {c.LInp(x)})", t.CDouble)
|
|
|
|
|
|
def simd_ceil(x: t.CDouble) -> t.CDouble:
|
|
"""LLVM intrinsic: ceil(x)"""
|
|
return c.LLVMIR(f"call double @llvm.ceil.f64(double {c.LInp(x)})", t.CDouble)
|
|
|
|
|
|
def simd_round(x: t.CDouble) -> t.CDouble:
|
|
"""LLVM intrinsic: round(x)"""
|
|
return c.LLVMIR(f"call double @llvm.round.f64(double {c.LInp(x)})", t.CDouble)
|
|
|
|
|
|
def simd_trunc(x: t.CDouble) -> t.CDouble:
|
|
"""LLVM intrinsic: trunc(x)"""
|
|
return c.LLVMIR(f"call double @llvm.trunc.f64(double {c.LInp(x)})", t.CDouble)
|
|
|
|
|
|
def simd_fma(a: t.CDouble, b: t.CDouble, c_val: t.CDouble) -> t.CDouble:
|
|
"""LLVM intrinsic: a*b + c (single rounding)"""
|
|
return c.LLVMIR(f"call double @llvm.fma.f64(double {c.LInp(a)}, double {c.LInp(b)}, double {c.LInp(c_val)})", t.CDouble)
|
|
|
|
|
|
def simd_copysign(mag: t.CDouble, sign: t.CDouble) -> t.CDouble:
|
|
"""LLVM intrinsic: copysign(mag, sign)"""
|
|
return c.LLVMIR(f"call double @llvm.copysign.f64(double {c.LInp(mag)}, double {c.LInp(sign)})", t.CDouble)
|
|
|
|
|
|
def simd_neg(x: t.CDouble) -> t.CDouble:
|
|
"""LLVM intrinsic: -x (via copysign, correct for negative/-0.0/NaN inputs)
|
|
|
|
copysign(x, -1.0) is wrong for x < 0 (returns -|x| instead of +|x|).
|
|
copysign(x, -x) = |x| * sign(-x) = -x is correct for all IEEE 754 values.
|
|
"""
|
|
return c.LLVMIR(f"call double @llvm.copysign.f64(double {c.LInp(x)}, double {c.LInp(t.CDouble(0.0) - x)})", t.CDouble)
|
|
|
|
|
|
def simd_minnum(a: t.CDouble, b: t.CDouble) -> t.CDouble:
|
|
"""LLVM intrinsic: min(a, b) (NaN-aware)"""
|
|
return c.LLVMIR(f"call double @llvm.minnum.f64(double {c.LInp(a)}, double {c.LInp(b)})", t.CDouble)
|
|
|
|
|
|
def simd_maxnum(a: t.CDouble, b: t.CDouble) -> t.CDouble:
|
|
"""LLVM intrinsic: max(a, b) (NaN-aware)"""
|
|
return c.LLVMIR(f"call double @llvm.maxnum.f64(double {c.LInp(a)}, double {c.LInp(b)})", t.CDouble)
|
|
|
|
|
|
def simd_exp(x: t.CDouble) -> t.CDouble:
|
|
"""LLVM intrinsic: e^x"""
|
|
return c.LLVMIR(f"call double @llvm.exp.f64(double {c.LInp(x)})", t.CDouble)
|
|
|
|
|
|
def simd_log(x: t.CDouble) -> t.CDouble:
|
|
"""LLVM intrinsic: ln(x)"""
|
|
return c.LLVMIR(f"call double @llvm.log.f64(double {c.LInp(x)})", t.CDouble)
|
|
|
|
|
|
def simd_sin(x: t.CDouble) -> t.CDouble:
|
|
"""LLVM intrinsic: sin(x)"""
|
|
return c.LLVMIR(f"call double @llvm.sin.f64(double {c.LInp(x)})", t.CDouble)
|
|
|
|
|
|
def simd_cos(x: t.CDouble) -> t.CDouble:
|
|
"""LLVM intrinsic: cos(x)"""
|
|
return c.LLVMIR(f"call double @llvm.cos.f64(double {c.LInp(x)})", t.CDouble)
|
|
|
|
|
|
def simd_pow(x: t.CDouble, y: t.CDouble) -> t.CDouble:
|
|
"""LLVM intrinsic: x^y"""
|
|
return c.LLVMIR(f"call double @llvm.pow.f64(double {c.LInp(x)}, double {c.LInp(y)})", t.CDouble)
|