1 Commits

Author SHA1 Message Date
f9d9a78e23 crypto/sha512: improve performance of riscv64 assembly when Zbb extension available
When both pointers are aligned and the Zbb extension is available, the REV8 instruction can be used to simplify crypto/sha512.

goos: linux
goarch: riscv64
pkg: crypto/sha512
                    │     old     │                dev                 │
                    │   sec/op    │   sec/op     vs base               │
Hash8Bytes/New-4      1.370µ ± 0%   1.310µ ± 0%  -4.42% (p=0.000 n=12)
Hash8Bytes/Sum384-4   1.438µ ± 0%   1.366µ ± 0%  -5.01% (p=0.000 n=12)
Hash8Bytes/Sum512-4   1.462µ ± 0%   1.403µ ± 0%  -4.04% (p=0.000 n=12)
Hash1K/New-4          8.808µ ± 0%   8.272µ ± 0%  -6.09% (p=0.000 n=12)
Hash1K/Sum384-4       8.869µ ± 0%   8.322µ ± 0%  -6.17% (p=0.000 n=12)
Hash1K/Sum512-4       8.892µ ± 0%   8.357µ ± 0%  -6.01% (p=0.000 n=12)
Hash8K/New-4          61.04µ ± 0%   57.19µ ± 0%  -6.30% (p=0.000 n=12)
Hash8K/Sum384-4       61.10µ ± 0%   57.24µ ± 0%  -6.31% (p=0.000 n=12)
Hash8K/Sum512-4       61.12µ ± 0%   57.27µ ± 0%  -6.30% (p=0.000 n=12)
geomean               9.164µ        8.648µ       -5.63%

                    │     old      │                 dev                 │
                    │     B/s      │     B/s       vs base               │
Hash8Bytes/New-4      5.569Mi ± 0%   5.827Mi ± 0%  +4.62% (p=0.000 n=12)
Hash8Bytes/Sum384-4   5.302Mi ± 0%   5.584Mi ± 0%  +5.31% (p=0.000 n=12)
Hash8Bytes/Sum512-4   5.217Mi ± 0%   5.436Mi ± 0%  +4.20% (p=0.000 n=12)
Hash1K/New-4          110.9Mi ± 0%   118.1Mi ± 0%  +6.47% (p=0.000 n=12)
Hash1K/Sum384-4       110.1Mi ± 0%   117.4Mi ± 0%  +6.57% (p=0.000 n=12)
Hash1K/Sum512-4       109.8Mi ± 0%   116.8Mi ± 0%  +6.40% (p=0.000 n=12)
Hash8K/New-4          128.0Mi ± 0%   136.6Mi ± 0%  +6.72% (p=0.000 n=12)
Hash8K/Sum384-4       127.9Mi ± 0%   136.5Mi ± 0%  +6.74% (p=0.000 n=12)
Hash8K/Sum512-4       127.8Mi ± 0%   136.4Mi ± 0%  +6.72% (p=0.000 n=12)
geomean               42.29Mi        44.81Mi       +5.97%

Change-Id: Icb9cdec11ce178689b0ec961e23cf3991dadd4b5
2026-01-28 23:09:08 +08:00
4 changed files with 154 additions and 84 deletions

View File

@@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build (loong64 || riscv64) && !purego
//go:build loong64 && !purego
package sha512

View File

@@ -0,0 +1,20 @@
// Copyright 2025 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !purego
package sha512
import (
"crypto/internal/fips140deps/cpu"
)
// TODO(julian): use Zvknhb when available.
//go:noescape
func blockRISCV64(dig *Digest, p []byte, hasZbb bool)
func block(dig *Digest, p []byte) {
blockRISCV64(dig, p, cpu.RISCV64HasZbb)
}

View File

@@ -4,6 +4,8 @@
//go:build !purego
#include "asm_riscv64.h"
#include "go_asm.h"
#include "textflag.h"
// SHA512 block routine. See sha512block.go for Go equivalent.
@@ -72,6 +74,11 @@
OR X5, X8, X5; \
MOV X5, (index*8)(X19)
#define MSGSCHEDULE0_ZBB(index) \
MOV ((index*8))(X29), X5; \
REV8 X5, X5; \
MOV X5, (index*8)(X19)
// Wt = SIGMA1(Wt-2) + Wt-7 + SIGMA0(Wt-15) + Wt-16; for 16 <= t <= 79
// SIGMA0(x) = ROTR(1,x) XOR ROTR(8,x) XOR SHR(7,x)
// SIGMA1(x) = ROTR(19,x) XOR ROTR(61,x) XOR SHR(6,x)
@@ -145,12 +152,118 @@
MSGSCHEDULE0(index); \
SHA512ROUND(index, a, b, c, d, e, f, g, h)
#define SHA512ROUND0_ZBB(index, a, b, c, d, e, f, g, h) \
MSGSCHEDULE0_ZBB(index); \
SHA512ROUND(index, a, b, c, d, e, f, g, h)
#define SHA512ROUND1(index, a, b, c, d, e, f, g, h) \
MSGSCHEDULE1(index); \
SHA512ROUND(index, a, b, c, d, e, f, g, h)
// func block(dig *Digest, p []byte)
TEXT ·block(SB),0,$128-32
#define SHA512ROUND0_0_15() \
SHA512ROUND0(0, X10, X11, X12, X13, X14, X15, X16, X17) \
SHA512ROUND0(1, X17, X10, X11, X12, X13, X14, X15, X16) \
SHA512ROUND0(2, X16, X17, X10, X11, X12, X13, X14, X15) \
SHA512ROUND0(3, X15, X16, X17, X10, X11, X12, X13, X14) \
SHA512ROUND0(4, X14, X15, X16, X17, X10, X11, X12, X13) \
SHA512ROUND0(5, X13, X14, X15, X16, X17, X10, X11, X12) \
SHA512ROUND0(6, X12, X13, X14, X15, X16, X17, X10, X11) \
SHA512ROUND0(7, X11, X12, X13, X14, X15, X16, X17, X10) \
SHA512ROUND0(8, X10, X11, X12, X13, X14, X15, X16, X17) \
SHA512ROUND0(9, X17, X10, X11, X12, X13, X14, X15, X16) \
SHA512ROUND0(10, X16, X17, X10, X11, X12, X13, X14, X15) \
SHA512ROUND0(11, X15, X16, X17, X10, X11, X12, X13, X14) \
SHA512ROUND0(12, X14, X15, X16, X17, X10, X11, X12, X13) \
SHA512ROUND0(13, X13, X14, X15, X16, X17, X10, X11, X12) \
SHA512ROUND0(14, X12, X13, X14, X15, X16, X17, X10, X11) \
SHA512ROUND0(15, X11, X12, X13, X14, X15, X16, X17, X10)
#define SHA512ROUND0_0_15_ZBB() \
SHA512ROUND0_ZBB(0, X10, X11, X12, X13, X14, X15, X16, X17) \
SHA512ROUND0_ZBB(1, X17, X10, X11, X12, X13, X14, X15, X16) \
SHA512ROUND0_ZBB(2, X16, X17, X10, X11, X12, X13, X14, X15) \
SHA512ROUND0_ZBB(3, X15, X16, X17, X10, X11, X12, X13, X14) \
SHA512ROUND0_ZBB(4, X14, X15, X16, X17, X10, X11, X12, X13) \
SHA512ROUND0_ZBB(5, X13, X14, X15, X16, X17, X10, X11, X12) \
SHA512ROUND0_ZBB(6, X12, X13, X14, X15, X16, X17, X10, X11) \
SHA512ROUND0_ZBB(7, X11, X12, X13, X14, X15, X16, X17, X10) \
SHA512ROUND0_ZBB(8, X10, X11, X12, X13, X14, X15, X16, X17) \
SHA512ROUND0_ZBB(9, X17, X10, X11, X12, X13, X14, X15, X16) \
SHA512ROUND0_ZBB(10, X16, X17, X10, X11, X12, X13, X14, X15) \
SHA512ROUND0_ZBB(11, X15, X16, X17, X10, X11, X12, X13, X14) \
SHA512ROUND0_ZBB(12, X14, X15, X16, X17, X10, X11, X12, X13) \
SHA512ROUND0_ZBB(13, X13, X14, X15, X16, X17, X10, X11, X12) \
SHA512ROUND0_ZBB(14, X12, X13, X14, X15, X16, X17, X10, X11) \
SHA512ROUND0_ZBB(15, X11, X12, X13, X14, X15, X16, X17, X10)
#define SHA512ROUND1_16_79() \
SHA512ROUND1(16, X10, X11, X12, X13, X14, X15, X16, X17) \
SHA512ROUND1(17, X17, X10, X11, X12, X13, X14, X15, X16) \
SHA512ROUND1(18, X16, X17, X10, X11, X12, X13, X14, X15) \
SHA512ROUND1(19, X15, X16, X17, X10, X11, X12, X13, X14) \
SHA512ROUND1(20, X14, X15, X16, X17, X10, X11, X12, X13) \
SHA512ROUND1(21, X13, X14, X15, X16, X17, X10, X11, X12) \
SHA512ROUND1(22, X12, X13, X14, X15, X16, X17, X10, X11) \
SHA512ROUND1(23, X11, X12, X13, X14, X15, X16, X17, X10) \
SHA512ROUND1(24, X10, X11, X12, X13, X14, X15, X16, X17) \
SHA512ROUND1(25, X17, X10, X11, X12, X13, X14, X15, X16) \
SHA512ROUND1(26, X16, X17, X10, X11, X12, X13, X14, X15) \
SHA512ROUND1(27, X15, X16, X17, X10, X11, X12, X13, X14) \
SHA512ROUND1(28, X14, X15, X16, X17, X10, X11, X12, X13) \
SHA512ROUND1(29, X13, X14, X15, X16, X17, X10, X11, X12) \
SHA512ROUND1(30, X12, X13, X14, X15, X16, X17, X10, X11) \
SHA512ROUND1(31, X11, X12, X13, X14, X15, X16, X17, X10) \
SHA512ROUND1(32, X10, X11, X12, X13, X14, X15, X16, X17) \
SHA512ROUND1(33, X17, X10, X11, X12, X13, X14, X15, X16) \
SHA512ROUND1(34, X16, X17, X10, X11, X12, X13, X14, X15) \
SHA512ROUND1(35, X15, X16, X17, X10, X11, X12, X13, X14) \
SHA512ROUND1(36, X14, X15, X16, X17, X10, X11, X12, X13) \
SHA512ROUND1(37, X13, X14, X15, X16, X17, X10, X11, X12) \
SHA512ROUND1(38, X12, X13, X14, X15, X16, X17, X10, X11) \
SHA512ROUND1(39, X11, X12, X13, X14, X15, X16, X17, X10) \
SHA512ROUND1(40, X10, X11, X12, X13, X14, X15, X16, X17) \
SHA512ROUND1(41, X17, X10, X11, X12, X13, X14, X15, X16) \
SHA512ROUND1(42, X16, X17, X10, X11, X12, X13, X14, X15) \
SHA512ROUND1(43, X15, X16, X17, X10, X11, X12, X13, X14) \
SHA512ROUND1(44, X14, X15, X16, X17, X10, X11, X12, X13) \
SHA512ROUND1(45, X13, X14, X15, X16, X17, X10, X11, X12) \
SHA512ROUND1(46, X12, X13, X14, X15, X16, X17, X10, X11) \
SHA512ROUND1(47, X11, X12, X13, X14, X15, X16, X17, X10) \
SHA512ROUND1(48, X10, X11, X12, X13, X14, X15, X16, X17) \
SHA512ROUND1(49, X17, X10, X11, X12, X13, X14, X15, X16) \
SHA512ROUND1(50, X16, X17, X10, X11, X12, X13, X14, X15) \
SHA512ROUND1(51, X15, X16, X17, X10, X11, X12, X13, X14) \
SHA512ROUND1(52, X14, X15, X16, X17, X10, X11, X12, X13) \
SHA512ROUND1(53, X13, X14, X15, X16, X17, X10, X11, X12) \
SHA512ROUND1(54, X12, X13, X14, X15, X16, X17, X10, X11) \
SHA512ROUND1(55, X11, X12, X13, X14, X15, X16, X17, X10) \
SHA512ROUND1(56, X10, X11, X12, X13, X14, X15, X16, X17) \
SHA512ROUND1(57, X17, X10, X11, X12, X13, X14, X15, X16) \
SHA512ROUND1(58, X16, X17, X10, X11, X12, X13, X14, X15) \
SHA512ROUND1(59, X15, X16, X17, X10, X11, X12, X13, X14) \
SHA512ROUND1(60, X14, X15, X16, X17, X10, X11, X12, X13) \
SHA512ROUND1(61, X13, X14, X15, X16, X17, X10, X11, X12) \
SHA512ROUND1(62, X12, X13, X14, X15, X16, X17, X10, X11) \
SHA512ROUND1(63, X11, X12, X13, X14, X15, X16, X17, X10) \
SHA512ROUND1(64, X10, X11, X12, X13, X14, X15, X16, X17) \
SHA512ROUND1(65, X17, X10, X11, X12, X13, X14, X15, X16) \
SHA512ROUND1(66, X16, X17, X10, X11, X12, X13, X14, X15) \
SHA512ROUND1(67, X15, X16, X17, X10, X11, X12, X13, X14) \
SHA512ROUND1(68, X14, X15, X16, X17, X10, X11, X12, X13) \
SHA512ROUND1(69, X13, X14, X15, X16, X17, X10, X11, X12) \
SHA512ROUND1(70, X12, X13, X14, X15, X16, X17, X10, X11) \
SHA512ROUND1(71, X11, X12, X13, X14, X15, X16, X17, X10) \
SHA512ROUND1(72, X10, X11, X12, X13, X14, X15, X16, X17) \
SHA512ROUND1(73, X17, X10, X11, X12, X13, X14, X15, X16) \
SHA512ROUND1(74, X16, X17, X10, X11, X12, X13, X14, X15) \
SHA512ROUND1(75, X15, X16, X17, X10, X11, X12, X13, X14) \
SHA512ROUND1(76, X14, X15, X16, X17, X10, X11, X12, X13) \
SHA512ROUND1(77, X13, X14, X15, X16, X17, X10, X11, X12) \
SHA512ROUND1(78, X12, X13, X14, X15, X16, X17, X10, X11) \
SHA512ROUND1(79, X11, X12, X13, X14, X15, X16, X17, X10)
// func blockRISCV64(dig *Digest, p []byte, hasZbb bool)
TEXT ·blockRISCV64(SB),0,$128-33
MOV p_base+8(FP), X29
MOV p_len+16(FP), X30
SRL $7, X30
@@ -159,6 +272,14 @@ TEXT ·block(SB),0,$128-32
ADD X29, X30, X28
BEQ X28, X29, end
AND $7, X29, X25
#ifndef hasZbb
MOVB hasZbb+32(FP), X5
SEQZ X5, X5
AND X5, X25
#endif
MOV $·_K(SB), X18 // const table
ADD $8, X2, X19 // message schedule
@@ -173,87 +294,15 @@ TEXT ·block(SB),0,$128-32
MOV (7*8)(X20), X17 // h = H7
loop:
SHA512ROUND0(0, X10, X11, X12, X13, X14, X15, X16, X17)
SHA512ROUND0(1, X17, X10, X11, X12, X13, X14, X15, X16)
SHA512ROUND0(2, X16, X17, X10, X11, X12, X13, X14, X15)
SHA512ROUND0(3, X15, X16, X17, X10, X11, X12, X13, X14)
SHA512ROUND0(4, X14, X15, X16, X17, X10, X11, X12, X13)
SHA512ROUND0(5, X13, X14, X15, X16, X17, X10, X11, X12)
SHA512ROUND0(6, X12, X13, X14, X15, X16, X17, X10, X11)
SHA512ROUND0(7, X11, X12, X13, X14, X15, X16, X17, X10)
SHA512ROUND0(8, X10, X11, X12, X13, X14, X15, X16, X17)
SHA512ROUND0(9, X17, X10, X11, X12, X13, X14, X15, X16)
SHA512ROUND0(10, X16, X17, X10, X11, X12, X13, X14, X15)
SHA512ROUND0(11, X15, X16, X17, X10, X11, X12, X13, X14)
SHA512ROUND0(12, X14, X15, X16, X17, X10, X11, X12, X13)
SHA512ROUND0(13, X13, X14, X15, X16, X17, X10, X11, X12)
SHA512ROUND0(14, X12, X13, X14, X15, X16, X17, X10, X11)
SHA512ROUND0(15, X11, X12, X13, X14, X15, X16, X17, X10)
BEQZ X25, aligned_zbb_load
SHA512ROUND0_0_15()
JMP block
SHA512ROUND1(16, X10, X11, X12, X13, X14, X15, X16, X17)
SHA512ROUND1(17, X17, X10, X11, X12, X13, X14, X15, X16)
SHA512ROUND1(18, X16, X17, X10, X11, X12, X13, X14, X15)
SHA512ROUND1(19, X15, X16, X17, X10, X11, X12, X13, X14)
SHA512ROUND1(20, X14, X15, X16, X17, X10, X11, X12, X13)
SHA512ROUND1(21, X13, X14, X15, X16, X17, X10, X11, X12)
SHA512ROUND1(22, X12, X13, X14, X15, X16, X17, X10, X11)
SHA512ROUND1(23, X11, X12, X13, X14, X15, X16, X17, X10)
SHA512ROUND1(24, X10, X11, X12, X13, X14, X15, X16, X17)
SHA512ROUND1(25, X17, X10, X11, X12, X13, X14, X15, X16)
SHA512ROUND1(26, X16, X17, X10, X11, X12, X13, X14, X15)
SHA512ROUND1(27, X15, X16, X17, X10, X11, X12, X13, X14)
SHA512ROUND1(28, X14, X15, X16, X17, X10, X11, X12, X13)
SHA512ROUND1(29, X13, X14, X15, X16, X17, X10, X11, X12)
SHA512ROUND1(30, X12, X13, X14, X15, X16, X17, X10, X11)
SHA512ROUND1(31, X11, X12, X13, X14, X15, X16, X17, X10)
SHA512ROUND1(32, X10, X11, X12, X13, X14, X15, X16, X17)
SHA512ROUND1(33, X17, X10, X11, X12, X13, X14, X15, X16)
SHA512ROUND1(34, X16, X17, X10, X11, X12, X13, X14, X15)
SHA512ROUND1(35, X15, X16, X17, X10, X11, X12, X13, X14)
SHA512ROUND1(36, X14, X15, X16, X17, X10, X11, X12, X13)
SHA512ROUND1(37, X13, X14, X15, X16, X17, X10, X11, X12)
SHA512ROUND1(38, X12, X13, X14, X15, X16, X17, X10, X11)
SHA512ROUND1(39, X11, X12, X13, X14, X15, X16, X17, X10)
SHA512ROUND1(40, X10, X11, X12, X13, X14, X15, X16, X17)
SHA512ROUND1(41, X17, X10, X11, X12, X13, X14, X15, X16)
SHA512ROUND1(42, X16, X17, X10, X11, X12, X13, X14, X15)
SHA512ROUND1(43, X15, X16, X17, X10, X11, X12, X13, X14)
SHA512ROUND1(44, X14, X15, X16, X17, X10, X11, X12, X13)
SHA512ROUND1(45, X13, X14, X15, X16, X17, X10, X11, X12)
SHA512ROUND1(46, X12, X13, X14, X15, X16, X17, X10, X11)
SHA512ROUND1(47, X11, X12, X13, X14, X15, X16, X17, X10)
SHA512ROUND1(48, X10, X11, X12, X13, X14, X15, X16, X17)
SHA512ROUND1(49, X17, X10, X11, X12, X13, X14, X15, X16)
SHA512ROUND1(50, X16, X17, X10, X11, X12, X13, X14, X15)
SHA512ROUND1(51, X15, X16, X17, X10, X11, X12, X13, X14)
SHA512ROUND1(52, X14, X15, X16, X17, X10, X11, X12, X13)
SHA512ROUND1(53, X13, X14, X15, X16, X17, X10, X11, X12)
SHA512ROUND1(54, X12, X13, X14, X15, X16, X17, X10, X11)
SHA512ROUND1(55, X11, X12, X13, X14, X15, X16, X17, X10)
SHA512ROUND1(56, X10, X11, X12, X13, X14, X15, X16, X17)
SHA512ROUND1(57, X17, X10, X11, X12, X13, X14, X15, X16)
SHA512ROUND1(58, X16, X17, X10, X11, X12, X13, X14, X15)
SHA512ROUND1(59, X15, X16, X17, X10, X11, X12, X13, X14)
SHA512ROUND1(60, X14, X15, X16, X17, X10, X11, X12, X13)
SHA512ROUND1(61, X13, X14, X15, X16, X17, X10, X11, X12)
SHA512ROUND1(62, X12, X13, X14, X15, X16, X17, X10, X11)
SHA512ROUND1(63, X11, X12, X13, X14, X15, X16, X17, X10)
SHA512ROUND1(64, X10, X11, X12, X13, X14, X15, X16, X17)
SHA512ROUND1(65, X17, X10, X11, X12, X13, X14, X15, X16)
SHA512ROUND1(66, X16, X17, X10, X11, X12, X13, X14, X15)
SHA512ROUND1(67, X15, X16, X17, X10, X11, X12, X13, X14)
SHA512ROUND1(68, X14, X15, X16, X17, X10, X11, X12, X13)
SHA512ROUND1(69, X13, X14, X15, X16, X17, X10, X11, X12)
SHA512ROUND1(70, X12, X13, X14, X15, X16, X17, X10, X11)
SHA512ROUND1(71, X11, X12, X13, X14, X15, X16, X17, X10)
SHA512ROUND1(72, X10, X11, X12, X13, X14, X15, X16, X17)
SHA512ROUND1(73, X17, X10, X11, X12, X13, X14, X15, X16)
SHA512ROUND1(74, X16, X17, X10, X11, X12, X13, X14, X15)
SHA512ROUND1(75, X15, X16, X17, X10, X11, X12, X13, X14)
SHA512ROUND1(76, X14, X15, X16, X17, X10, X11, X12, X13)
SHA512ROUND1(77, X13, X14, X15, X16, X17, X10, X11, X12)
SHA512ROUND1(78, X12, X13, X14, X15, X16, X17, X10, X11)
SHA512ROUND1(79, X11, X12, X13, X14, X15, X16, X17, X10)
aligned_zbb_load:
SHA512ROUND0_0_15_ZBB()
block:
SHA512ROUND1_16_79()
MOV (0*8)(X20), X5
MOV (1*8)(X20), X6

View File

@@ -27,7 +27,8 @@ var (
LOONG64HasLSX = cpu.Loong64.HasLSX
LOONG64HasLASX = cpu.Loong64.HasLASX
RISCV64HasV = cpu.RISCV64.HasV
RISCV64HasV = cpu.RISCV64.HasV
RISCV64HasZbb = cpu.RISCV64.HasZbb
S390XHasAES = cpu.S390X.HasAES
S390XHasAESCBC = cpu.S390X.HasAESCBC