You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
277 lines
6.5 KiB
277 lines
6.5 KiB
#!/usr/bin/env perl |
|
|
|
# ==================================================================== |
|
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL |
|
# project. The module is, however, dual licensed under OpenSSL and |
|
# CRYPTOGAMS licenses depending on where you obtain it. For further |
|
# details see http://www.openssl.org/~appro/cryptogams/. |
|
# ==================================================================== |
|
|
|
# April 2007. |
|
# |
|
# Performance improvement over vanilla C code varies from 85% to 45% |
|
# depending on key length and benchmark. Unfortunately in this context |
|
# these are not very impressive results [for code that utilizes "wide" |
|
# 64x64=128-bit multiplication, which is not commonly available to C |
|
# programmers], at least hand-coded bn_asm.c replacement is known to |
|
# provide 30-40% better results for longest keys. Well, on a second |
|
# thought it's not very surprising, because z-CPUs are single-issue |
|
# and _strictly_ in-order execution, while bn_mul_mont is more or less |
|
# dependent on CPU ability to pipe-line instructions and have several |
|
# of them "in-flight" at the same time. I mean while other methods, |
|
# for example Karatsuba, aim to minimize amount of multiplications at |
|
# the cost of other operations increase, bn_mul_mont aim to neatly |
|
# "overlap" multiplications and the other operations [and on most |
|
# platforms even minimize the amount of the other operations, in |
|
# particular references to memory]. But it's possible to improve this |
|
# module performance by implementing dedicated squaring code-path and |
|
# possibly by unrolling loops... |
|
|
|
# January 2009. |
|
# |
|
# Reschedule to minimize/avoid Address Generation Interlock hazard, |
|
# make inner loops counter-based. |
|
|
|
# November 2010. |
|
# |
|
# Adapt for -m31 build. If kernel supports what's called "highgprs" |
|
# feature on Linux [see /proc/cpuinfo], it's possible to use 64-bit |
|
# instructions and achieve "64-bit" performance even in 31-bit legacy |
|
# application context. The feature is not specific to any particular |
|
# processor, as long as it's "z-CPU". Latter implies that the code |
|
# remains z/Architecture specific. Compatibility with 32-bit BN_ULONG |
|
# is achieved by swapping words after 64-bit loads, follow _dswap-s. |
|
# On z990 it was measured to perform 2.6-2.2 times better than |
|
# compiler-generated code, less for longer keys... |
|
|
|
$flavour = shift; |
|
|
|
if ($flavour =~ /3[12]/) { |
|
$SIZE_T=4; |
|
$g=""; |
|
} else { |
|
$SIZE_T=8; |
|
$g="g"; |
|
} |
|
|
|
while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} |
|
open STDOUT,">$output"; |
|
|
|
$stdframe=16*$SIZE_T+4*8; |
|
|
|
$mn0="%r0"; |
|
$num="%r1"; |
|
|
|
# int bn_mul_mont( |
|
$rp="%r2"; # BN_ULONG *rp, |
|
$ap="%r3"; # const BN_ULONG *ap, |
|
$bp="%r4"; # const BN_ULONG *bp, |
|
$np="%r5"; # const BN_ULONG *np, |
|
$n0="%r6"; # const BN_ULONG *n0, |
|
#$num="160(%r15)" # int num); |
|
|
|
$bi="%r2"; # zaps rp |
|
$j="%r7"; |
|
|
|
$ahi="%r8"; |
|
$alo="%r9"; |
|
$nhi="%r10"; |
|
$nlo="%r11"; |
|
$AHI="%r12"; |
|
$NHI="%r13"; |
|
$count="%r14"; |
|
$sp="%r15"; |
|
|
|
$code.=<<___; |
|
.text |
|
.globl bn_mul_mont |
|
.type bn_mul_mont,\@function |
|
bn_mul_mont: |
|
lgf $num,`$stdframe+$SIZE_T-4`($sp) # pull $num |
|
sla $num,`log($SIZE_T)/log(2)` # $num to enumerate bytes |
|
la $bp,0($num,$bp) |
|
|
|
st${g} %r2,2*$SIZE_T($sp) |
|
|
|
cghi $num,16 # |
|
lghi %r2,0 # |
|
blr %r14 # if($num<16) return 0; |
|
___ |
|
$code.=<<___ if ($flavour =~ /3[12]/); |
|
tmll $num,4 |
|
bnzr %r14 # if ($num&1) return 0; |
|
___ |
|
$code.=<<___ if ($flavour !~ /3[12]/); |
|
cghi $num,96 # |
|
bhr %r14 # if($num>96) return 0; |
|
___ |
|
$code.=<<___; |
|
stm${g} %r3,%r15,3*$SIZE_T($sp) |
|
|
|
lghi $rp,-$stdframe-8 # leave room for carry bit |
|
lcgr $j,$num # -$num |
|
lgr %r0,$sp |
|
la $rp,0($rp,$sp) |
|
la $sp,0($j,$rp) # alloca |
|
st${g} %r0,0($sp) # back chain |
|
|
|
sra $num,3 # restore $num |
|
la $bp,0($j,$bp) # restore $bp |
|
ahi $num,-1 # adjust $num for inner loop |
|
lg $n0,0($n0) # pull n0 |
|
_dswap $n0 |
|
|
|
lg $bi,0($bp) |
|
_dswap $bi |
|
lg $alo,0($ap) |
|
_dswap $alo |
|
mlgr $ahi,$bi # ap[0]*bp[0] |
|
lgr $AHI,$ahi |
|
|
|
lgr $mn0,$alo # "tp[0]"*n0 |
|
msgr $mn0,$n0 |
|
|
|
lg $nlo,0($np) # |
|
_dswap $nlo |
|
mlgr $nhi,$mn0 # np[0]*m1 |
|
algr $nlo,$alo # +="tp[0]" |
|
lghi $NHI,0 |
|
alcgr $NHI,$nhi |
|
|
|
la $j,8(%r0) # j=1 |
|
lr $count,$num |
|
|
|
.align 16 |
|
.L1st: |
|
lg $alo,0($j,$ap) |
|
_dswap $alo |
|
mlgr $ahi,$bi # ap[j]*bp[0] |
|
algr $alo,$AHI |
|
lghi $AHI,0 |
|
alcgr $AHI,$ahi |
|
|
|
lg $nlo,0($j,$np) |
|
_dswap $nlo |
|
mlgr $nhi,$mn0 # np[j]*m1 |
|
algr $nlo,$NHI |
|
lghi $NHI,0 |
|
alcgr $nhi,$NHI # +="tp[j]" |
|
algr $nlo,$alo |
|
alcgr $NHI,$nhi |
|
|
|
stg $nlo,$stdframe-8($j,$sp) # tp[j-1]= |
|
la $j,8($j) # j++ |
|
brct $count,.L1st |
|
|
|
algr $NHI,$AHI |
|
lghi $AHI,0 |
|
alcgr $AHI,$AHI # upmost overflow bit |
|
stg $NHI,$stdframe-8($j,$sp) |
|
stg $AHI,$stdframe($j,$sp) |
|
la $bp,8($bp) # bp++ |
|
|
|
.Louter: |
|
lg $bi,0($bp) # bp[i] |
|
_dswap $bi |
|
lg $alo,0($ap) |
|
_dswap $alo |
|
mlgr $ahi,$bi # ap[0]*bp[i] |
|
alg $alo,$stdframe($sp) # +=tp[0] |
|
lghi $AHI,0 |
|
alcgr $AHI,$ahi |
|
|
|
lgr $mn0,$alo |
|
msgr $mn0,$n0 # tp[0]*n0 |
|
|
|
lg $nlo,0($np) # np[0] |
|
_dswap $nlo |
|
mlgr $nhi,$mn0 # np[0]*m1 |
|
algr $nlo,$alo # +="tp[0]" |
|
lghi $NHI,0 |
|
alcgr $NHI,$nhi |
|
|
|
la $j,8(%r0) # j=1 |
|
lr $count,$num |
|
|
|
.align 16 |
|
.Linner: |
|
lg $alo,0($j,$ap) |
|
_dswap $alo |
|
mlgr $ahi,$bi # ap[j]*bp[i] |
|
algr $alo,$AHI |
|
lghi $AHI,0 |
|
alcgr $ahi,$AHI |
|
alg $alo,$stdframe($j,$sp)# +=tp[j] |
|
alcgr $AHI,$ahi |
|
|
|
lg $nlo,0($j,$np) |
|
_dswap $nlo |
|
mlgr $nhi,$mn0 # np[j]*m1 |
|
algr $nlo,$NHI |
|
lghi $NHI,0 |
|
alcgr $nhi,$NHI |
|
algr $nlo,$alo # +="tp[j]" |
|
alcgr $NHI,$nhi |
|
|
|
stg $nlo,$stdframe-8($j,$sp) # tp[j-1]= |
|
la $j,8($j) # j++ |
|
brct $count,.Linner |
|
|
|
algr $NHI,$AHI |
|
lghi $AHI,0 |
|
alcgr $AHI,$AHI |
|
alg $NHI,$stdframe($j,$sp)# accumulate previous upmost overflow bit |
|
lghi $ahi,0 |
|
alcgr $AHI,$ahi # new upmost overflow bit |
|
stg $NHI,$stdframe-8($j,$sp) |
|
stg $AHI,$stdframe($j,$sp) |
|
|
|
la $bp,8($bp) # bp++ |
|
cl${g} $bp,`$stdframe+8+4*$SIZE_T`($j,$sp) # compare to &bp[num] |
|
jne .Louter |
|
|
|
l${g} $rp,`$stdframe+8+2*$SIZE_T`($j,$sp) # reincarnate rp |
|
la $ap,$stdframe($sp) |
|
ahi $num,1 # restore $num, incidentally clears "borrow" |
|
|
|
la $j,0(%r0) |
|
lr $count,$num |
|
.Lsub: lg $alo,0($j,$ap) |
|
lg $nlo,0($j,$np) |
|
_dswap $nlo |
|
slbgr $alo,$nlo |
|
stg $alo,0($j,$rp) |
|
la $j,8($j) |
|
brct $count,.Lsub |
|
lghi $ahi,0 |
|
slbgr $AHI,$ahi # handle upmost carry |
|
|
|
ngr $ap,$AHI |
|
lghi $np,-1 |
|
xgr $np,$AHI |
|
ngr $np,$rp |
|
ogr $ap,$np # ap=borrow?tp:rp |
|
|
|
la $j,0(%r0) |
|
lgr $count,$num |
|
.Lcopy: lg $alo,0($j,$ap) # copy or in-place refresh |
|
_dswap $alo |
|
stg $j,$stdframe($j,$sp) # zap tp |
|
stg $alo,0($j,$rp) |
|
la $j,8($j) |
|
brct $count,.Lcopy |
|
|
|
la %r1,`$stdframe+8+6*$SIZE_T`($j,$sp) |
|
lm${g} %r6,%r15,0(%r1) |
|
lghi %r2,1 # signal "processed" |
|
br %r14 |
|
.size bn_mul_mont,.-bn_mul_mont |
|
.string "Montgomery Multiplication for s390x, CRYPTOGAMS by <appro\@openssl.org>" |
|
___ |
|
|
|
foreach (split("\n",$code)) { |
|
s/\`([^\`]*)\`/eval $1/ge; |
|
s/_dswap\s+(%r[0-9]+)/sprintf("rllg\t%s,%s,32",$1,$1) if($SIZE_T==4)/e; |
|
print $_,"\n"; |
|
} |
|
close STDOUT;
|
|
|