crypto: morus - remove generic and x86 implementations
MORUS was not selected as a winner in the CAESAR competition, which is not surprising since it is considered to be cryptographically broken [0]. (Note that this is not an implementation defect, but a flaw in the underlying algorithm). Since it is unlikely to be in use currently, let's remove it before we're stuck with it. [0] https://eprint.iacr.org/2019/172.pdf Reviewed-by: Ondrej Mosnacek <omosnace@redhat.com> Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
This commit is contained in:
Ard Biesheuvel
Herbert Xu
parent
f248caf9a5
commit
5cb97700be
@ -39,12 +39,6 @@ obj-$(CONFIG_CRYPTO_AEGIS128_AESNI_SSE2) += aegis128-aesni.o
|
||||
obj-$(CONFIG_CRYPTO_AEGIS128L_AESNI_SSE2) += aegis128l-aesni.o
|
||||
obj-$(CONFIG_CRYPTO_AEGIS256_AESNI_SSE2) += aegis256-aesni.o
|
||||
|
||||
obj-$(CONFIG_CRYPTO_MORUS640_GLUE) += morus640_glue.o
|
||||
obj-$(CONFIG_CRYPTO_MORUS1280_GLUE) += morus1280_glue.o
|
||||
|
||||
obj-$(CONFIG_CRYPTO_MORUS640_SSE2) += morus640-sse2.o
|
||||
obj-$(CONFIG_CRYPTO_MORUS1280_SSE2) += morus1280-sse2.o
|
||||
|
||||
obj-$(CONFIG_CRYPTO_NHPOLY1305_SSE2) += nhpoly1305-sse2.o
|
||||
obj-$(CONFIG_CRYPTO_NHPOLY1305_AVX2) += nhpoly1305-avx2.o
|
||||
|
||||
@ -62,8 +56,6 @@ endif
|
||||
ifeq ($(avx2_supported),yes)
|
||||
obj-$(CONFIG_CRYPTO_CAMELLIA_AESNI_AVX2_X86_64) += camellia-aesni-avx2.o
|
||||
obj-$(CONFIG_CRYPTO_SERPENT_AVX2_X86_64) += serpent-avx2.o
|
||||
|
||||
obj-$(CONFIG_CRYPTO_MORUS1280_AVX2) += morus1280-avx2.o
|
||||
endif
|
||||
|
||||
twofish-i586-y := twofish-i586-asm_32.o twofish_glue.o
|
||||
@ -81,9 +73,6 @@ aegis128-aesni-y := aegis128-aesni-asm.o aegis128-aesni-glue.o
|
||||
aegis128l-aesni-y := aegis128l-aesni-asm.o aegis128l-aesni-glue.o
|
||||
aegis256-aesni-y := aegis256-aesni-asm.o aegis256-aesni-glue.o
|
||||
|
||||
morus640-sse2-y := morus640-sse2-asm.o morus640-sse2-glue.o
|
||||
morus1280-sse2-y := morus1280-sse2-asm.o morus1280-sse2-glue.o
|
||||
|
||||
nhpoly1305-sse2-y := nh-sse2-x86_64.o nhpoly1305-sse2-glue.o
|
||||
|
||||
ifeq ($(avx_supported),yes)
|
||||
@ -102,8 +91,6 @@ ifeq ($(avx2_supported),yes)
|
||||
chacha-x86_64-y += chacha-avx2-x86_64.o
|
||||
serpent-avx2-y := serpent-avx2-asm_64.o serpent_avx2_glue.o
|
||||
|
||||
morus1280-avx2-y := morus1280-avx2-asm.o morus1280-avx2-glue.o
|
||||
|
||||
nhpoly1305-avx2-y := nh-avx2-x86_64.o nhpoly1305-avx2-glue.o
|
||||
endif
|
||||
|
||||
|
||||
@ -1,619 +0,0 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* AVX2 implementation of MORUS-1280
|
||||
*
|
||||
* Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
|
||||
* Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/linkage.h>
|
||||
#include <asm/frame.h>
|
||||
|
||||
#define SHUFFLE_MASK(i0, i1, i2, i3) \
|
||||
(i0 | (i1 << 2) | (i2 << 4) | (i3 << 6))
|
||||
|
||||
#define MASK1 SHUFFLE_MASK(3, 0, 1, 2)
|
||||
#define MASK2 SHUFFLE_MASK(2, 3, 0, 1)
|
||||
#define MASK3 SHUFFLE_MASK(1, 2, 3, 0)
|
||||
|
||||
#define STATE0 %ymm0
|
||||
#define STATE0_LOW %xmm0
|
||||
#define STATE1 %ymm1
|
||||
#define STATE2 %ymm2
|
||||
#define STATE3 %ymm3
|
||||
#define STATE4 %ymm4
|
||||
#define KEY %ymm5
|
||||
#define MSG %ymm5
|
||||
#define MSG_LOW %xmm5
|
||||
#define T0 %ymm6
|
||||
#define T0_LOW %xmm6
|
||||
#define T1 %ymm7
|
||||
|
||||
.section .rodata.cst32.morus1280_const, "aM", @progbits, 32
|
||||
.align 32
|
||||
.Lmorus1280_const:
|
||||
.byte 0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d
|
||||
.byte 0x15, 0x22, 0x37, 0x59, 0x90, 0xe9, 0x79, 0x62
|
||||
.byte 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1
|
||||
.byte 0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd
|
||||
|
||||
.section .rodata.cst32.morus1280_counter, "aM", @progbits, 32
|
||||
.align 32
|
||||
.Lmorus1280_counter:
|
||||
.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
|
||||
.byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
|
||||
.byte 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17
|
||||
.byte 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f
|
||||
|
||||
.text
|
||||
|
||||
.macro morus1280_round s0, s1, s2, s3, s4, b, w
|
||||
vpand \s1, \s2, T0
|
||||
vpxor T0, \s0, \s0
|
||||
vpxor \s3, \s0, \s0
|
||||
vpsllq $\b, \s0, T0
|
||||
vpsrlq $(64 - \b), \s0, \s0
|
||||
vpxor T0, \s0, \s0
|
||||
vpermq $\w, \s3, \s3
|
||||
.endm
|
||||
|
||||
/*
|
||||
* __morus1280_update: internal ABI
|
||||
* input:
|
||||
* STATE[0-4] - input state
|
||||
* MSG - message block
|
||||
* output:
|
||||
* STATE[0-4] - output state
|
||||
* changed:
|
||||
* T0
|
||||
*/
|
||||
__morus1280_update:
|
||||
morus1280_round STATE0, STATE1, STATE2, STATE3, STATE4, 13, MASK1
|
||||
vpxor MSG, STATE1, STATE1
|
||||
morus1280_round STATE1, STATE2, STATE3, STATE4, STATE0, 46, MASK2
|
||||
vpxor MSG, STATE2, STATE2
|
||||
morus1280_round STATE2, STATE3, STATE4, STATE0, STATE1, 38, MASK3
|
||||
vpxor MSG, STATE3, STATE3
|
||||
morus1280_round STATE3, STATE4, STATE0, STATE1, STATE2, 7, MASK2
|
||||
vpxor MSG, STATE4, STATE4
|
||||
morus1280_round STATE4, STATE0, STATE1, STATE2, STATE3, 4, MASK1
|
||||
ret
|
||||
ENDPROC(__morus1280_update)
|
||||
|
||||
/*
|
||||
* __morus1280_update_zero: internal ABI
|
||||
* input:
|
||||
* STATE[0-4] - input state
|
||||
* output:
|
||||
* STATE[0-4] - output state
|
||||
* changed:
|
||||
* T0
|
||||
*/
|
||||
__morus1280_update_zero:
|
||||
morus1280_round STATE0, STATE1, STATE2, STATE3, STATE4, 13, MASK1
|
||||
morus1280_round STATE1, STATE2, STATE3, STATE4, STATE0, 46, MASK2
|
||||
morus1280_round STATE2, STATE3, STATE4, STATE0, STATE1, 38, MASK3
|
||||
morus1280_round STATE3, STATE4, STATE0, STATE1, STATE2, 7, MASK2
|
||||
morus1280_round STATE4, STATE0, STATE1, STATE2, STATE3, 4, MASK1
|
||||
ret
|
||||
ENDPROC(__morus1280_update_zero)
|
||||
|
||||
/*
|
||||
* __load_partial: internal ABI
|
||||
* input:
|
||||
* %rsi - src
|
||||
* %rcx - bytes
|
||||
* output:
|
||||
* MSG - message block
|
||||
* changed:
|
||||
* %r8
|
||||
* %r9
|
||||
*/
|
||||
__load_partial:
|
||||
xor %r9d, %r9d
|
||||
vpxor MSG, MSG, MSG
|
||||
|
||||
mov %rcx, %r8
|
||||
and $0x1, %r8
|
||||
jz .Lld_partial_1
|
||||
|
||||
mov %rcx, %r8
|
||||
and $0x1E, %r8
|
||||
add %rsi, %r8
|
||||
mov (%r8), %r9b
|
||||
|
||||
.Lld_partial_1:
|
||||
mov %rcx, %r8
|
||||
and $0x2, %r8
|
||||
jz .Lld_partial_2
|
||||
|
||||
mov %rcx, %r8
|
||||
and $0x1C, %r8
|
||||
add %rsi, %r8
|
||||
shl $16, %r9
|
||||
mov (%r8), %r9w
|
||||
|
||||
.Lld_partial_2:
|
||||
mov %rcx, %r8
|
||||
and $0x4, %r8
|
||||
jz .Lld_partial_4
|
||||
|
||||
mov %rcx, %r8
|
||||
and $0x18, %r8
|
||||
add %rsi, %r8
|
||||
shl $32, %r9
|
||||
mov (%r8), %r8d
|
||||
xor %r8, %r9
|
||||
|
||||
.Lld_partial_4:
|
||||
movq %r9, MSG_LOW
|
||||
|
||||
mov %rcx, %r8
|
||||
and $0x8, %r8
|
||||
jz .Lld_partial_8
|
||||
|
||||
mov %rcx, %r8
|
||||
and $0x10, %r8
|
||||
add %rsi, %r8
|
||||
pshufd $MASK2, MSG_LOW, MSG_LOW
|
||||
pinsrq $0, (%r8), MSG_LOW
|
||||
|
||||
.Lld_partial_8:
|
||||
mov %rcx, %r8
|
||||
and $0x10, %r8
|
||||
jz .Lld_partial_16
|
||||
|
||||
vpermq $MASK2, MSG, MSG
|
||||
movdqu (%rsi), MSG_LOW
|
||||
|
||||
.Lld_partial_16:
|
||||
ret
|
||||
ENDPROC(__load_partial)
|
||||
|
||||
/*
|
||||
* __store_partial: internal ABI
|
||||
* input:
|
||||
* %rdx - dst
|
||||
* %rcx - bytes
|
||||
* output:
|
||||
* T0 - message block
|
||||
* changed:
|
||||
* %r8
|
||||
* %r9
|
||||
* %r10
|
||||
*/
|
||||
__store_partial:
|
||||
mov %rcx, %r8
|
||||
mov %rdx, %r9
|
||||
|
||||
cmp $16, %r8
|
||||
jl .Lst_partial_16
|
||||
|
||||
movdqu T0_LOW, (%r9)
|
||||
vpermq $MASK2, T0, T0
|
||||
|
||||
sub $16, %r8
|
||||
add $16, %r9
|
||||
|
||||
.Lst_partial_16:
|
||||
movq T0_LOW, %r10
|
||||
|
||||
cmp $8, %r8
|
||||
jl .Lst_partial_8
|
||||
|
||||
mov %r10, (%r9)
|
||||
pextrq $1, T0_LOW, %r10
|
||||
|
||||
sub $8, %r8
|
||||
add $8, %r9
|
||||
|
||||
.Lst_partial_8:
|
||||
cmp $4, %r8
|
||||
jl .Lst_partial_4
|
||||
|
||||
mov %r10d, (%r9)
|
||||
shr $32, %r10
|
||||
|
||||
sub $4, %r8
|
||||
add $4, %r9
|
||||
|
||||
.Lst_partial_4:
|
||||
cmp $2, %r8
|
||||
jl .Lst_partial_2
|
||||
|
||||
mov %r10w, (%r9)
|
||||
shr $16, %r10
|
||||
|
||||
sub $2, %r8
|
||||
add $2, %r9
|
||||
|
||||
.Lst_partial_2:
|
||||
cmp $1, %r8
|
||||
jl .Lst_partial_1
|
||||
|
||||
mov %r10b, (%r9)
|
||||
|
||||
.Lst_partial_1:
|
||||
ret
|
||||
ENDPROC(__store_partial)
|
||||
|
||||
/*
|
||||
* void crypto_morus1280_avx2_init(void *state, const void *key,
|
||||
* const void *iv);
|
||||
*/
|
||||
ENTRY(crypto_morus1280_avx2_init)
|
||||
FRAME_BEGIN
|
||||
|
||||
/* load IV: */
|
||||
vpxor STATE0, STATE0, STATE0
|
||||
movdqu (%rdx), STATE0_LOW
|
||||
/* load key: */
|
||||
vmovdqu (%rsi), KEY
|
||||
vmovdqa KEY, STATE1
|
||||
/* load all ones: */
|
||||
vpcmpeqd STATE2, STATE2, STATE2
|
||||
/* load all zeros: */
|
||||
vpxor STATE3, STATE3, STATE3
|
||||
/* load the constant: */
|
||||
vmovdqa .Lmorus1280_const, STATE4
|
||||
|
||||
/* update 16 times with zero: */
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
|
||||
/* xor-in the key again after updates: */
|
||||
vpxor KEY, STATE1, STATE1
|
||||
|
||||
/* store the state: */
|
||||
vmovdqu STATE0, (0 * 32)(%rdi)
|
||||
vmovdqu STATE1, (1 * 32)(%rdi)
|
||||
vmovdqu STATE2, (2 * 32)(%rdi)
|
||||
vmovdqu STATE3, (3 * 32)(%rdi)
|
||||
vmovdqu STATE4, (4 * 32)(%rdi)
|
||||
|
||||
FRAME_END
|
||||
ret
|
||||
ENDPROC(crypto_morus1280_avx2_init)
|
||||
|
||||
/*
|
||||
* void crypto_morus1280_avx2_ad(void *state, const void *data,
|
||||
* unsigned int length);
|
||||
*/
|
||||
ENTRY(crypto_morus1280_avx2_ad)
|
||||
FRAME_BEGIN
|
||||
|
||||
cmp $32, %rdx
|
||||
jb .Lad_out
|
||||
|
||||
/* load the state: */
|
||||
vmovdqu (0 * 32)(%rdi), STATE0
|
||||
vmovdqu (1 * 32)(%rdi), STATE1
|
||||
vmovdqu (2 * 32)(%rdi), STATE2
|
||||
vmovdqu (3 * 32)(%rdi), STATE3
|
||||
vmovdqu (4 * 32)(%rdi), STATE4
|
||||
|
||||
mov %rsi, %r8
|
||||
and $0x1F, %r8
|
||||
jnz .Lad_u_loop
|
||||
|
||||
.align 4
|
||||
.Lad_a_loop:
|
||||
vmovdqa (%rsi), MSG
|
||||
call __morus1280_update
|
||||
sub $32, %rdx
|
||||
add $32, %rsi
|
||||
cmp $32, %rdx
|
||||
jge .Lad_a_loop
|
||||
|
||||
jmp .Lad_cont
|
||||
.align 4
|
||||
.Lad_u_loop:
|
||||
vmovdqu (%rsi), MSG
|
||||
call __morus1280_update
|
||||
sub $32, %rdx
|
||||
add $32, %rsi
|
||||
cmp $32, %rdx
|
||||
jge .Lad_u_loop
|
||||
|
||||
.Lad_cont:
|
||||
/* store the state: */
|
||||
vmovdqu STATE0, (0 * 32)(%rdi)
|
||||
vmovdqu STATE1, (1 * 32)(%rdi)
|
||||
vmovdqu STATE2, (2 * 32)(%rdi)
|
||||
vmovdqu STATE3, (3 * 32)(%rdi)
|
||||
vmovdqu STATE4, (4 * 32)(%rdi)
|
||||
|
||||
.Lad_out:
|
||||
FRAME_END
|
||||
ret
|
||||
ENDPROC(crypto_morus1280_avx2_ad)
|
||||
|
||||
/*
|
||||
* void crypto_morus1280_avx2_enc(void *state, const void *src, void *dst,
|
||||
* unsigned int length);
|
||||
*/
|
||||
ENTRY(crypto_morus1280_avx2_enc)
|
||||
FRAME_BEGIN
|
||||
|
||||
cmp $32, %rcx
|
||||
jb .Lenc_out
|
||||
|
||||
/* load the state: */
|
||||
vmovdqu (0 * 32)(%rdi), STATE0
|
||||
vmovdqu (1 * 32)(%rdi), STATE1
|
||||
vmovdqu (2 * 32)(%rdi), STATE2
|
||||
vmovdqu (3 * 32)(%rdi), STATE3
|
||||
vmovdqu (4 * 32)(%rdi), STATE4
|
||||
|
||||
mov %rsi, %r8
|
||||
or %rdx, %r8
|
||||
and $0x1F, %r8
|
||||
jnz .Lenc_u_loop
|
||||
|
||||
.align 4
|
||||
.Lenc_a_loop:
|
||||
vmovdqa (%rsi), MSG
|
||||
vmovdqa MSG, T0
|
||||
vpxor STATE0, T0, T0
|
||||
vpermq $MASK3, STATE1, T1
|
||||
vpxor T1, T0, T0
|
||||
vpand STATE2, STATE3, T1
|
||||
vpxor T1, T0, T0
|
||||
vmovdqa T0, (%rdx)
|
||||
|
||||
call __morus1280_update
|
||||
sub $32, %rcx
|
||||
add $32, %rsi
|
||||
add $32, %rdx
|
||||
cmp $32, %rcx
|
||||
jge .Lenc_a_loop
|
||||
|
||||
jmp .Lenc_cont
|
||||
.align 4
|
||||
.Lenc_u_loop:
|
||||
vmovdqu (%rsi), MSG
|
||||
vmovdqa MSG, T0
|
||||
vpxor STATE0, T0, T0
|
||||
vpermq $MASK3, STATE1, T1
|
||||
vpxor T1, T0, T0
|
||||
vpand STATE2, STATE3, T1
|
||||
vpxor T1, T0, T0
|
||||
vmovdqu T0, (%rdx)
|
||||
|
||||
call __morus1280_update
|
||||
sub $32, %rcx
|
||||
add $32, %rsi
|
||||
add $32, %rdx
|
||||
cmp $32, %rcx
|
||||
jge .Lenc_u_loop
|
||||
|
||||
.Lenc_cont:
|
||||
/* store the state: */
|
||||
vmovdqu STATE0, (0 * 32)(%rdi)
|
||||
vmovdqu STATE1, (1 * 32)(%rdi)
|
||||
vmovdqu STATE2, (2 * 32)(%rdi)
|
||||
vmovdqu STATE3, (3 * 32)(%rdi)
|
||||
vmovdqu STATE4, (4 * 32)(%rdi)
|
||||
|
||||
.Lenc_out:
|
||||
FRAME_END
|
||||
ret
|
||||
ENDPROC(crypto_morus1280_avx2_enc)
|
||||
|
||||
/*
|
||||
* void crypto_morus1280_avx2_enc_tail(void *state, const void *src, void *dst,
|
||||
* unsigned int length);
|
||||
*/
|
||||
ENTRY(crypto_morus1280_avx2_enc_tail)
|
||||
FRAME_BEGIN
|
||||
|
||||
/* load the state: */
|
||||
vmovdqu (0 * 32)(%rdi), STATE0
|
||||
vmovdqu (1 * 32)(%rdi), STATE1
|
||||
vmovdqu (2 * 32)(%rdi), STATE2
|
||||
vmovdqu (3 * 32)(%rdi), STATE3
|
||||
vmovdqu (4 * 32)(%rdi), STATE4
|
||||
|
||||
/* encrypt message: */
|
||||
call __load_partial
|
||||
|
||||
vmovdqa MSG, T0
|
||||
vpxor STATE0, T0, T0
|
||||
vpermq $MASK3, STATE1, T1
|
||||
vpxor T1, T0, T0
|
||||
vpand STATE2, STATE3, T1
|
||||
vpxor T1, T0, T0
|
||||
|
||||
call __store_partial
|
||||
|
||||
call __morus1280_update
|
||||
|
||||
/* store the state: */
|
||||
vmovdqu STATE0, (0 * 32)(%rdi)
|
||||
vmovdqu STATE1, (1 * 32)(%rdi)
|
||||
vmovdqu STATE2, (2 * 32)(%rdi)
|
||||
vmovdqu STATE3, (3 * 32)(%rdi)
|
||||
vmovdqu STATE4, (4 * 32)(%rdi)
|
||||
|
||||
FRAME_END
|
||||
ret
|
||||
ENDPROC(crypto_morus1280_avx2_enc_tail)
|
||||
|
||||
/*
|
||||
* void crypto_morus1280_avx2_dec(void *state, const void *src, void *dst,
|
||||
* unsigned int length);
|
||||
*/
|
||||
ENTRY(crypto_morus1280_avx2_dec)
|
||||
FRAME_BEGIN
|
||||
|
||||
cmp $32, %rcx
|
||||
jb .Ldec_out
|
||||
|
||||
/* load the state: */
|
||||
vmovdqu (0 * 32)(%rdi), STATE0
|
||||
vmovdqu (1 * 32)(%rdi), STATE1
|
||||
vmovdqu (2 * 32)(%rdi), STATE2
|
||||
vmovdqu (3 * 32)(%rdi), STATE3
|
||||
vmovdqu (4 * 32)(%rdi), STATE4
|
||||
|
||||
mov %rsi, %r8
|
||||
or %rdx, %r8
|
||||
and $0x1F, %r8
|
||||
jnz .Ldec_u_loop
|
||||
|
||||
.align 4
|
||||
.Ldec_a_loop:
|
||||
vmovdqa (%rsi), MSG
|
||||
vpxor STATE0, MSG, MSG
|
||||
vpermq $MASK3, STATE1, T0
|
||||
vpxor T0, MSG, MSG
|
||||
vpand STATE2, STATE3, T0
|
||||
vpxor T0, MSG, MSG
|
||||
vmovdqa MSG, (%rdx)
|
||||
|
||||
call __morus1280_update
|
||||
sub $32, %rcx
|
||||
add $32, %rsi
|
||||
add $32, %rdx
|
||||
cmp $32, %rcx
|
||||
jge .Ldec_a_loop
|
||||
|
||||
jmp .Ldec_cont
|
||||
.align 4
|
||||
.Ldec_u_loop:
|
||||
vmovdqu (%rsi), MSG
|
||||
vpxor STATE0, MSG, MSG
|
||||
vpermq $MASK3, STATE1, T0
|
||||
vpxor T0, MSG, MSG
|
||||
vpand STATE2, STATE3, T0
|
||||
vpxor T0, MSG, MSG
|
||||
vmovdqu MSG, (%rdx)
|
||||
|
||||
call __morus1280_update
|
||||
sub $32, %rcx
|
||||
add $32, %rsi
|
||||
add $32, %rdx
|
||||
cmp $32, %rcx
|
||||
jge .Ldec_u_loop
|
||||
|
||||
.Ldec_cont:
|
||||
/* store the state: */
|
||||
vmovdqu STATE0, (0 * 32)(%rdi)
|
||||
vmovdqu STATE1, (1 * 32)(%rdi)
|
||||
vmovdqu STATE2, (2 * 32)(%rdi)
|
||||
vmovdqu STATE3, (3 * 32)(%rdi)
|
||||
vmovdqu STATE4, (4 * 32)(%rdi)
|
||||
|
||||
.Ldec_out:
|
||||
FRAME_END
|
||||
ret
|
||||
ENDPROC(crypto_morus1280_avx2_dec)
|
||||
|
||||
/*
|
||||
* void crypto_morus1280_avx2_dec_tail(void *state, const void *src, void *dst,
|
||||
* unsigned int length);
|
||||
*/
|
||||
ENTRY(crypto_morus1280_avx2_dec_tail)
|
||||
FRAME_BEGIN
|
||||
|
||||
/* load the state: */
|
||||
vmovdqu (0 * 32)(%rdi), STATE0
|
||||
vmovdqu (1 * 32)(%rdi), STATE1
|
||||
vmovdqu (2 * 32)(%rdi), STATE2
|
||||
vmovdqu (3 * 32)(%rdi), STATE3
|
||||
vmovdqu (4 * 32)(%rdi), STATE4
|
||||
|
||||
/* decrypt message: */
|
||||
call __load_partial
|
||||
|
||||
vpxor STATE0, MSG, MSG
|
||||
vpermq $MASK3, STATE1, T0
|
||||
vpxor T0, MSG, MSG
|
||||
vpand STATE2, STATE3, T0
|
||||
vpxor T0, MSG, MSG
|
||||
vmovdqa MSG, T0
|
||||
|
||||
call __store_partial
|
||||
|
||||
/* mask with byte count: */
|
||||
movq %rcx, T0_LOW
|
||||
vpbroadcastb T0_LOW, T0
|
||||
vmovdqa .Lmorus1280_counter, T1
|
||||
vpcmpgtb T1, T0, T0
|
||||
vpand T0, MSG, MSG
|
||||
|
||||
call __morus1280_update
|
||||
|
||||
/* store the state: */
|
||||
vmovdqu STATE0, (0 * 32)(%rdi)
|
||||
vmovdqu STATE1, (1 * 32)(%rdi)
|
||||
vmovdqu STATE2, (2 * 32)(%rdi)
|
||||
vmovdqu STATE3, (3 * 32)(%rdi)
|
||||
vmovdqu STATE4, (4 * 32)(%rdi)
|
||||
|
||||
FRAME_END
|
||||
ret
|
||||
ENDPROC(crypto_morus1280_avx2_dec_tail)
|
||||
|
||||
/*
|
||||
* void crypto_morus1280_avx2_final(void *state, void *tag_xor,
|
||||
* u64 assoclen, u64 cryptlen);
|
||||
*/
|
||||
ENTRY(crypto_morus1280_avx2_final)
|
||||
FRAME_BEGIN
|
||||
|
||||
/* load the state: */
|
||||
vmovdqu (0 * 32)(%rdi), STATE0
|
||||
vmovdqu (1 * 32)(%rdi), STATE1
|
||||
vmovdqu (2 * 32)(%rdi), STATE2
|
||||
vmovdqu (3 * 32)(%rdi), STATE3
|
||||
vmovdqu (4 * 32)(%rdi), STATE4
|
||||
|
||||
/* xor state[0] into state[4]: */
|
||||
vpxor STATE0, STATE4, STATE4
|
||||
|
||||
/* prepare length block: */
|
||||
vpxor MSG, MSG, MSG
|
||||
vpinsrq $0, %rdx, MSG_LOW, MSG_LOW
|
||||
vpinsrq $1, %rcx, MSG_LOW, MSG_LOW
|
||||
vpsllq $3, MSG, MSG /* multiply by 8 (to get bit count) */
|
||||
|
||||
/* update state: */
|
||||
call __morus1280_update
|
||||
call __morus1280_update
|
||||
call __morus1280_update
|
||||
call __morus1280_update
|
||||
call __morus1280_update
|
||||
call __morus1280_update
|
||||
call __morus1280_update
|
||||
call __morus1280_update
|
||||
call __morus1280_update
|
||||
call __morus1280_update
|
||||
|
||||
/* xor tag: */
|
||||
vmovdqu (%rsi), MSG
|
||||
|
||||
vpxor STATE0, MSG, MSG
|
||||
vpermq $MASK3, STATE1, T0
|
||||
vpxor T0, MSG, MSG
|
||||
vpand STATE2, STATE3, T0
|
||||
vpxor T0, MSG, MSG
|
||||
vmovdqu MSG, (%rsi)
|
||||
|
||||
FRAME_END
|
||||
ret
|
||||
ENDPROC(crypto_morus1280_avx2_final)
|
||||
@ -1,62 +0,0 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
/*
|
||||
* The MORUS-1280 Authenticated-Encryption Algorithm
|
||||
* Glue for AVX2 implementation
|
||||
*
|
||||
* Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
|
||||
* Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <crypto/internal/aead.h>
|
||||
#include <crypto/internal/simd.h>
|
||||
#include <crypto/morus1280_glue.h>
|
||||
#include <linux/module.h>
|
||||
#include <asm/fpu/api.h>
|
||||
#include <asm/cpu_device_id.h>
|
||||
|
||||
asmlinkage void crypto_morus1280_avx2_init(void *state, const void *key,
|
||||
const void *iv);
|
||||
asmlinkage void crypto_morus1280_avx2_ad(void *state, const void *data,
|
||||
unsigned int length);
|
||||
|
||||
asmlinkage void crypto_morus1280_avx2_enc(void *state, const void *src,
|
||||
void *dst, unsigned int length);
|
||||
asmlinkage void crypto_morus1280_avx2_dec(void *state, const void *src,
|
||||
void *dst, unsigned int length);
|
||||
|
||||
asmlinkage void crypto_morus1280_avx2_enc_tail(void *state, const void *src,
|
||||
void *dst, unsigned int length);
|
||||
asmlinkage void crypto_morus1280_avx2_dec_tail(void *state, const void *src,
|
||||
void *dst, unsigned int length);
|
||||
|
||||
asmlinkage void crypto_morus1280_avx2_final(void *state, void *tag_xor,
|
||||
u64 assoclen, u64 cryptlen);
|
||||
|
||||
MORUS1280_DECLARE_ALG(avx2, "morus1280-avx2", 400);
|
||||
|
||||
static struct simd_aead_alg *simd_alg;
|
||||
|
||||
static int __init crypto_morus1280_avx2_module_init(void)
|
||||
{
|
||||
if (!boot_cpu_has(X86_FEATURE_AVX2) ||
|
||||
!boot_cpu_has(X86_FEATURE_OSXSAVE) ||
|
||||
!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
|
||||
return -ENODEV;
|
||||
|
||||
return simd_register_aeads_compat(&crypto_morus1280_avx2_alg, 1,
|
||||
&simd_alg);
|
||||
}
|
||||
|
||||
static void __exit crypto_morus1280_avx2_module_exit(void)
|
||||
{
|
||||
simd_unregister_aeads(&crypto_morus1280_avx2_alg, 1, &simd_alg);
|
||||
}
|
||||
|
||||
module_init(crypto_morus1280_avx2_module_init);
|
||||
module_exit(crypto_morus1280_avx2_module_exit);
|
||||
|
||||
MODULE_LICENSE("GPL");
|
||||
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
|
||||
MODULE_DESCRIPTION("MORUS-1280 AEAD algorithm -- AVX2 implementation");
|
||||
MODULE_ALIAS_CRYPTO("morus1280");
|
||||
MODULE_ALIAS_CRYPTO("morus1280-avx2");
|
||||
@ -1,893 +0,0 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* SSE2 implementation of MORUS-1280
|
||||
*
|
||||
* Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
|
||||
* Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/linkage.h>
|
||||
#include <asm/frame.h>
|
||||
|
||||
#define SHUFFLE_MASK(i0, i1, i2, i3) \
|
||||
(i0 | (i1 << 2) | (i2 << 4) | (i3 << 6))
|
||||
|
||||
#define MASK2 SHUFFLE_MASK(2, 3, 0, 1)
|
||||
|
||||
#define STATE0_LO %xmm0
|
||||
#define STATE0_HI %xmm1
|
||||
#define STATE1_LO %xmm2
|
||||
#define STATE1_HI %xmm3
|
||||
#define STATE2_LO %xmm4
|
||||
#define STATE2_HI %xmm5
|
||||
#define STATE3_LO %xmm6
|
||||
#define STATE3_HI %xmm7
|
||||
#define STATE4_LO %xmm8
|
||||
#define STATE4_HI %xmm9
|
||||
#define KEY_LO %xmm10
|
||||
#define KEY_HI %xmm11
|
||||
#define MSG_LO %xmm10
|
||||
#define MSG_HI %xmm11
|
||||
#define T0_LO %xmm12
|
||||
#define T0_HI %xmm13
|
||||
#define T1_LO %xmm14
|
||||
#define T1_HI %xmm15
|
||||
|
||||
.section .rodata.cst16.morus640_const, "aM", @progbits, 16
|
||||
.align 16
|
||||
.Lmorus640_const_0:
|
||||
.byte 0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d
|
||||
.byte 0x15, 0x22, 0x37, 0x59, 0x90, 0xe9, 0x79, 0x62
|
||||
.Lmorus640_const_1:
|
||||
.byte 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1
|
||||
.byte 0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd
|
||||
|
||||
.section .rodata.cst16.morus640_counter, "aM", @progbits, 16
|
||||
.align 16
|
||||
.Lmorus640_counter_0:
|
||||
.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
|
||||
.byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
|
||||
.Lmorus640_counter_1:
|
||||
.byte 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17
|
||||
.byte 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f
|
||||
|
||||
.text
|
||||
|
||||
.macro rol1 hi, lo
|
||||
/*
|
||||
* HI_1 | HI_0 || LO_1 | LO_0
|
||||
* ==>
|
||||
* HI_0 | HI_1 || LO_1 | LO_0
|
||||
* ==>
|
||||
* HI_0 | LO_1 || LO_0 | HI_1
|
||||
*/
|
||||
pshufd $MASK2, \hi, \hi
|
||||
movdqa \hi, T0_LO
|
||||
punpcklqdq \lo, T0_LO
|
||||
punpckhqdq \hi, \lo
|
||||
movdqa \lo, \hi
|
||||
movdqa T0_LO, \lo
|
||||
.endm
|
||||
|
||||
.macro rol2 hi, lo
|
||||
movdqa \lo, T0_LO
|
||||
movdqa \hi, \lo
|
||||
movdqa T0_LO, \hi
|
||||
.endm
|
||||
|
||||
.macro rol3 hi, lo
|
||||
/*
|
||||
* HI_1 | HI_0 || LO_1 | LO_0
|
||||
* ==>
|
||||
* HI_0 | HI_1 || LO_1 | LO_0
|
||||
* ==>
|
||||
* LO_0 | HI_1 || HI_0 | LO_1
|
||||
*/
|
||||
pshufd $MASK2, \hi, \hi
|
||||
movdqa \lo, T0_LO
|
||||
punpckhqdq \hi, T0_LO
|
||||
punpcklqdq \lo, \hi
|
||||
movdqa T0_LO, \lo
|
||||
.endm
|
||||
|
||||
.macro morus1280_round s0_l, s0_h, s1_l, s1_h, s2_l, s2_h, s3_l, s3_h, s4_l, s4_h, b, w
|
||||
movdqa \s1_l, T0_LO
|
||||
pand \s2_l, T0_LO
|
||||
pxor T0_LO, \s0_l
|
||||
|
||||
movdqa \s1_h, T0_LO
|
||||
pand \s2_h, T0_LO
|
||||
pxor T0_LO, \s0_h
|
||||
|
||||
pxor \s3_l, \s0_l
|
||||
pxor \s3_h, \s0_h
|
||||
|
||||
movdqa \s0_l, T0_LO
|
||||
psllq $\b, T0_LO
|
||||
psrlq $(64 - \b), \s0_l
|
||||
pxor T0_LO, \s0_l
|
||||
|
||||
movdqa \s0_h, T0_LO
|
||||
psllq $\b, T0_LO
|
||||
psrlq $(64 - \b), \s0_h
|
||||
pxor T0_LO, \s0_h
|
||||
|
||||
\w \s3_h, \s3_l
|
||||
.endm
|
||||
|
||||
/*
|
||||
* __morus1280_update: internal ABI
|
||||
* input:
|
||||
* STATE[0-4] - input state
|
||||
* MSG - message block
|
||||
* output:
|
||||
* STATE[0-4] - output state
|
||||
* changed:
|
||||
* T0
|
||||
*/
|
||||
__morus1280_update:
|
||||
morus1280_round \
|
||||
STATE0_LO, STATE0_HI, \
|
||||
STATE1_LO, STATE1_HI, \
|
||||
STATE2_LO, STATE2_HI, \
|
||||
STATE3_LO, STATE3_HI, \
|
||||
STATE4_LO, STATE4_HI, \
|
||||
13, rol1
|
||||
pxor MSG_LO, STATE1_LO
|
||||
pxor MSG_HI, STATE1_HI
|
||||
morus1280_round \
|
||||
STATE1_LO, STATE1_HI, \
|
||||
STATE2_LO, STATE2_HI, \
|
||||
STATE3_LO, STATE3_HI, \
|
||||
STATE4_LO, STATE4_HI, \
|
||||
STATE0_LO, STATE0_HI, \
|
||||
46, rol2
|
||||
pxor MSG_LO, STATE2_LO
|
||||
pxor MSG_HI, STATE2_HI
|
||||
morus1280_round \
|
||||
STATE2_LO, STATE2_HI, \
|
||||
STATE3_LO, STATE3_HI, \
|
||||
STATE4_LO, STATE4_HI, \
|
||||
STATE0_LO, STATE0_HI, \
|
||||
STATE1_LO, STATE1_HI, \
|
||||
38, rol3
|
||||
pxor MSG_LO, STATE3_LO
|
||||
pxor MSG_HI, STATE3_HI
|
||||
morus1280_round \
|
||||
STATE3_LO, STATE3_HI, \
|
||||
STATE4_LO, STATE4_HI, \
|
||||
STATE0_LO, STATE0_HI, \
|
||||
STATE1_LO, STATE1_HI, \
|
||||
STATE2_LO, STATE2_HI, \
|
||||
7, rol2
|
||||
pxor MSG_LO, STATE4_LO
|
||||
pxor MSG_HI, STATE4_HI
|
||||
morus1280_round \
|
||||
STATE4_LO, STATE4_HI, \
|
||||
STATE0_LO, STATE0_HI, \
|
||||
STATE1_LO, STATE1_HI, \
|
||||
STATE2_LO, STATE2_HI, \
|
||||
STATE3_LO, STATE3_HI, \
|
||||
4, rol1
|
||||
ret
|
||||
ENDPROC(__morus1280_update)
|
||||
|
||||
/*
|
||||
* __morus1280_update_zero: internal ABI
|
||||
* input:
|
||||
* STATE[0-4] - input state
|
||||
* output:
|
||||
* STATE[0-4] - output state
|
||||
* changed:
|
||||
* T0
|
||||
*/
|
||||
__morus1280_update_zero:
|
||||
morus1280_round \
|
||||
STATE0_LO, STATE0_HI, \
|
||||
STATE1_LO, STATE1_HI, \
|
||||
STATE2_LO, STATE2_HI, \
|
||||
STATE3_LO, STATE3_HI, \
|
||||
STATE4_LO, STATE4_HI, \
|
||||
13, rol1
|
||||
morus1280_round \
|
||||
STATE1_LO, STATE1_HI, \
|
||||
STATE2_LO, STATE2_HI, \
|
||||
STATE3_LO, STATE3_HI, \
|
||||
STATE4_LO, STATE4_HI, \
|
||||
STATE0_LO, STATE0_HI, \
|
||||
46, rol2
|
||||
morus1280_round \
|
||||
STATE2_LO, STATE2_HI, \
|
||||
STATE3_LO, STATE3_HI, \
|
||||
STATE4_LO, STATE4_HI, \
|
||||
STATE0_LO, STATE0_HI, \
|
||||
STATE1_LO, STATE1_HI, \
|
||||
38, rol3
|
||||
morus1280_round \
|
||||
STATE3_LO, STATE3_HI, \
|
||||
STATE4_LO, STATE4_HI, \
|
||||
STATE0_LO, STATE0_HI, \
|
||||
STATE1_LO, STATE1_HI, \
|
||||
STATE2_LO, STATE2_HI, \
|
||||
7, rol2
|
||||
morus1280_round \
|
||||
STATE4_LO, STATE4_HI, \
|
||||
STATE0_LO, STATE0_HI, \
|
||||
STATE1_LO, STATE1_HI, \
|
||||
STATE2_LO, STATE2_HI, \
|
||||
STATE3_LO, STATE3_HI, \
|
||||
4, rol1
|
||||
ret
|
||||
ENDPROC(__morus1280_update_zero)
|
||||
|
||||
/*
|
||||
* __load_partial: internal ABI
|
||||
* input:
|
||||
* %rsi - src
|
||||
* %rcx - bytes
|
||||
* output:
|
||||
* MSG - message block
|
||||
* changed:
|
||||
* %r8
|
||||
* %r9
|
||||
*/
|
||||
__load_partial:
|
||||
xor %r9d, %r9d
|
||||
pxor MSG_LO, MSG_LO
|
||||
pxor MSG_HI, MSG_HI
|
||||
|
||||
mov %rcx, %r8
|
||||
and $0x1, %r8
|
||||
jz .Lld_partial_1
|
||||
|
||||
mov %rcx, %r8
|
||||
and $0x1E, %r8
|
||||
add %rsi, %r8
|
||||
mov (%r8), %r9b
|
||||
|
||||
.Lld_partial_1:
|
||||
mov %rcx, %r8
|
||||
and $0x2, %r8
|
||||
jz .Lld_partial_2
|
||||
|
||||
mov %rcx, %r8
|
||||
and $0x1C, %r8
|
||||
add %rsi, %r8
|
||||
shl $16, %r9
|
||||
mov (%r8), %r9w
|
||||
|
||||
.Lld_partial_2:
|
||||
mov %rcx, %r8
|
||||
and $0x4, %r8
|
||||
jz .Lld_partial_4
|
||||
|
||||
mov %rcx, %r8
|
||||
and $0x18, %r8
|
||||
add %rsi, %r8
|
||||
shl $32, %r9
|
||||
mov (%r8), %r8d
|
||||
xor %r8, %r9
|
||||
|
||||
.Lld_partial_4:
|
||||
movq %r9, MSG_LO
|
||||
|
||||
mov %rcx, %r8
|
||||
and $0x8, %r8
|
||||
jz .Lld_partial_8
|
||||
|
||||
mov %rcx, %r8
|
||||
and $0x10, %r8
|
||||
add %rsi, %r8
|
||||
pslldq $8, MSG_LO
|
||||
movq (%r8), T0_LO
|
||||
pxor T0_LO, MSG_LO
|
||||
|
||||
.Lld_partial_8:
|
||||
mov %rcx, %r8
|
||||
and $0x10, %r8
|
||||
jz .Lld_partial_16
|
||||
|
||||
movdqa MSG_LO, MSG_HI
|
||||
movdqu (%rsi), MSG_LO
|
||||
|
||||
.Lld_partial_16:
|
||||
ret
|
||||
ENDPROC(__load_partial)
|
||||
|
||||
/*
|
||||
* __store_partial: internal ABI
|
||||
* input:
|
||||
* %rdx - dst
|
||||
* %rcx - bytes
|
||||
* output:
|
||||
* T0 - message block
|
||||
* changed:
|
||||
* %r8
|
||||
* %r9
|
||||
* %r10
|
||||
*/
|
||||
__store_partial:
|
||||
mov %rcx, %r8
|
||||
mov %rdx, %r9
|
||||
|
||||
cmp $16, %r8
|
||||
jl .Lst_partial_16
|
||||
|
||||
movdqu T0_LO, (%r9)
|
||||
movdqa T0_HI, T0_LO
|
||||
|
||||
sub $16, %r8
|
||||
add $16, %r9
|
||||
|
||||
.Lst_partial_16:
|
||||
movq T0_LO, %r10
|
||||
|
||||
cmp $8, %r8
|
||||
jl .Lst_partial_8
|
||||
|
||||
mov %r10, (%r9)
|
||||
psrldq $8, T0_LO
|
||||
movq T0_LO, %r10
|
||||
|
||||
sub $8, %r8
|
||||
add $8, %r9
|
||||
|
||||
.Lst_partial_8:
|
||||
cmp $4, %r8
|
||||
jl .Lst_partial_4
|
||||
|
||||
mov %r10d, (%r9)
|
||||
shr $32, %r10
|
||||
|
||||
sub $4, %r8
|
||||
add $4, %r9
|
||||
|
||||
.Lst_partial_4:
|
||||
cmp $2, %r8
|
||||
jl .Lst_partial_2
|
||||
|
||||
mov %r10w, (%r9)
|
||||
shr $16, %r10
|
||||
|
||||
sub $2, %r8
|
||||
add $2, %r9
|
||||
|
||||
.Lst_partial_2:
|
||||
cmp $1, %r8
|
||||
jl .Lst_partial_1
|
||||
|
||||
mov %r10b, (%r9)
|
||||
|
||||
.Lst_partial_1:
|
||||
ret
|
||||
ENDPROC(__store_partial)
|
||||
|
||||
/*
|
||||
* void crypto_morus1280_sse2_init(void *state, const void *key,
|
||||
* const void *iv);
|
||||
*/
|
||||
ENTRY(crypto_morus1280_sse2_init)
|
||||
FRAME_BEGIN
|
||||
|
||||
/* load IV: */
|
||||
pxor STATE0_HI, STATE0_HI
|
||||
movdqu (%rdx), STATE0_LO
|
||||
/* load key: */
|
||||
movdqu 0(%rsi), KEY_LO
|
||||
movdqu 16(%rsi), KEY_HI
|
||||
movdqa KEY_LO, STATE1_LO
|
||||
movdqa KEY_HI, STATE1_HI
|
||||
/* load all ones: */
|
||||
pcmpeqd STATE2_LO, STATE2_LO
|
||||
pcmpeqd STATE2_HI, STATE2_HI
|
||||
/* load all zeros: */
|
||||
pxor STATE3_LO, STATE3_LO
|
||||
pxor STATE3_HI, STATE3_HI
|
||||
/* load the constant: */
|
||||
movdqa .Lmorus640_const_0, STATE4_LO
|
||||
movdqa .Lmorus640_const_1, STATE4_HI
|
||||
|
||||
/* update 16 times with zero: */
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
call __morus1280_update_zero
|
||||
|
||||
/* xor-in the key again after updates: */
|
||||
pxor KEY_LO, STATE1_LO
|
||||
pxor KEY_HI, STATE1_HI
|
||||
|
||||
/* store the state: */
|
||||
movdqu STATE0_LO, (0 * 16)(%rdi)
|
||||
movdqu STATE0_HI, (1 * 16)(%rdi)
|
||||
movdqu STATE1_LO, (2 * 16)(%rdi)
|
||||
movdqu STATE1_HI, (3 * 16)(%rdi)
|
||||
movdqu STATE2_LO, (4 * 16)(%rdi)
|
||||
movdqu STATE2_HI, (5 * 16)(%rdi)
|
||||
movdqu STATE3_LO, (6 * 16)(%rdi)
|
||||
movdqu STATE3_HI, (7 * 16)(%rdi)
|
||||
movdqu STATE4_LO, (8 * 16)(%rdi)
|
||||
movdqu STATE4_HI, (9 * 16)(%rdi)
|
||||
|
||||
FRAME_END
|
||||
ret
|
||||
ENDPROC(crypto_morus1280_sse2_init)
|
||||
|
||||
/*
|
||||
* void crypto_morus1280_sse2_ad(void *state, const void *data,
|
||||
* unsigned int length);
|
||||
*/
|
||||
ENTRY(crypto_morus1280_sse2_ad)
|
||||
FRAME_BEGIN
|
||||
|
||||
cmp $32, %rdx
|
||||
jb .Lad_out
|
||||
|
||||
/* load the state: */
|
||||
movdqu (0 * 16)(%rdi), STATE0_LO
|
||||
movdqu (1 * 16)(%rdi), STATE0_HI
|
||||
movdqu (2 * 16)(%rdi), STATE1_LO
|
||||
movdqu (3 * 16)(%rdi), STATE1_HI
|
||||
movdqu (4 * 16)(%rdi), STATE2_LO
|
||||
movdqu (5 * 16)(%rdi), STATE2_HI
|
||||
movdqu (6 * 16)(%rdi), STATE3_LO
|
||||
movdqu (7 * 16)(%rdi), STATE3_HI
|
||||
movdqu (8 * 16)(%rdi), STATE4_LO
|
||||
movdqu (9 * 16)(%rdi), STATE4_HI
|
||||
|
||||
mov %rsi, %r8
|
||||
and $0xF, %r8
|
||||
jnz .Lad_u_loop
|
||||
|
||||
.align 4
|
||||
.Lad_a_loop:
|
||||
movdqa 0(%rsi), MSG_LO
|
||||
movdqa 16(%rsi), MSG_HI
|
||||
call __morus1280_update
|
||||
sub $32, %rdx
|
||||
add $32, %rsi
|
||||
cmp $32, %rdx
|
||||
jge .Lad_a_loop
|
||||
|
||||
jmp .Lad_cont
|
||||
.align 4
|
||||
.Lad_u_loop:
|
||||
movdqu 0(%rsi), MSG_LO
|
||||
movdqu 16(%rsi), MSG_HI
|
||||
call __morus1280_update
|
||||
sub $32, %rdx
|
||||
add $32, %rsi
|
||||
cmp $32, %rdx
|
||||
jge .Lad_u_loop
|
||||
|
||||
.Lad_cont:
|
||||
/* store the state: */
|
||||
movdqu STATE0_LO, (0 * 16)(%rdi)
|
||||
movdqu STATE0_HI, (1 * 16)(%rdi)
|
||||
movdqu STATE1_LO, (2 * 16)(%rdi)
|
||||
movdqu STATE1_HI, (3 * 16)(%rdi)
|
||||
movdqu STATE2_LO, (4 * 16)(%rdi)
|
||||
movdqu STATE2_HI, (5 * 16)(%rdi)
|
||||
movdqu STATE3_LO, (6 * 16)(%rdi)
|
||||
movdqu STATE3_HI, (7 * 16)(%rdi)
|
||||
movdqu STATE4_LO, (8 * 16)(%rdi)
|
||||
movdqu STATE4_HI, (9 * 16)(%rdi)
|
||||
|
||||
.Lad_out:
|
||||
FRAME_END
|
||||
ret
|
||||
ENDPROC(crypto_morus1280_sse2_ad)
|
||||
|
||||
/*
|
||||
* void crypto_morus1280_sse2_enc(void *state, const void *src, void *dst,
|
||||
* unsigned int length);
|
||||
*/
|
||||
ENTRY(crypto_morus1280_sse2_enc)
|
||||
FRAME_BEGIN
|
||||
|
||||
cmp $32, %rcx
|
||||
jb .Lenc_out
|
||||
|
||||
/* load the state: */
|
||||
movdqu (0 * 16)(%rdi), STATE0_LO
|
||||
movdqu (1 * 16)(%rdi), STATE0_HI
|
||||
movdqu (2 * 16)(%rdi), STATE1_LO
|
||||
movdqu (3 * 16)(%rdi), STATE1_HI
|
||||
movdqu (4 * 16)(%rdi), STATE2_LO
|
||||
movdqu (5 * 16)(%rdi), STATE2_HI
|
||||
movdqu (6 * 16)(%rdi), STATE3_LO
|
||||
movdqu (7 * 16)(%rdi), STATE3_HI
|
||||
movdqu (8 * 16)(%rdi), STATE4_LO
|
||||
movdqu (9 * 16)(%rdi), STATE4_HI
|
||||
|
||||
mov %rsi, %r8
|
||||
or %rdx, %r8
|
||||
and $0xF, %r8
|
||||
jnz .Lenc_u_loop
|
||||
|
||||
.align 4
|
||||
.Lenc_a_loop:
|
||||
movdqa 0(%rsi), MSG_LO
|
||||
movdqa 16(%rsi), MSG_HI
|
||||
movdqa STATE1_LO, T1_LO
|
||||
movdqa STATE1_HI, T1_HI
|
||||
rol3 T1_HI, T1_LO
|
||||
movdqa MSG_LO, T0_LO
|
||||
movdqa MSG_HI, T0_HI
|
||||
pxor T1_LO, T0_LO
|
||||
pxor T1_HI, T0_HI
|
||||
pxor STATE0_LO, T0_LO
|
||||
pxor STATE0_HI, T0_HI
|
||||
movdqa STATE2_LO, T1_LO
|
||||
movdqa STATE2_HI, T1_HI
|
||||
pand STATE3_LO, T1_LO
|
||||
pand STATE3_HI, T1_HI
|
||||
pxor T1_LO, T0_LO
|
||||
pxor T1_HI, T0_HI
|
||||
movdqa T0_LO, 0(%rdx)
|
||||
movdqa T0_HI, 16(%rdx)
|
||||
|
||||
call __morus1280_update
|
||||
sub $32, %rcx
|
||||
add $32, %rsi
|
||||
add $32, %rdx
|
||||
cmp $32, %rcx
|
||||
jge .Lenc_a_loop
|
||||
|
||||
jmp .Lenc_cont
|
||||
.align 4
|
||||
.Lenc_u_loop:
|
||||
movdqu 0(%rsi), MSG_LO
|
||||
movdqu 16(%rsi), MSG_HI
|
||||
movdqa STATE1_LO, T1_LO
|
||||
movdqa STATE1_HI, T1_HI
|
||||
rol3 T1_HI, T1_LO
|
||||
movdqa MSG_LO, T0_LO
|
||||
movdqa MSG_HI, T0_HI
|
||||
pxor T1_LO, T0_LO
|
||||
pxor T1_HI, T0_HI
|
||||
pxor STATE0_LO, T0_LO
|
||||
pxor STATE0_HI, T0_HI
|
||||
movdqa STATE2_LO, T1_LO
|
||||
movdqa STATE2_HI, T1_HI
|
||||
pand STATE3_LO, T1_LO
|
||||
pand STATE3_HI, T1_HI
|
||||
pxor T1_LO, T0_LO
|
||||
pxor T1_HI, T0_HI
|
||||
movdqu T0_LO, 0(%rdx)
|
||||
movdqu T0_HI, 16(%rdx)
|
||||
|
||||
call __morus1280_update
|
||||
sub $32, %rcx
|
||||
add $32, %rsi
|
||||
add $32, %rdx
|
||||
cmp $32, %rcx
|
||||
jge .Lenc_u_loop
|
||||
|
||||
.Lenc_cont:
|
||||
/* store the state: */
|
||||
movdqu STATE0_LO, (0 * 16)(%rdi)
|
||||
movdqu STATE0_HI, (1 * 16)(%rdi)
|
||||
movdqu STATE1_LO, (2 * 16)(%rdi)
|
||||
movdqu STATE1_HI, (3 * 16)(%rdi)
|
||||
movdqu STATE2_LO, (4 * 16)(%rdi)
|
||||
movdqu STATE2_HI, (5 * 16)(%rdi)
|
||||
movdqu STATE3_LO, (6 * 16)(%rdi)
|
||||
movdqu STATE3_HI, (7 * 16)(%rdi)
|
||||
movdqu STATE4_LO, (8 * 16)(%rdi)
|
||||
movdqu STATE4_HI, (9 * 16)(%rdi)
|
||||
|
||||
.Lenc_out:
|
||||
FRAME_END
|
||||
ret
|
||||
ENDPROC(crypto_morus1280_sse2_enc)
|
||||
|
||||
/*
|
||||
* void crypto_morus1280_sse2_enc_tail(void *state, const void *src, void *dst,
|
||||
* unsigned int length);
|
||||
*/
|
||||
ENTRY(crypto_morus1280_sse2_enc_tail)
|
||||
FRAME_BEGIN
|
||||
|
||||
/* load the state: */
|
||||
movdqu (0 * 16)(%rdi), STATE0_LO
|
||||
movdqu (1 * 16)(%rdi), STATE0_HI
|
||||
movdqu (2 * 16)(%rdi), STATE1_LO
|
||||
movdqu (3 * 16)(%rdi), STATE1_HI
|
||||
movdqu (4 * 16)(%rdi), STATE2_LO
|
||||
movdqu (5 * 16)(%rdi), STATE2_HI
|
||||
movdqu (6 * 16)(%rdi), STATE3_LO
|
||||
movdqu (7 * 16)(%rdi), STATE3_HI
|
||||
movdqu (8 * 16)(%rdi), STATE4_LO
|
||||
movdqu (9 * 16)(%rdi), STATE4_HI
|
||||
|
||||
/* encrypt message: */
|
||||
call __load_partial
|
||||
|
||||
movdqa STATE1_LO, T1_LO
|
||||
movdqa STATE1_HI, T1_HI
|
||||
rol3 T1_HI, T1_LO
|
||||
movdqa MSG_LO, T0_LO
|
||||
movdqa MSG_HI, T0_HI
|
||||
pxor T1_LO, T0_LO
|
||||
pxor T1_HI, T0_HI
|
||||
pxor STATE0_LO, T0_LO
|
||||
pxor STATE0_HI, T0_HI
|
||||
movdqa STATE2_LO, T1_LO
|
||||
movdqa STATE2_HI, T1_HI
|
||||
pand STATE3_LO, T1_LO
|
||||
pand STATE3_HI, T1_HI
|
||||
pxor T1_LO, T0_LO
|
||||
pxor T1_HI, T0_HI
|
||||
|
||||
call __store_partial
|
||||
|
||||
call __morus1280_update
|
||||
|
||||
/* store the state: */
|
||||
movdqu STATE0_LO, (0 * 16)(%rdi)
|
||||
movdqu STATE0_HI, (1 * 16)(%rdi)
|
||||
movdqu STATE1_LO, (2 * 16)(%rdi)
|
||||
movdqu STATE1_HI, (3 * 16)(%rdi)
|
||||
movdqu STATE2_LO, (4 * 16)(%rdi)
|
||||
movdqu STATE2_HI, (5 * 16)(%rdi)
|
||||
movdqu STATE3_LO, (6 * 16)(%rdi)
|
||||
movdqu STATE3_HI, (7 * 16)(%rdi)
|
||||
movdqu STATE4_LO, (8 * 16)(%rdi)
|
||||
movdqu STATE4_HI, (9 * 16)(%rdi)
|
||||
|
||||
FRAME_END
|
||||
ret
|
||||
ENDPROC(crypto_morus1280_sse2_enc_tail)
|
||||
|
||||
/*
|
||||
* void crypto_morus1280_sse2_dec(void *state, const void *src, void *dst,
|
||||
* unsigned int length);
|
||||
*/
|
||||
ENTRY(crypto_morus1280_sse2_dec)
|
||||
FRAME_BEGIN
|
||||
|
||||
cmp $32, %rcx
|
||||
jb .Ldec_out
|
||||
|
||||
/* load the state: */
|
||||
movdqu (0 * 16)(%rdi), STATE0_LO
|
||||
movdqu (1 * 16)(%rdi), STATE0_HI
|
||||
movdqu (2 * 16)(%rdi), STATE1_LO
|
||||
movdqu (3 * 16)(%rdi), STATE1_HI
|
||||
movdqu (4 * 16)(%rdi), STATE2_LO
|
||||
movdqu (5 * 16)(%rdi), STATE2_HI
|
||||
movdqu (6 * 16)(%rdi), STATE3_LO
|
||||
movdqu (7 * 16)(%rdi), STATE3_HI
|
||||
movdqu (8 * 16)(%rdi), STATE4_LO
|
||||
movdqu (9 * 16)(%rdi), STATE4_HI
|
||||
|
||||
mov %rsi, %r8
|
||||
or %rdx, %r8
|
||||
and $0xF, %r8
|
||||
jnz .Ldec_u_loop
|
||||
|
||||
.align 4
|
||||
.Ldec_a_loop:
|
||||
movdqa 0(%rsi), MSG_LO
|
||||
movdqa 16(%rsi), MSG_HI
|
||||
pxor STATE0_LO, MSG_LO
|
||||
pxor STATE0_HI, MSG_HI
|
||||
movdqa STATE1_LO, T1_LO
|
||||
movdqa STATE1_HI, T1_HI
|
||||
rol3 T1_HI, T1_LO
|
||||
pxor T1_LO, MSG_LO
|
||||
pxor T1_HI, MSG_HI
|
||||
movdqa STATE2_LO, T1_LO
|
||||
movdqa STATE2_HI, T1_HI
|
||||
pand STATE3_LO, T1_LO
|
||||
pand STATE3_HI, T1_HI
|
||||
pxor T1_LO, MSG_LO
|
||||
pxor T1_HI, MSG_HI
|
||||
movdqa MSG_LO, 0(%rdx)
|
||||
movdqa MSG_HI, 16(%rdx)
|
||||
|
||||
call __morus1280_update
|
||||
sub $32, %rcx
|
||||
add $32, %rsi
|
||||
add $32, %rdx
|
||||
cmp $32, %rcx
|
||||
jge .Ldec_a_loop
|
||||
|
||||
jmp .Ldec_cont
|
||||
.align 4
|
||||
.Ldec_u_loop:
|
||||
movdqu 0(%rsi), MSG_LO
|
||||
movdqu 16(%rsi), MSG_HI
|
||||
pxor STATE0_LO, MSG_LO
|
||||
pxor STATE0_HI, MSG_HI
|
||||
movdqa STATE1_LO, T1_LO
|
||||
movdqa STATE1_HI, T1_HI
|
||||
rol3 T1_HI, T1_LO
|
||||
pxor T1_LO, MSG_LO
|
||||
pxor T1_HI, MSG_HI
|
||||
movdqa STATE2_LO, T1_LO
|
||||
movdqa STATE2_HI, T1_HI
|
||||
pand STATE3_LO, T1_LO
|
||||
pand STATE3_HI, T1_HI
|
||||
pxor T1_LO, MSG_LO
|
||||
pxor T1_HI, MSG_HI
|
||||
movdqu MSG_LO, 0(%rdx)
|
||||
movdqu MSG_HI, 16(%rdx)
|
||||
|
||||
call __morus1280_update
|
||||
sub $32, %rcx
|
||||
add $32, %rsi
|
||||
add $32, %rdx
|
||||
cmp $32, %rcx
|
||||
jge .Ldec_u_loop
|
||||
|
||||
.Ldec_cont:
|
||||
/* store the state: */
|
||||
movdqu STATE0_LO, (0 * 16)(%rdi)
|
||||
movdqu STATE0_HI, (1 * 16)(%rdi)
|
||||
movdqu STATE1_LO, (2 * 16)(%rdi)
|
||||
movdqu STATE1_HI, (3 * 16)(%rdi)
|
||||
movdqu STATE2_LO, (4 * 16)(%rdi)
|
||||
movdqu STATE2_HI, (5 * 16)(%rdi)
|
||||
movdqu STATE3_LO, (6 * 16)(%rdi)
|
||||
movdqu STATE3_HI, (7 * 16)(%rdi)
|
||||
movdqu STATE4_LO, (8 * 16)(%rdi)
|
||||
movdqu STATE4_HI, (9 * 16)(%rdi)
|
||||
|
||||
.Ldec_out:
|
||||
FRAME_END
|
||||
ret
|
||||
ENDPROC(crypto_morus1280_sse2_dec)
|
||||
|
||||
/*
|
||||
* void crypto_morus1280_sse2_dec_tail(void *state, const void *src, void *dst,
|
||||
* unsigned int length);
|
||||
*/
|
||||
ENTRY(crypto_morus1280_sse2_dec_tail)
|
||||
FRAME_BEGIN
|
||||
|
||||
/* load the state: */
|
||||
movdqu (0 * 16)(%rdi), STATE0_LO
|
||||
movdqu (1 * 16)(%rdi), STATE0_HI
|
||||
movdqu (2 * 16)(%rdi), STATE1_LO
|
||||
movdqu (3 * 16)(%rdi), STATE1_HI
|
||||
movdqu (4 * 16)(%rdi), STATE2_LO
|
||||
movdqu (5 * 16)(%rdi), STATE2_HI
|
||||
movdqu (6 * 16)(%rdi), STATE3_LO
|
||||
movdqu (7 * 16)(%rdi), STATE3_HI
|
||||
movdqu (8 * 16)(%rdi), STATE4_LO
|
||||
movdqu (9 * 16)(%rdi), STATE4_HI
|
||||
|
||||
/* decrypt message: */
|
||||
call __load_partial
|
||||
|
||||
pxor STATE0_LO, MSG_LO
|
||||
pxor STATE0_HI, MSG_HI
|
||||
movdqa STATE1_LO, T1_LO
|
||||
movdqa STATE1_HI, T1_HI
|
||||
rol3 T1_HI, T1_LO
|
||||
pxor T1_LO, MSG_LO
|
||||
pxor T1_HI, MSG_HI
|
||||
movdqa STATE2_LO, T1_LO
|
||||
movdqa STATE2_HI, T1_HI
|
||||
pand STATE3_LO, T1_LO
|
||||
pand STATE3_HI, T1_HI
|
||||
pxor T1_LO, MSG_LO
|
||||
pxor T1_HI, MSG_HI
|
||||
movdqa MSG_LO, T0_LO
|
||||
movdqa MSG_HI, T0_HI
|
||||
|
||||
call __store_partial
|
||||
|
||||
/* mask with byte count: */
|
||||
movq %rcx, T0_LO
|
||||
punpcklbw T0_LO, T0_LO
|
||||
punpcklbw T0_LO, T0_LO
|
||||
punpcklbw T0_LO, T0_LO
|
||||
punpcklbw T0_LO, T0_LO
|
||||
movdqa T0_LO, T0_HI
|
||||
movdqa .Lmorus640_counter_0, T1_LO
|
||||
movdqa .Lmorus640_counter_1, T1_HI
|
||||
pcmpgtb T1_LO, T0_LO
|
||||
pcmpgtb T1_HI, T0_HI
|
||||
pand T0_LO, MSG_LO
|
||||
pand T0_HI, MSG_HI
|
||||
|
||||
call __morus1280_update
|
||||
|
||||
/* store the state: */
|
||||
movdqu STATE0_LO, (0 * 16)(%rdi)
|
||||
movdqu STATE0_HI, (1 * 16)(%rdi)
|
||||
movdqu STATE1_LO, (2 * 16)(%rdi)
|
||||
movdqu STATE1_HI, (3 * 16)(%rdi)
|
||||
movdqu STATE2_LO, (4 * 16)(%rdi)
|
||||
movdqu STATE2_HI, (5 * 16)(%rdi)
|
||||
movdqu STATE3_LO, (6 * 16)(%rdi)
|
||||
movdqu STATE3_HI, (7 * 16)(%rdi)
|
||||
movdqu STATE4_LO, (8 * 16)(%rdi)
|
||||
movdqu STATE4_HI, (9 * 16)(%rdi)
|
||||
|
||||
FRAME_END
|
||||
ret
|
||||
ENDPROC(crypto_morus1280_sse2_dec_tail)
|
||||
|
||||
/*
|
||||
* void crypto_morus1280_sse2_final(void *state, void *tag_xor,
|
||||
* u64 assoclen, u64 cryptlen);
|
||||
*/
|
||||
ENTRY(crypto_morus1280_sse2_final)
|
||||
FRAME_BEGIN
|
||||
|
||||
/* load the state: */
|
||||
movdqu (0 * 16)(%rdi), STATE0_LO
|
||||
movdqu (1 * 16)(%rdi), STATE0_HI
|
||||
movdqu (2 * 16)(%rdi), STATE1_LO
|
||||
movdqu (3 * 16)(%rdi), STATE1_HI
|
||||
movdqu (4 * 16)(%rdi), STATE2_LO
|
||||
movdqu (5 * 16)(%rdi), STATE2_HI
|
||||
movdqu (6 * 16)(%rdi), STATE3_LO
|
||||
movdqu (7 * 16)(%rdi), STATE3_HI
|
||||
movdqu (8 * 16)(%rdi), STATE4_LO
|
||||
movdqu (9 * 16)(%rdi), STATE4_HI
|
||||
|
||||
/* xor state[0] into state[4]: */
|
||||
pxor STATE0_LO, STATE4_LO
|
||||
pxor STATE0_HI, STATE4_HI
|
||||
|
||||
/* prepare length block: */
|
||||
movq %rdx, MSG_LO
|
||||
movq %rcx, T0_LO
|
||||
pslldq $8, T0_LO
|
||||
pxor T0_LO, MSG_LO
|
||||
psllq $3, MSG_LO /* multiply by 8 (to get bit count) */
|
||||
pxor MSG_HI, MSG_HI
|
||||
|
||||
/* update state: */
|
||||
call __morus1280_update
|
||||
call __morus1280_update
|
||||
call __morus1280_update
|
||||
call __morus1280_update
|
||||
call __morus1280_update
|
||||
call __morus1280_update
|
||||
call __morus1280_update
|
||||
call __morus1280_update
|
||||
call __morus1280_update
|
||||
call __morus1280_update
|
||||
|
||||
/* xor tag: */
|
||||
movdqu 0(%rsi), MSG_LO
|
||||
movdqu 16(%rsi), MSG_HI
|
||||
|
||||
pxor STATE0_LO, MSG_LO
|
||||
pxor STATE0_HI, MSG_HI
|
||||
movdqa STATE1_LO, T0_LO
|
||||
movdqa STATE1_HI, T0_HI
|
||||
rol3 T0_HI, T0_LO
|
||||
pxor T0_LO, MSG_LO
|
||||
pxor T0_HI, MSG_HI
|
||||
movdqa STATE2_LO, T0_LO
|
||||
movdqa STATE2_HI, T0_HI
|
||||
pand STATE3_LO, T0_LO
|
||||
pand STATE3_HI, T0_HI
|
||||
pxor T0_LO, MSG_LO
|
||||
pxor T0_HI, MSG_HI
|
||||
|
||||
movdqu MSG_LO, 0(%rsi)
|
||||
movdqu MSG_HI, 16(%rsi)
|
||||
|
||||
FRAME_END
|
||||
ret
|
||||
ENDPROC(crypto_morus1280_sse2_final)
|
||||
@ -1,61 +0,0 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
/*
|
||||
* The MORUS-1280 Authenticated-Encryption Algorithm
|
||||
* Glue for SSE2 implementation
|
||||
*
|
||||
* Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
|
||||
* Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <crypto/internal/aead.h>
|
||||
#include <crypto/internal/simd.h>
|
||||
#include <crypto/morus1280_glue.h>
|
||||
#include <linux/module.h>
|
||||
#include <asm/fpu/api.h>
|
||||
#include <asm/cpu_device_id.h>
|
||||
|
||||
asmlinkage void crypto_morus1280_sse2_init(void *state, const void *key,
|
||||
const void *iv);
|
||||
asmlinkage void crypto_morus1280_sse2_ad(void *state, const void *data,
|
||||
unsigned int length);
|
||||
|
||||
asmlinkage void crypto_morus1280_sse2_enc(void *state, const void *src,
|
||||
void *dst, unsigned int length);
|
||||
asmlinkage void crypto_morus1280_sse2_dec(void *state, const void *src,
|
||||
void *dst, unsigned int length);
|
||||
|
||||
asmlinkage void crypto_morus1280_sse2_enc_tail(void *state, const void *src,
|
||||
void *dst, unsigned int length);
|
||||
asmlinkage void crypto_morus1280_sse2_dec_tail(void *state, const void *src,
|
||||
void *dst, unsigned int length);
|
||||
|
||||
asmlinkage void crypto_morus1280_sse2_final(void *state, void *tag_xor,
|
||||
u64 assoclen, u64 cryptlen);
|
||||
|
||||
MORUS1280_DECLARE_ALG(sse2, "morus1280-sse2", 350);
|
||||
|
||||
static struct simd_aead_alg *simd_alg;
|
||||
|
||||
static int __init crypto_morus1280_sse2_module_init(void)
|
||||
{
|
||||
if (!boot_cpu_has(X86_FEATURE_XMM2) ||
|
||||
!cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL))
|
||||
return -ENODEV;
|
||||
|
||||
return simd_register_aeads_compat(&crypto_morus1280_sse2_alg, 1,
|
||||
&simd_alg);
|
||||
}
|
||||
|
||||
static void __exit crypto_morus1280_sse2_module_exit(void)
|
||||
{
|
||||
simd_unregister_aeads(&crypto_morus1280_sse2_alg, 1, &simd_alg);
|
||||
}
|
||||
|
||||
module_init(crypto_morus1280_sse2_module_init);
|
||||
module_exit(crypto_morus1280_sse2_module_exit);
|
||||
|
||||
MODULE_LICENSE("GPL");
|
||||
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
|
||||
MODULE_DESCRIPTION("MORUS-1280 AEAD algorithm -- SSE2 implementation");
|
||||
MODULE_ALIAS_CRYPTO("morus1280");
|
||||
MODULE_ALIAS_CRYPTO("morus1280-sse2");
|
||||
@ -1,205 +0,0 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
/*
|
||||
* The MORUS-1280 Authenticated-Encryption Algorithm
|
||||
* Common x86 SIMD glue skeleton
|
||||
*
|
||||
* Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
|
||||
* Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <crypto/internal/aead.h>
|
||||
#include <crypto/internal/skcipher.h>
|
||||
#include <crypto/morus1280_glue.h>
|
||||
#include <crypto/scatterwalk.h>
|
||||
#include <linux/err.h>
|
||||
#include <linux/init.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/scatterlist.h>
|
||||
#include <asm/fpu/api.h>
|
||||
|
||||
struct morus1280_state {
|
||||
struct morus1280_block s[MORUS_STATE_BLOCKS];
|
||||
};
|
||||
|
||||
struct morus1280_ops {
|
||||
int (*skcipher_walk_init)(struct skcipher_walk *walk,
|
||||
struct aead_request *req, bool atomic);
|
||||
|
||||
void (*crypt_blocks)(void *state, const void *src, void *dst,
|
||||
unsigned int length);
|
||||
void (*crypt_tail)(void *state, const void *src, void *dst,
|
||||
unsigned int length);
|
||||
};
|
||||
|
||||
static void crypto_morus1280_glue_process_ad(
|
||||
struct morus1280_state *state,
|
||||
const struct morus1280_glue_ops *ops,
|
||||
struct scatterlist *sg_src, unsigned int assoclen)
|
||||
{
|
||||
struct scatter_walk walk;
|
||||
struct morus1280_block buf;
|
||||
unsigned int pos = 0;
|
||||
|
||||
scatterwalk_start(&walk, sg_src);
|
||||
while (assoclen != 0) {
|
||||
unsigned int size = scatterwalk_clamp(&walk, assoclen);
|
||||
unsigned int left = size;
|
||||
void *mapped = scatterwalk_map(&walk);
|
||||
const u8 *src = (const u8 *)mapped;
|
||||
|
||||
if (pos + size >= MORUS1280_BLOCK_SIZE) {
|
||||
if (pos > 0) {
|
||||
unsigned int fill = MORUS1280_BLOCK_SIZE - pos;
|
||||
memcpy(buf.bytes + pos, src, fill);
|
||||
ops->ad(state, buf.bytes, MORUS1280_BLOCK_SIZE);
|
||||
pos = 0;
|
||||
left -= fill;
|
||||
src += fill;
|
||||
}
|
||||
|
||||
ops->ad(state, src, left);
|
||||
src += left & ~(MORUS1280_BLOCK_SIZE - 1);
|
||||
left &= MORUS1280_BLOCK_SIZE - 1;
|
||||
}
|
||||
|
||||
memcpy(buf.bytes + pos, src, left);
|
||||
|
||||
pos += left;
|
||||
assoclen -= size;
|
||||
scatterwalk_unmap(mapped);
|
||||
scatterwalk_advance(&walk, size);
|
||||
scatterwalk_done(&walk, 0, assoclen);
|
||||
}
|
||||
|
||||
if (pos > 0) {
|
||||
memset(buf.bytes + pos, 0, MORUS1280_BLOCK_SIZE - pos);
|
||||
ops->ad(state, buf.bytes, MORUS1280_BLOCK_SIZE);
|
||||
}
|
||||
}
|
||||
|
||||
static void crypto_morus1280_glue_process_crypt(struct morus1280_state *state,
|
||||
struct morus1280_ops ops,
|
||||
struct skcipher_walk *walk)
|
||||
{
|
||||
while (walk->nbytes >= MORUS1280_BLOCK_SIZE) {
|
||||
ops.crypt_blocks(state, walk->src.virt.addr,
|
||||
walk->dst.virt.addr,
|
||||
round_down(walk->nbytes,
|
||||
MORUS1280_BLOCK_SIZE));
|
||||
skcipher_walk_done(walk, walk->nbytes % MORUS1280_BLOCK_SIZE);
|
||||
}
|
||||
|
||||
if (walk->nbytes) {
|
||||
ops.crypt_tail(state, walk->src.virt.addr, walk->dst.virt.addr,
|
||||
walk->nbytes);
|
||||
skcipher_walk_done(walk, 0);
|
||||
}
|
||||
}
|
||||
|
||||
int crypto_morus1280_glue_setkey(struct crypto_aead *aead, const u8 *key,
|
||||
unsigned int keylen)
|
||||
{
|
||||
struct morus1280_ctx *ctx = crypto_aead_ctx(aead);
|
||||
|
||||
if (keylen == MORUS1280_BLOCK_SIZE) {
|
||||
memcpy(ctx->key.bytes, key, MORUS1280_BLOCK_SIZE);
|
||||
} else if (keylen == MORUS1280_BLOCK_SIZE / 2) {
|
||||
memcpy(ctx->key.bytes, key, keylen);
|
||||
memcpy(ctx->key.bytes + keylen, key, keylen);
|
||||
} else {
|
||||
crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(crypto_morus1280_glue_setkey);
|
||||
|
||||
int crypto_morus1280_glue_setauthsize(struct crypto_aead *tfm,
|
||||
unsigned int authsize)
|
||||
{
|
||||
return (authsize <= MORUS_MAX_AUTH_SIZE) ? 0 : -EINVAL;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(crypto_morus1280_glue_setauthsize);
|
||||
|
||||
static void crypto_morus1280_glue_crypt(struct aead_request *req,
|
||||
struct morus1280_ops ops,
|
||||
unsigned int cryptlen,
|
||||
struct morus1280_block *tag_xor)
|
||||
{
|
||||
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
|
||||
struct morus1280_ctx *ctx = crypto_aead_ctx(tfm);
|
||||
struct morus1280_state state;
|
||||
struct skcipher_walk walk;
|
||||
|
||||
ops.skcipher_walk_init(&walk, req, true);
|
||||
|
||||
kernel_fpu_begin();
|
||||
|
||||
ctx->ops->init(&state, &ctx->key, req->iv);
|
||||
crypto_morus1280_glue_process_ad(&state, ctx->ops, req->src, req->assoclen);
|
||||
crypto_morus1280_glue_process_crypt(&state, ops, &walk);
|
||||
ctx->ops->final(&state, tag_xor, req->assoclen, cryptlen);
|
||||
|
||||
kernel_fpu_end();
|
||||
}
|
||||
|
||||
int crypto_morus1280_glue_encrypt(struct aead_request *req)
|
||||
{
|
||||
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
|
||||
struct morus1280_ctx *ctx = crypto_aead_ctx(tfm);
|
||||
struct morus1280_ops OPS = {
|
||||
.skcipher_walk_init = skcipher_walk_aead_encrypt,
|
||||
.crypt_blocks = ctx->ops->enc,
|
||||
.crypt_tail = ctx->ops->enc_tail,
|
||||
};
|
||||
|
||||
struct morus1280_block tag = {};
|
||||
unsigned int authsize = crypto_aead_authsize(tfm);
|
||||
unsigned int cryptlen = req->cryptlen;
|
||||
|
||||
crypto_morus1280_glue_crypt(req, OPS, cryptlen, &tag);
|
||||
|
||||
scatterwalk_map_and_copy(tag.bytes, req->dst,
|
||||
req->assoclen + cryptlen, authsize, 1);
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(crypto_morus1280_glue_encrypt);
|
||||
|
||||
int crypto_morus1280_glue_decrypt(struct aead_request *req)
|
||||
{
|
||||
static const u8 zeros[MORUS1280_BLOCK_SIZE] = {};
|
||||
|
||||
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
|
||||
struct morus1280_ctx *ctx = crypto_aead_ctx(tfm);
|
||||
struct morus1280_ops OPS = {
|
||||
.skcipher_walk_init = skcipher_walk_aead_decrypt,
|
||||
.crypt_blocks = ctx->ops->dec,
|
||||
.crypt_tail = ctx->ops->dec_tail,
|
||||
};
|
||||
|
||||
struct morus1280_block tag;
|
||||
unsigned int authsize = crypto_aead_authsize(tfm);
|
||||
unsigned int cryptlen = req->cryptlen - authsize;
|
||||
|
||||
scatterwalk_map_and_copy(tag.bytes, req->src,
|
||||
req->assoclen + cryptlen, authsize, 0);
|
||||
|
||||
crypto_morus1280_glue_crypt(req, OPS, cryptlen, &tag);
|
||||
|
||||
return crypto_memneq(tag.bytes, zeros, authsize) ? -EBADMSG : 0;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(crypto_morus1280_glue_decrypt);
|
||||
|
||||
void crypto_morus1280_glue_init_ops(struct crypto_aead *aead,
|
||||
const struct morus1280_glue_ops *ops)
|
||||
{
|
||||
struct morus1280_ctx *ctx = crypto_aead_ctx(aead);
|
||||
ctx->ops = ops;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(crypto_morus1280_glue_init_ops);
|
||||
|
||||
MODULE_LICENSE("GPL");
|
||||
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
|
||||
MODULE_DESCRIPTION("MORUS-1280 AEAD mode -- glue for x86 optimizations");
|
||||
@ -1,612 +0,0 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* SSE2 implementation of MORUS-640
|
||||
*
|
||||
* Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
|
||||
* Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/linkage.h>
|
||||
#include <asm/frame.h>
|
||||
|
||||
#define SHUFFLE_MASK(i0, i1, i2, i3) \
|
||||
(i0 | (i1 << 2) | (i2 << 4) | (i3 << 6))
|
||||
|
||||
#define MASK1 SHUFFLE_MASK(3, 0, 1, 2)
|
||||
#define MASK2 SHUFFLE_MASK(2, 3, 0, 1)
|
||||
#define MASK3 SHUFFLE_MASK(1, 2, 3, 0)
|
||||
|
||||
#define STATE0 %xmm0
|
||||
#define STATE1 %xmm1
|
||||
#define STATE2 %xmm2
|
||||
#define STATE3 %xmm3
|
||||
#define STATE4 %xmm4
|
||||
#define KEY %xmm5
|
||||
#define MSG %xmm5
|
||||
#define T0 %xmm6
|
||||
#define T1 %xmm7
|
||||
|
||||
.section .rodata.cst16.morus640_const, "aM", @progbits, 32
|
||||
.align 16
|
||||
.Lmorus640_const_0:
|
||||
.byte 0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d
|
||||
.byte 0x15, 0x22, 0x37, 0x59, 0x90, 0xe9, 0x79, 0x62
|
||||
.Lmorus640_const_1:
|
||||
.byte 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1
|
||||
.byte 0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd
|
||||
|
||||
.section .rodata.cst16.morus640_counter, "aM", @progbits, 16
|
||||
.align 16
|
||||
.Lmorus640_counter:
|
||||
.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
|
||||
.byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
|
||||
|
||||
.text
|
||||
|
||||
.macro morus640_round s0, s1, s2, s3, s4, b, w
|
||||
movdqa \s1, T0
|
||||
pand \s2, T0
|
||||
pxor T0, \s0
|
||||
pxor \s3, \s0
|
||||
movdqa \s0, T0
|
||||
pslld $\b, T0
|
||||
psrld $(32 - \b), \s0
|
||||
pxor T0, \s0
|
||||
pshufd $\w, \s3, \s3
|
||||
.endm
|
||||
|
||||
/*
|
||||
* __morus640_update: internal ABI
|
||||
* input:
|
||||
* STATE[0-4] - input state
|
||||
* MSG - message block
|
||||
* output:
|
||||
* STATE[0-4] - output state
|
||||
* changed:
|
||||
* T0
|
||||
*/
|
||||
__morus640_update:
|
||||
morus640_round STATE0, STATE1, STATE2, STATE3, STATE4, 5, MASK1
|
||||
pxor MSG, STATE1
|
||||
morus640_round STATE1, STATE2, STATE3, STATE4, STATE0, 31, MASK2
|
||||
pxor MSG, STATE2
|
||||
morus640_round STATE2, STATE3, STATE4, STATE0, STATE1, 7, MASK3
|
||||
pxor MSG, STATE3
|
||||
morus640_round STATE3, STATE4, STATE0, STATE1, STATE2, 22, MASK2
|
||||
pxor MSG, STATE4
|
||||
morus640_round STATE4, STATE0, STATE1, STATE2, STATE3, 13, MASK1
|
||||
ret
|
||||
ENDPROC(__morus640_update)
|
||||
|
||||
|
||||
/*
|
||||
* __morus640_update_zero: internal ABI
|
||||
* input:
|
||||
* STATE[0-4] - input state
|
||||
* output:
|
||||
* STATE[0-4] - output state
|
||||
* changed:
|
||||
* T0
|
||||
*/
|
||||
__morus640_update_zero:
|
||||
morus640_round STATE0, STATE1, STATE2, STATE3, STATE4, 5, MASK1
|
||||
morus640_round STATE1, STATE2, STATE3, STATE4, STATE0, 31, MASK2
|
||||
morus640_round STATE2, STATE3, STATE4, STATE0, STATE1, 7, MASK3
|
||||
morus640_round STATE3, STATE4, STATE0, STATE1, STATE2, 22, MASK2
|
||||
morus640_round STATE4, STATE0, STATE1, STATE2, STATE3, 13, MASK1
|
||||
ret
|
||||
ENDPROC(__morus640_update_zero)
|
||||
|
||||
/*
|
||||
* __load_partial: internal ABI
|
||||
* input:
|
||||
* %rsi - src
|
||||
* %rcx - bytes
|
||||
* output:
|
||||
* MSG - message block
|
||||
* changed:
|
||||
* T0
|
||||
* %r8
|
||||
* %r9
|
||||
*/
|
||||
__load_partial:
|
||||
xor %r9d, %r9d
|
||||
pxor MSG, MSG
|
||||
|
||||
mov %rcx, %r8
|
||||
and $0x1, %r8
|
||||
jz .Lld_partial_1
|
||||
|
||||
mov %rcx, %r8
|
||||
and $0x1E, %r8
|
||||
add %rsi, %r8
|
||||
mov (%r8), %r9b
|
||||
|
||||
.Lld_partial_1:
|
||||
mov %rcx, %r8
|
||||
and $0x2, %r8
|
||||
jz .Lld_partial_2
|
||||
|
||||
mov %rcx, %r8
|
||||
and $0x1C, %r8
|
||||
add %rsi, %r8
|
||||
shl $16, %r9
|
||||
mov (%r8), %r9w
|
||||
|
||||
.Lld_partial_2:
|
||||
mov %rcx, %r8
|
||||
and $0x4, %r8
|
||||
jz .Lld_partial_4
|
||||
|
||||
mov %rcx, %r8
|
||||
and $0x18, %r8
|
||||
add %rsi, %r8
|
||||
shl $32, %r9
|
||||
mov (%r8), %r8d
|
||||
xor %r8, %r9
|
||||
|
||||
.Lld_partial_4:
|
||||
movq %r9, MSG
|
||||
|
||||
mov %rcx, %r8
|
||||
and $0x8, %r8
|
||||
jz .Lld_partial_8
|
||||
|
||||
mov %rcx, %r8
|
||||
and $0x10, %r8
|
||||
add %rsi, %r8
|
||||
pslldq $8, MSG
|
||||
movq (%r8), T0
|
||||
pxor T0, MSG
|
||||
|
||||
.Lld_partial_8:
|
||||
ret
|
||||
ENDPROC(__load_partial)
|
||||
|
||||
/*
|
||||
* __store_partial: internal ABI
|
||||
* input:
|
||||
* %rdx - dst
|
||||
* %rcx - bytes
|
||||
* output:
|
||||
* T0 - message block
|
||||
* changed:
|
||||
* %r8
|
||||
* %r9
|
||||
* %r10
|
||||
*/
|
||||
__store_partial:
|
||||
mov %rcx, %r8
|
||||
mov %rdx, %r9
|
||||
|
||||
movq T0, %r10
|
||||
|
||||
cmp $8, %r8
|
||||
jl .Lst_partial_8
|
||||
|
||||
mov %r10, (%r9)
|
||||
psrldq $8, T0
|
||||
movq T0, %r10
|
||||
|
||||
sub $8, %r8
|
||||
add $8, %r9
|
||||
|
||||
.Lst_partial_8:
|
||||
cmp $4, %r8
|
||||
jl .Lst_partial_4
|
||||
|
||||
mov %r10d, (%r9)
|
||||
shr $32, %r10
|
||||
|
||||
sub $4, %r8
|
||||
add $4, %r9
|
||||
|
||||
.Lst_partial_4:
|
||||
cmp $2, %r8
|
||||
jl .Lst_partial_2
|
||||
|
||||
mov %r10w, (%r9)
|
||||
shr $16, %r10
|
||||
|
||||
sub $2, %r8
|
||||
add $2, %r9
|
||||
|
||||
.Lst_partial_2:
|
||||
cmp $1, %r8
|
||||
jl .Lst_partial_1
|
||||
|
||||
mov %r10b, (%r9)
|
||||
|
||||
.Lst_partial_1:
|
||||
ret
|
||||
ENDPROC(__store_partial)
|
||||
|
||||
/*
|
||||
* void crypto_morus640_sse2_init(void *state, const void *key, const void *iv);
|
||||
*/
|
||||
ENTRY(crypto_morus640_sse2_init)
|
||||
FRAME_BEGIN
|
||||
|
||||
/* load IV: */
|
||||
movdqu (%rdx), STATE0
|
||||
/* load key: */
|
||||
movdqu (%rsi), KEY
|
||||
movdqa KEY, STATE1
|
||||
/* load all ones: */
|
||||
pcmpeqd STATE2, STATE2
|
||||
/* load the constants: */
|
||||
movdqa .Lmorus640_const_0, STATE3
|
||||
movdqa .Lmorus640_const_1, STATE4
|
||||
|
||||
/* update 16 times with zero: */
|
||||
call __morus640_update_zero
|
||||
call __morus640_update_zero
|
||||
call __morus640_update_zero
|
||||
call __morus640_update_zero
|
||||
call __morus640_update_zero
|
||||
call __morus640_update_zero
|
||||
call __morus640_update_zero
|
||||
call __morus640_update_zero
|
||||
call __morus640_update_zero
|
||||
call __morus640_update_zero
|
||||
call __morus640_update_zero
|
||||
call __morus640_update_zero
|
||||
call __morus640_update_zero
|
||||
call __morus640_update_zero
|
||||
call __morus640_update_zero
|
||||
call __morus640_update_zero
|
||||
|
||||
/* xor-in the key again after updates: */
|
||||
pxor KEY, STATE1
|
||||
|
||||
/* store the state: */
|
||||
movdqu STATE0, (0 * 16)(%rdi)
|
||||
movdqu STATE1, (1 * 16)(%rdi)
|
||||
movdqu STATE2, (2 * 16)(%rdi)
|
||||
movdqu STATE3, (3 * 16)(%rdi)
|
||||
movdqu STATE4, (4 * 16)(%rdi)
|
||||
|
||||
FRAME_END
|
||||
ret
|
||||
ENDPROC(crypto_morus640_sse2_init)
|
||||
|
||||
/*
|
||||
* void crypto_morus640_sse2_ad(void *state, const void *data,
|
||||
* unsigned int length);
|
||||
*/
|
||||
ENTRY(crypto_morus640_sse2_ad)
|
||||
FRAME_BEGIN
|
||||
|
||||
cmp $16, %rdx
|
||||
jb .Lad_out
|
||||
|
||||
/* load the state: */
|
||||
movdqu (0 * 16)(%rdi), STATE0
|
||||
movdqu (1 * 16)(%rdi), STATE1
|
||||
movdqu (2 * 16)(%rdi), STATE2
|
||||
movdqu (3 * 16)(%rdi), STATE3
|
||||
movdqu (4 * 16)(%rdi), STATE4
|
||||
|
||||
mov %rsi, %r8
|
||||
and $0xF, %r8
|
||||
jnz .Lad_u_loop
|
||||
|
||||
.align 4
|
||||
.Lad_a_loop:
|
||||
movdqa (%rsi), MSG
|
||||
call __morus640_update
|
||||
sub $16, %rdx
|
||||
add $16, %rsi
|
||||
cmp $16, %rdx
|
||||
jge .Lad_a_loop
|
||||
|
||||
jmp .Lad_cont
|
||||
.align 4
|
||||
.Lad_u_loop:
|
||||
movdqu (%rsi), MSG
|
||||
call __morus640_update
|
||||
sub $16, %rdx
|
||||
add $16, %rsi
|
||||
cmp $16, %rdx
|
||||
jge .Lad_u_loop
|
||||
|
||||
.Lad_cont:
|
||||
/* store the state: */
|
||||
movdqu STATE0, (0 * 16)(%rdi)
|
||||
movdqu STATE1, (1 * 16)(%rdi)
|
||||
movdqu STATE2, (2 * 16)(%rdi)
|
||||
movdqu STATE3, (3 * 16)(%rdi)
|
||||
movdqu STATE4, (4 * 16)(%rdi)
|
||||
|
||||
.Lad_out:
|
||||
FRAME_END
|
||||
ret
|
||||
ENDPROC(crypto_morus640_sse2_ad)
|
||||
|
||||
/*
|
||||
* void crypto_morus640_sse2_enc(void *state, const void *src, void *dst,
|
||||
* unsigned int length);
|
||||
*/
|
||||
ENTRY(crypto_morus640_sse2_enc)
|
||||
FRAME_BEGIN
|
||||
|
||||
cmp $16, %rcx
|
||||
jb .Lenc_out
|
||||
|
||||
/* load the state: */
|
||||
movdqu (0 * 16)(%rdi), STATE0
|
||||
movdqu (1 * 16)(%rdi), STATE1
|
||||
movdqu (2 * 16)(%rdi), STATE2
|
||||
movdqu (3 * 16)(%rdi), STATE3
|
||||
movdqu (4 * 16)(%rdi), STATE4
|
||||
|
||||
mov %rsi, %r8
|
||||
or %rdx, %r8
|
||||
and $0xF, %r8
|
||||
jnz .Lenc_u_loop
|
||||
|
||||
.align 4
|
||||
.Lenc_a_loop:
|
||||
movdqa (%rsi), MSG
|
||||
movdqa MSG, T0
|
||||
pxor STATE0, T0
|
||||
pshufd $MASK3, STATE1, T1
|
||||
pxor T1, T0
|
||||
movdqa STATE2, T1
|
||||
pand STATE3, T1
|
||||
pxor T1, T0
|
||||
movdqa T0, (%rdx)
|
||||
|
||||
call __morus640_update
|
||||
sub $16, %rcx
|
||||
add $16, %rsi
|
||||
add $16, %rdx
|
||||
cmp $16, %rcx
|
||||
jge .Lenc_a_loop
|
||||
|
||||
jmp .Lenc_cont
|
||||
.align 4
|
||||
.Lenc_u_loop:
|
||||
movdqu (%rsi), MSG
|
||||
movdqa MSG, T0
|
||||
pxor STATE0, T0
|
||||
pshufd $MASK3, STATE1, T1
|
||||
pxor T1, T0
|
||||
movdqa STATE2, T1
|
||||
pand STATE3, T1
|
||||
pxor T1, T0
|
||||
movdqu T0, (%rdx)
|
||||
|
||||
call __morus640_update
|
||||
sub $16, %rcx
|
||||
add $16, %rsi
|
||||
add $16, %rdx
|
||||
cmp $16, %rcx
|
||||
jge .Lenc_u_loop
|
||||
|
||||
.Lenc_cont:
|
||||
/* store the state: */
|
||||
movdqu STATE0, (0 * 16)(%rdi)
|
||||
movdqu STATE1, (1 * 16)(%rdi)
|
||||
movdqu STATE2, (2 * 16)(%rdi)
|
||||
movdqu STATE3, (3 * 16)(%rdi)
|
||||
movdqu STATE4, (4 * 16)(%rdi)
|
||||
|
||||
.Lenc_out:
|
||||
FRAME_END
|
||||
ret
|
||||
ENDPROC(crypto_morus640_sse2_enc)
|
||||
|
||||
/*
|
||||
* void crypto_morus640_sse2_enc_tail(void *state, const void *src, void *dst,
|
||||
* unsigned int length);
|
||||
*/
|
||||
ENTRY(crypto_morus640_sse2_enc_tail)
|
||||
FRAME_BEGIN
|
||||
|
||||
/* load the state: */
|
||||
movdqu (0 * 16)(%rdi), STATE0
|
||||
movdqu (1 * 16)(%rdi), STATE1
|
||||
movdqu (2 * 16)(%rdi), STATE2
|
||||
movdqu (3 * 16)(%rdi), STATE3
|
||||
movdqu (4 * 16)(%rdi), STATE4
|
||||
|
||||
/* encrypt message: */
|
||||
call __load_partial
|
||||
|
||||
movdqa MSG, T0
|
||||
pxor STATE0, T0
|
||||
pshufd $MASK3, STATE1, T1
|
||||
pxor T1, T0
|
||||
movdqa STATE2, T1
|
||||
pand STATE3, T1
|
||||
pxor T1, T0
|
||||
|
||||
call __store_partial
|
||||
|
||||
call __morus640_update
|
||||
|
||||
/* store the state: */
|
||||
movdqu STATE0, (0 * 16)(%rdi)
|
||||
movdqu STATE1, (1 * 16)(%rdi)
|
||||
movdqu STATE2, (2 * 16)(%rdi)
|
||||
movdqu STATE3, (3 * 16)(%rdi)
|
||||
movdqu STATE4, (4 * 16)(%rdi)
|
||||
|
||||
FRAME_END
|
||||
ret
|
||||
ENDPROC(crypto_morus640_sse2_enc_tail)
|
||||
|
||||
/*
|
||||
* void crypto_morus640_sse2_dec(void *state, const void *src, void *dst,
|
||||
* unsigned int length);
|
||||
*/
|
||||
ENTRY(crypto_morus640_sse2_dec)
|
||||
FRAME_BEGIN
|
||||
|
||||
cmp $16, %rcx
|
||||
jb .Ldec_out
|
||||
|
||||
/* load the state: */
|
||||
movdqu (0 * 16)(%rdi), STATE0
|
||||
movdqu (1 * 16)(%rdi), STATE1
|
||||
movdqu (2 * 16)(%rdi), STATE2
|
||||
movdqu (3 * 16)(%rdi), STATE3
|
||||
movdqu (4 * 16)(%rdi), STATE4
|
||||
|
||||
mov %rsi, %r8
|
||||
or %rdx, %r8
|
||||
and $0xF, %r8
|
||||
jnz .Ldec_u_loop
|
||||
|
||||
.align 4
|
||||
.Ldec_a_loop:
|
||||
movdqa (%rsi), MSG
|
||||
pxor STATE0, MSG
|
||||
pshufd $MASK3, STATE1, T0
|
||||
pxor T0, MSG
|
||||
movdqa STATE2, T0
|
||||
pand STATE3, T0
|
||||
pxor T0, MSG
|
||||
movdqa MSG, (%rdx)
|
||||
|
||||
call __morus640_update
|
||||
sub $16, %rcx
|
||||
add $16, %rsi
|
||||
add $16, %rdx
|
||||
cmp $16, %rcx
|
||||
jge .Ldec_a_loop
|
||||
|
||||
jmp .Ldec_cont
|
||||
.align 4
|
||||
.Ldec_u_loop:
|
||||
movdqu (%rsi), MSG
|
||||
pxor STATE0, MSG
|
||||
pshufd $MASK3, STATE1, T0
|
||||
pxor T0, MSG
|
||||
movdqa STATE2, T0
|
||||
pand STATE3, T0
|
||||
pxor T0, MSG
|
||||
movdqu MSG, (%rdx)
|
||||
|
||||
call __morus640_update
|
||||
sub $16, %rcx
|
||||
add $16, %rsi
|
||||
add $16, %rdx
|
||||
cmp $16, %rcx
|
||||
jge .Ldec_u_loop
|
||||
|
||||
.Ldec_cont:
|
||||
/* store the state: */
|
||||
movdqu STATE0, (0 * 16)(%rdi)
|
||||
movdqu STATE1, (1 * 16)(%rdi)
|
||||
movdqu STATE2, (2 * 16)(%rdi)
|
||||
movdqu STATE3, (3 * 16)(%rdi)
|
||||
movdqu STATE4, (4 * 16)(%rdi)
|
||||
|
||||
.Ldec_out:
|
||||
FRAME_END
|
||||
ret
|
||||
ENDPROC(crypto_morus640_sse2_dec)
|
||||
|
||||
/*
|
||||
* void crypto_morus640_sse2_dec_tail(void *state, const void *src, void *dst,
|
||||
* unsigned int length);
|
||||
*/
|
||||
ENTRY(crypto_morus640_sse2_dec_tail)
|
||||
FRAME_BEGIN
|
||||
|
||||
/* load the state: */
|
||||
movdqu (0 * 16)(%rdi), STATE0
|
||||
movdqu (1 * 16)(%rdi), STATE1
|
||||
movdqu (2 * 16)(%rdi), STATE2
|
||||
movdqu (3 * 16)(%rdi), STATE3
|
||||
movdqu (4 * 16)(%rdi), STATE4
|
||||
|
||||
/* decrypt message: */
|
||||
call __load_partial
|
||||
|
||||
pxor STATE0, MSG
|
||||
pshufd $MASK3, STATE1, T0
|
||||
pxor T0, MSG
|
||||
movdqa STATE2, T0
|
||||
pand STATE3, T0
|
||||
pxor T0, MSG
|
||||
movdqa MSG, T0
|
||||
|
||||
call __store_partial
|
||||
|
||||
/* mask with byte count: */
|
||||
movq %rcx, T0
|
||||
punpcklbw T0, T0
|
||||
punpcklbw T0, T0
|
||||
punpcklbw T0, T0
|
||||
punpcklbw T0, T0
|
||||
movdqa .Lmorus640_counter, T1
|
||||
pcmpgtb T1, T0
|
||||
pand T0, MSG
|
||||
|
||||
call __morus640_update
|
||||
|
||||
/* store the state: */
|
||||
movdqu STATE0, (0 * 16)(%rdi)
|
||||
movdqu STATE1, (1 * 16)(%rdi)
|
||||
movdqu STATE2, (2 * 16)(%rdi)
|
||||
movdqu STATE3, (3 * 16)(%rdi)
|
||||
movdqu STATE4, (4 * 16)(%rdi)
|
||||
|
||||
FRAME_END
|
||||
ret
|
||||
ENDPROC(crypto_morus640_sse2_dec_tail)
|
||||
|
||||
/*
|
||||
* void crypto_morus640_sse2_final(void *state, void *tag_xor,
|
||||
* u64 assoclen, u64 cryptlen);
|
||||
*/
|
||||
ENTRY(crypto_morus640_sse2_final)
|
||||
FRAME_BEGIN
|
||||
|
||||
/* load the state: */
|
||||
movdqu (0 * 16)(%rdi), STATE0
|
||||
movdqu (1 * 16)(%rdi), STATE1
|
||||
movdqu (2 * 16)(%rdi), STATE2
|
||||
movdqu (3 * 16)(%rdi), STATE3
|
||||
movdqu (4 * 16)(%rdi), STATE4
|
||||
|
||||
/* xor state[0] into state[4]: */
|
||||
pxor STATE0, STATE4
|
||||
|
||||
/* prepare length block: */
|
||||
movq %rdx, MSG
|
||||
movq %rcx, T0
|
||||
pslldq $8, T0
|
||||
pxor T0, MSG
|
||||
psllq $3, MSG /* multiply by 8 (to get bit count) */
|
||||
|
||||
/* update state: */
|
||||
call __morus640_update
|
||||
call __morus640_update
|
||||
call __morus640_update
|
||||
call __morus640_update
|
||||
call __morus640_update
|
||||
call __morus640_update
|
||||
call __morus640_update
|
||||
call __morus640_update
|
||||
call __morus640_update
|
||||
call __morus640_update
|
||||
|
||||
/* xor tag: */
|
||||
movdqu (%rsi), MSG
|
||||
|
||||
pxor STATE0, MSG
|
||||
pshufd $MASK3, STATE1, T0
|
||||
pxor T0, MSG
|
||||
movdqa STATE2, T0
|
||||
pand STATE3, T0
|
||||
pxor T0, MSG
|
||||
|
||||
movdqu MSG, (%rsi)
|
||||
|
||||
FRAME_END
|
||||
ret
|
||||
ENDPROC(crypto_morus640_sse2_final)
|
||||
@ -1,61 +0,0 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
/*
|
||||
* The MORUS-640 Authenticated-Encryption Algorithm
|
||||
* Glue for SSE2 implementation
|
||||
*
|
||||
* Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
|
||||
* Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <crypto/internal/aead.h>
|
||||
#include <crypto/internal/simd.h>
|
||||
#include <crypto/morus640_glue.h>
|
||||
#include <linux/module.h>
|
||||
#include <asm/fpu/api.h>
|
||||
#include <asm/cpu_device_id.h>
|
||||
|
||||
asmlinkage void crypto_morus640_sse2_init(void *state, const void *key,
|
||||
const void *iv);
|
||||
asmlinkage void crypto_morus640_sse2_ad(void *state, const void *data,
|
||||
unsigned int length);
|
||||
|
||||
asmlinkage void crypto_morus640_sse2_enc(void *state, const void *src,
|
||||
void *dst, unsigned int length);
|
||||
asmlinkage void crypto_morus640_sse2_dec(void *state, const void *src,
|
||||
void *dst, unsigned int length);
|
||||
|
||||
asmlinkage void crypto_morus640_sse2_enc_tail(void *state, const void *src,
|
||||
void *dst, unsigned int length);
|
||||
asmlinkage void crypto_morus640_sse2_dec_tail(void *state, const void *src,
|
||||
void *dst, unsigned int length);
|
||||
|
||||
asmlinkage void crypto_morus640_sse2_final(void *state, void *tag_xor,
|
||||
u64 assoclen, u64 cryptlen);
|
||||
|
||||
MORUS640_DECLARE_ALG(sse2, "morus640-sse2", 400);
|
||||
|
||||
static struct simd_aead_alg *simd_alg;
|
||||
|
||||
static int __init crypto_morus640_sse2_module_init(void)
|
||||
{
|
||||
if (!boot_cpu_has(X86_FEATURE_XMM2) ||
|
||||
!cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL))
|
||||
return -ENODEV;
|
||||
|
||||
return simd_register_aeads_compat(&crypto_morus640_sse2_alg, 1,
|
||||
&simd_alg);
|
||||
}
|
||||
|
||||
static void __exit crypto_morus640_sse2_module_exit(void)
|
||||
{
|
||||
simd_unregister_aeads(&crypto_morus640_sse2_alg, 1, &simd_alg);
|
||||
}
|
||||
|
||||
module_init(crypto_morus640_sse2_module_init);
|
||||
module_exit(crypto_morus640_sse2_module_exit);
|
||||
|
||||
MODULE_LICENSE("GPL");
|
||||
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
|
||||
MODULE_DESCRIPTION("MORUS-640 AEAD algorithm -- SSE2 implementation");
|
||||
MODULE_ALIAS_CRYPTO("morus640");
|
||||
MODULE_ALIAS_CRYPTO("morus640-sse2");
|
||||
@ -1,200 +0,0 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
/*
|
||||
* The MORUS-640 Authenticated-Encryption Algorithm
|
||||
* Common x86 SIMD glue skeleton
|
||||
*
|
||||
* Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
|
||||
* Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <crypto/internal/aead.h>
|
||||
#include <crypto/internal/skcipher.h>
|
||||
#include <crypto/morus640_glue.h>
|
||||
#include <crypto/scatterwalk.h>
|
||||
#include <linux/err.h>
|
||||
#include <linux/init.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/scatterlist.h>
|
||||
#include <asm/fpu/api.h>
|
||||
|
||||
struct morus640_state {
|
||||
struct morus640_block s[MORUS_STATE_BLOCKS];
|
||||
};
|
||||
|
||||
struct morus640_ops {
|
||||
int (*skcipher_walk_init)(struct skcipher_walk *walk,
|
||||
struct aead_request *req, bool atomic);
|
||||
|
||||
void (*crypt_blocks)(void *state, const void *src, void *dst,
|
||||
unsigned int length);
|
||||
void (*crypt_tail)(void *state, const void *src, void *dst,
|
||||
unsigned int length);
|
||||
};
|
||||
|
||||
static void crypto_morus640_glue_process_ad(
|
||||
struct morus640_state *state,
|
||||
const struct morus640_glue_ops *ops,
|
||||
struct scatterlist *sg_src, unsigned int assoclen)
|
||||
{
|
||||
struct scatter_walk walk;
|
||||
struct morus640_block buf;
|
||||
unsigned int pos = 0;
|
||||
|
||||
scatterwalk_start(&walk, sg_src);
|
||||
while (assoclen != 0) {
|
||||
unsigned int size = scatterwalk_clamp(&walk, assoclen);
|
||||
unsigned int left = size;
|
||||
void *mapped = scatterwalk_map(&walk);
|
||||
const u8 *src = (const u8 *)mapped;
|
||||
|
||||
if (pos + size >= MORUS640_BLOCK_SIZE) {
|
||||
if (pos > 0) {
|
||||
unsigned int fill = MORUS640_BLOCK_SIZE - pos;
|
||||
memcpy(buf.bytes + pos, src, fill);
|
||||
ops->ad(state, buf.bytes, MORUS640_BLOCK_SIZE);
|
||||
pos = 0;
|
||||
left -= fill;
|
||||
src += fill;
|
||||
}
|
||||
|
||||
ops->ad(state, src, left);
|
||||
src += left & ~(MORUS640_BLOCK_SIZE - 1);
|
||||
left &= MORUS640_BLOCK_SIZE - 1;
|
||||
}
|
||||
|
||||
memcpy(buf.bytes + pos, src, left);
|
||||
|
||||
pos += left;
|
||||
assoclen -= size;
|
||||
scatterwalk_unmap(mapped);
|
||||
scatterwalk_advance(&walk, size);
|
||||
scatterwalk_done(&walk, 0, assoclen);
|
||||
}
|
||||
|
||||
if (pos > 0) {
|
||||
memset(buf.bytes + pos, 0, MORUS640_BLOCK_SIZE - pos);
|
||||
ops->ad(state, buf.bytes, MORUS640_BLOCK_SIZE);
|
||||
}
|
||||
}
|
||||
|
||||
static void crypto_morus640_glue_process_crypt(struct morus640_state *state,
|
||||
struct morus640_ops ops,
|
||||
struct skcipher_walk *walk)
|
||||
{
|
||||
while (walk->nbytes >= MORUS640_BLOCK_SIZE) {
|
||||
ops.crypt_blocks(state, walk->src.virt.addr,
|
||||
walk->dst.virt.addr,
|
||||
round_down(walk->nbytes, MORUS640_BLOCK_SIZE));
|
||||
skcipher_walk_done(walk, walk->nbytes % MORUS640_BLOCK_SIZE);
|
||||
}
|
||||
|
||||
if (walk->nbytes) {
|
||||
ops.crypt_tail(state, walk->src.virt.addr, walk->dst.virt.addr,
|
||||
walk->nbytes);
|
||||
skcipher_walk_done(walk, 0);
|
||||
}
|
||||
}
|
||||
|
||||
int crypto_morus640_glue_setkey(struct crypto_aead *aead, const u8 *key,
|
||||
unsigned int keylen)
|
||||
{
|
||||
struct morus640_ctx *ctx = crypto_aead_ctx(aead);
|
||||
|
||||
if (keylen != MORUS640_BLOCK_SIZE) {
|
||||
crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
memcpy(ctx->key.bytes, key, MORUS640_BLOCK_SIZE);
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(crypto_morus640_glue_setkey);
|
||||
|
||||
int crypto_morus640_glue_setauthsize(struct crypto_aead *tfm,
|
||||
unsigned int authsize)
|
||||
{
|
||||
return (authsize <= MORUS_MAX_AUTH_SIZE) ? 0 : -EINVAL;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(crypto_morus640_glue_setauthsize);
|
||||
|
||||
static void crypto_morus640_glue_crypt(struct aead_request *req,
|
||||
struct morus640_ops ops,
|
||||
unsigned int cryptlen,
|
||||
struct morus640_block *tag_xor)
|
||||
{
|
||||
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
|
||||
struct morus640_ctx *ctx = crypto_aead_ctx(tfm);
|
||||
struct morus640_state state;
|
||||
struct skcipher_walk walk;
|
||||
|
||||
ops.skcipher_walk_init(&walk, req, true);
|
||||
|
||||
kernel_fpu_begin();
|
||||
|
||||
ctx->ops->init(&state, &ctx->key, req->iv);
|
||||
crypto_morus640_glue_process_ad(&state, ctx->ops, req->src, req->assoclen);
|
||||
crypto_morus640_glue_process_crypt(&state, ops, &walk);
|
||||
ctx->ops->final(&state, tag_xor, req->assoclen, cryptlen);
|
||||
|
||||
kernel_fpu_end();
|
||||
}
|
||||
|
||||
int crypto_morus640_glue_encrypt(struct aead_request *req)
|
||||
{
|
||||
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
|
||||
struct morus640_ctx *ctx = crypto_aead_ctx(tfm);
|
||||
struct morus640_ops OPS = {
|
||||
.skcipher_walk_init = skcipher_walk_aead_encrypt,
|
||||
.crypt_blocks = ctx->ops->enc,
|
||||
.crypt_tail = ctx->ops->enc_tail,
|
||||
};
|
||||
|
||||
struct morus640_block tag = {};
|
||||
unsigned int authsize = crypto_aead_authsize(tfm);
|
||||
unsigned int cryptlen = req->cryptlen;
|
||||
|
||||
crypto_morus640_glue_crypt(req, OPS, cryptlen, &tag);
|
||||
|
||||
scatterwalk_map_and_copy(tag.bytes, req->dst,
|
||||
req->assoclen + cryptlen, authsize, 1);
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(crypto_morus640_glue_encrypt);
|
||||
|
||||
int crypto_morus640_glue_decrypt(struct aead_request *req)
|
||||
{
|
||||
static const u8 zeros[MORUS640_BLOCK_SIZE] = {};
|
||||
|
||||
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
|
||||
struct morus640_ctx *ctx = crypto_aead_ctx(tfm);
|
||||
struct morus640_ops OPS = {
|
||||
.skcipher_walk_init = skcipher_walk_aead_decrypt,
|
||||
.crypt_blocks = ctx->ops->dec,
|
||||
.crypt_tail = ctx->ops->dec_tail,
|
||||
};
|
||||
|
||||
struct morus640_block tag;
|
||||
unsigned int authsize = crypto_aead_authsize(tfm);
|
||||
unsigned int cryptlen = req->cryptlen - authsize;
|
||||
|
||||
scatterwalk_map_and_copy(tag.bytes, req->src,
|
||||
req->assoclen + cryptlen, authsize, 0);
|
||||
|
||||
crypto_morus640_glue_crypt(req, OPS, cryptlen, &tag);
|
||||
|
||||
return crypto_memneq(tag.bytes, zeros, authsize) ? -EBADMSG : 0;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(crypto_morus640_glue_decrypt);
|
||||
|
||||
void crypto_morus640_glue_init_ops(struct crypto_aead *aead,
|
||||
const struct morus640_glue_ops *ops)
|
||||
{
|
||||
struct morus640_ctx *ctx = crypto_aead_ctx(aead);
|
||||
ctx->ops = ops;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(crypto_morus640_glue_init_ops);
|
||||
|
||||
MODULE_LICENSE("GPL");
|
||||
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
|
||||
MODULE_DESCRIPTION("MORUS-640 AEAD mode -- glue for x86 optimizations");
|
||||
@ -344,62 +344,6 @@ config CRYPTO_AEGIS256_AESNI_SSE2
|
||||
help
|
||||
AESNI+SSE2 implementation of the AEGIS-256 dedicated AEAD algorithm.
|
||||
|
||||
config CRYPTO_MORUS640
|
||||
tristate "MORUS-640 AEAD algorithm"
|
||||
select CRYPTO_AEAD
|
||||
help
|
||||
Support for the MORUS-640 dedicated AEAD algorithm.
|
||||
|
||||
config CRYPTO_MORUS640_GLUE
|
||||
tristate
|
||||
depends on X86
|
||||
select CRYPTO_AEAD
|
||||
select CRYPTO_SIMD
|
||||
help
|
||||
Common glue for SIMD optimizations of the MORUS-640 dedicated AEAD
|
||||
algorithm.
|
||||
|
||||
config CRYPTO_MORUS640_SSE2
|
||||
tristate "MORUS-640 AEAD algorithm (x86_64 SSE2 implementation)"
|
||||
depends on X86 && 64BIT
|
||||
select CRYPTO_AEAD
|
||||
select CRYPTO_MORUS640_GLUE
|
||||
help
|
||||
SSE2 implementation of the MORUS-640 dedicated AEAD algorithm.
|
||||
|
||||
config CRYPTO_MORUS1280
|
||||
tristate "MORUS-1280 AEAD algorithm"
|
||||
select CRYPTO_AEAD
|
||||
help
|
||||
Support for the MORUS-1280 dedicated AEAD algorithm.
|
||||
|
||||
config CRYPTO_MORUS1280_GLUE
|
||||
tristate
|
||||
depends on X86
|
||||
select CRYPTO_AEAD
|
||||
select CRYPTO_SIMD
|
||||
help
|
||||
Common glue for SIMD optimizations of the MORUS-1280 dedicated AEAD
|
||||
algorithm.
|
||||
|
||||
config CRYPTO_MORUS1280_SSE2
|
||||
tristate "MORUS-1280 AEAD algorithm (x86_64 SSE2 implementation)"
|
||||
depends on X86 && 64BIT
|
||||
select CRYPTO_AEAD
|
||||
select CRYPTO_MORUS1280_GLUE
|
||||
help
|
||||
SSE2 optimizedimplementation of the MORUS-1280 dedicated AEAD
|
||||
algorithm.
|
||||
|
||||
config CRYPTO_MORUS1280_AVX2
|
||||
tristate "MORUS-1280 AEAD algorithm (x86_64 AVX2 implementation)"
|
||||
depends on X86 && 64BIT
|
||||
select CRYPTO_AEAD
|
||||
select CRYPTO_MORUS1280_GLUE
|
||||
help
|
||||
AVX2 optimized implementation of the MORUS-1280 dedicated AEAD
|
||||
algorithm.
|
||||
|
||||
config CRYPTO_SEQIV
|
||||
tristate "Sequence Number IV Generator"
|
||||
select CRYPTO_AEAD
|
||||
|
||||
@ -92,8 +92,6 @@ obj-$(CONFIG_CRYPTO_CHACHA20POLY1305) += chacha20poly1305.o
|
||||
obj-$(CONFIG_CRYPTO_AEGIS128) += aegis128.o
|
||||
obj-$(CONFIG_CRYPTO_AEGIS128L) += aegis128l.o
|
||||
obj-$(CONFIG_CRYPTO_AEGIS256) += aegis256.o
|
||||
obj-$(CONFIG_CRYPTO_MORUS640) += morus640.o
|
||||
obj-$(CONFIG_CRYPTO_MORUS1280) += morus1280.o
|
||||
obj-$(CONFIG_CRYPTO_PCRYPT) += pcrypt.o
|
||||
obj-$(CONFIG_CRYPTO_CRYPTD) += cryptd.o
|
||||
obj-$(CONFIG_CRYPTO_DES) += des_generic.o
|
||||
|
||||
@ -1,542 +0,0 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
/*
|
||||
* The MORUS-1280 Authenticated-Encryption Algorithm
|
||||
*
|
||||
* Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
|
||||
* Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <asm/unaligned.h>
|
||||
#include <crypto/algapi.h>
|
||||
#include <crypto/internal/aead.h>
|
||||
#include <crypto/internal/skcipher.h>
|
||||
#include <crypto/morus_common.h>
|
||||
#include <crypto/scatterwalk.h>
|
||||
#include <linux/err.h>
|
||||
#include <linux/init.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/scatterlist.h>
|
||||
|
||||
#define MORUS1280_WORD_SIZE 8
|
||||
#define MORUS1280_BLOCK_SIZE (MORUS_BLOCK_WORDS * MORUS1280_WORD_SIZE)
|
||||
#define MORUS1280_BLOCK_ALIGN (__alignof__(__le64))
|
||||
#define MORUS1280_ALIGNED(p) IS_ALIGNED((uintptr_t)p, MORUS1280_BLOCK_ALIGN)
|
||||
|
||||
struct morus1280_block {
|
||||
u64 words[MORUS_BLOCK_WORDS];
|
||||
};
|
||||
|
||||
union morus1280_block_in {
|
||||
__le64 words[MORUS_BLOCK_WORDS];
|
||||
u8 bytes[MORUS1280_BLOCK_SIZE];
|
||||
};
|
||||
|
||||
struct morus1280_state {
|
||||
struct morus1280_block s[MORUS_STATE_BLOCKS];
|
||||
};
|
||||
|
||||
struct morus1280_ctx {
|
||||
struct morus1280_block key;
|
||||
};
|
||||
|
||||
struct morus1280_ops {
|
||||
int (*skcipher_walk_init)(struct skcipher_walk *walk,
|
||||
struct aead_request *req, bool atomic);
|
||||
|
||||
void (*crypt_chunk)(struct morus1280_state *state,
|
||||
u8 *dst, const u8 *src, unsigned int size);
|
||||
};
|
||||
|
||||
static const struct morus1280_block crypto_morus1280_const[1] = {
|
||||
{ .words = {
|
||||
U64_C(0x0d08050302010100),
|
||||
U64_C(0x6279e99059372215),
|
||||
U64_C(0xf12fc26d55183ddb),
|
||||
U64_C(0xdd28b57342311120),
|
||||
} },
|
||||
};
|
||||
|
||||
static void crypto_morus1280_round(struct morus1280_block *b0,
|
||||
struct morus1280_block *b1,
|
||||
struct morus1280_block *b2,
|
||||
struct morus1280_block *b3,
|
||||
struct morus1280_block *b4,
|
||||
const struct morus1280_block *m,
|
||||
unsigned int b, unsigned int w)
|
||||
{
|
||||
unsigned int i;
|
||||
struct morus1280_block tmp;
|
||||
|
||||
for (i = 0; i < MORUS_BLOCK_WORDS; i++) {
|
||||
b0->words[i] ^= b1->words[i] & b2->words[i];
|
||||
b0->words[i] ^= b3->words[i];
|
||||
b0->words[i] ^= m->words[i];
|
||||
b0->words[i] = rol64(b0->words[i], b);
|
||||
}
|
||||
|
||||
tmp = *b3;
|
||||
for (i = 0; i < MORUS_BLOCK_WORDS; i++)
|
||||
b3->words[(i + w) % MORUS_BLOCK_WORDS] = tmp.words[i];
|
||||
}
|
||||
|
||||
static void crypto_morus1280_update(struct morus1280_state *state,
|
||||
const struct morus1280_block *m)
|
||||
{
|
||||
static const struct morus1280_block z = {};
|
||||
|
||||
struct morus1280_block *s = state->s;
|
||||
|
||||
crypto_morus1280_round(&s[0], &s[1], &s[2], &s[3], &s[4], &z, 13, 1);
|
||||
crypto_morus1280_round(&s[1], &s[2], &s[3], &s[4], &s[0], m, 46, 2);
|
||||
crypto_morus1280_round(&s[2], &s[3], &s[4], &s[0], &s[1], m, 38, 3);
|
||||
crypto_morus1280_round(&s[3], &s[4], &s[0], &s[1], &s[2], m, 7, 2);
|
||||
crypto_morus1280_round(&s[4], &s[0], &s[1], &s[2], &s[3], m, 4, 1);
|
||||
}
|
||||
|
||||
static void crypto_morus1280_load_a(struct morus1280_block *dst, const u8 *src)
|
||||
{
|
||||
unsigned int i;
|
||||
for (i = 0; i < MORUS_BLOCK_WORDS; i++) {
|
||||
dst->words[i] = le64_to_cpu(*(const __le64 *)src);
|
||||
src += MORUS1280_WORD_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
static void crypto_morus1280_load_u(struct morus1280_block *dst, const u8 *src)
|
||||
{
|
||||
unsigned int i;
|
||||
for (i = 0; i < MORUS_BLOCK_WORDS; i++) {
|
||||
dst->words[i] = get_unaligned_le64(src);
|
||||
src += MORUS1280_WORD_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
static void crypto_morus1280_load(struct morus1280_block *dst, const u8 *src)
|
||||
{
|
||||
if (MORUS1280_ALIGNED(src))
|
||||
crypto_morus1280_load_a(dst, src);
|
||||
else
|
||||
crypto_morus1280_load_u(dst, src);
|
||||
}
|
||||
|
||||
static void crypto_morus1280_store_a(u8 *dst, const struct morus1280_block *src)
|
||||
{
|
||||
unsigned int i;
|
||||
for (i = 0; i < MORUS_BLOCK_WORDS; i++) {
|
||||
*(__le64 *)dst = cpu_to_le64(src->words[i]);
|
||||
dst += MORUS1280_WORD_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
static void crypto_morus1280_store_u(u8 *dst, const struct morus1280_block *src)
|
||||
{
|
||||
unsigned int i;
|
||||
for (i = 0; i < MORUS_BLOCK_WORDS; i++) {
|
||||
put_unaligned_le64(src->words[i], dst);
|
||||
dst += MORUS1280_WORD_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
static void crypto_morus1280_store(u8 *dst, const struct morus1280_block *src)
|
||||
{
|
||||
if (MORUS1280_ALIGNED(dst))
|
||||
crypto_morus1280_store_a(dst, src);
|
||||
else
|
||||
crypto_morus1280_store_u(dst, src);
|
||||
}
|
||||
|
||||
static void crypto_morus1280_ad(struct morus1280_state *state, const u8 *src,
|
||||
unsigned int size)
|
||||
{
|
||||
struct morus1280_block m;
|
||||
|
||||
if (MORUS1280_ALIGNED(src)) {
|
||||
while (size >= MORUS1280_BLOCK_SIZE) {
|
||||
crypto_morus1280_load_a(&m, src);
|
||||
crypto_morus1280_update(state, &m);
|
||||
|
||||
size -= MORUS1280_BLOCK_SIZE;
|
||||
src += MORUS1280_BLOCK_SIZE;
|
||||
}
|
||||
} else {
|
||||
while (size >= MORUS1280_BLOCK_SIZE) {
|
||||
crypto_morus1280_load_u(&m, src);
|
||||
crypto_morus1280_update(state, &m);
|
||||
|
||||
size -= MORUS1280_BLOCK_SIZE;
|
||||
src += MORUS1280_BLOCK_SIZE;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void crypto_morus1280_core(const struct morus1280_state *state,
|
||||
struct morus1280_block *blk)
|
||||
{
|
||||
unsigned int i;
|
||||
|
||||
for (i = 0; i < MORUS_BLOCK_WORDS; i++)
|
||||
blk->words[(i + 3) % MORUS_BLOCK_WORDS] ^= state->s[1].words[i];
|
||||
|
||||
for (i = 0; i < MORUS_BLOCK_WORDS; i++) {
|
||||
blk->words[i] ^= state->s[0].words[i];
|
||||
blk->words[i] ^= state->s[2].words[i] & state->s[3].words[i];
|
||||
}
|
||||
}
|
||||
|
||||
static void crypto_morus1280_encrypt_chunk(struct morus1280_state *state,
|
||||
u8 *dst, const u8 *src,
|
||||
unsigned int size)
|
||||
{
|
||||
struct morus1280_block c, m;
|
||||
|
||||
if (MORUS1280_ALIGNED(src) && MORUS1280_ALIGNED(dst)) {
|
||||
while (size >= MORUS1280_BLOCK_SIZE) {
|
||||
crypto_morus1280_load_a(&m, src);
|
||||
c = m;
|
||||
crypto_morus1280_core(state, &c);
|
||||
crypto_morus1280_store_a(dst, &c);
|
||||
crypto_morus1280_update(state, &m);
|
||||
|
||||
src += MORUS1280_BLOCK_SIZE;
|
||||
dst += MORUS1280_BLOCK_SIZE;
|
||||
size -= MORUS1280_BLOCK_SIZE;
|
||||
}
|
||||
} else {
|
||||
while (size >= MORUS1280_BLOCK_SIZE) {
|
||||
crypto_morus1280_load_u(&m, src);
|
||||
c = m;
|
||||
crypto_morus1280_core(state, &c);
|
||||
crypto_morus1280_store_u(dst, &c);
|
||||
crypto_morus1280_update(state, &m);
|
||||
|
||||
src += MORUS1280_BLOCK_SIZE;
|
||||
dst += MORUS1280_BLOCK_SIZE;
|
||||
size -= MORUS1280_BLOCK_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
if (size > 0) {
|
||||
union morus1280_block_in tail;
|
||||
|
||||
memcpy(tail.bytes, src, size);
|
||||
memset(tail.bytes + size, 0, MORUS1280_BLOCK_SIZE - size);
|
||||
|
||||
crypto_morus1280_load_a(&m, tail.bytes);
|
||||
c = m;
|
||||
crypto_morus1280_core(state, &c);
|
||||
crypto_morus1280_store_a(tail.bytes, &c);
|
||||
crypto_morus1280_update(state, &m);
|
||||
|
||||
memcpy(dst, tail.bytes, size);
|
||||
}
|
||||
}
|
||||
|
||||
static void crypto_morus1280_decrypt_chunk(struct morus1280_state *state,
|
||||
u8 *dst, const u8 *src,
|
||||
unsigned int size)
|
||||
{
|
||||
struct morus1280_block m;
|
||||
|
||||
if (MORUS1280_ALIGNED(src) && MORUS1280_ALIGNED(dst)) {
|
||||
while (size >= MORUS1280_BLOCK_SIZE) {
|
||||
crypto_morus1280_load_a(&m, src);
|
||||
crypto_morus1280_core(state, &m);
|
||||
crypto_morus1280_store_a(dst, &m);
|
||||
crypto_morus1280_update(state, &m);
|
||||
|
||||
src += MORUS1280_BLOCK_SIZE;
|
||||
dst += MORUS1280_BLOCK_SIZE;
|
||||
size -= MORUS1280_BLOCK_SIZE;
|
||||
}
|
||||
} else {
|
||||
while (size >= MORUS1280_BLOCK_SIZE) {
|
||||
crypto_morus1280_load_u(&m, src);
|
||||
crypto_morus1280_core(state, &m);
|
||||
crypto_morus1280_store_u(dst, &m);
|
||||
crypto_morus1280_update(state, &m);
|
||||
|
||||
src += MORUS1280_BLOCK_SIZE;
|
||||
dst += MORUS1280_BLOCK_SIZE;
|
||||
size -= MORUS1280_BLOCK_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
if (size > 0) {
|
||||
union morus1280_block_in tail;
|
||||
|
||||
memcpy(tail.bytes, src, size);
|
||||
memset(tail.bytes + size, 0, MORUS1280_BLOCK_SIZE - size);
|
||||
|
||||
crypto_morus1280_load_a(&m, tail.bytes);
|
||||
crypto_morus1280_core(state, &m);
|
||||
crypto_morus1280_store_a(tail.bytes, &m);
|
||||
memset(tail.bytes + size, 0, MORUS1280_BLOCK_SIZE - size);
|
||||
crypto_morus1280_load_a(&m, tail.bytes);
|
||||
crypto_morus1280_update(state, &m);
|
||||
|
||||
memcpy(dst, tail.bytes, size);
|
||||
}
|
||||
}
|
||||
|
||||
static void crypto_morus1280_init(struct morus1280_state *state,
|
||||
const struct morus1280_block *key,
|
||||
const u8 *iv)
|
||||
{
|
||||
static const struct morus1280_block z = {};
|
||||
|
||||
union morus1280_block_in tmp;
|
||||
unsigned int i;
|
||||
|
||||
memcpy(tmp.bytes, iv, MORUS_NONCE_SIZE);
|
||||
memset(tmp.bytes + MORUS_NONCE_SIZE, 0,
|
||||
MORUS1280_BLOCK_SIZE - MORUS_NONCE_SIZE);
|
||||
|
||||
crypto_morus1280_load(&state->s[0], tmp.bytes);
|
||||
state->s[1] = *key;
|
||||
for (i = 0; i < MORUS_BLOCK_WORDS; i++)
|
||||
state->s[2].words[i] = U64_C(0xFFFFFFFFFFFFFFFF);
|
||||
state->s[3] = z;
|
||||
state->s[4] = crypto_morus1280_const[0];
|
||||
|
||||
for (i = 0; i < 16; i++)
|
||||
crypto_morus1280_update(state, &z);
|
||||
|
||||
for (i = 0; i < MORUS_BLOCK_WORDS; i++)
|
||||
state->s[1].words[i] ^= key->words[i];
|
||||
}
|
||||
|
||||
static void crypto_morus1280_process_ad(struct morus1280_state *state,
|
||||
struct scatterlist *sg_src,
|
||||
unsigned int assoclen)
|
||||
{
|
||||
struct scatter_walk walk;
|
||||
struct morus1280_block m;
|
||||
union morus1280_block_in buf;
|
||||
unsigned int pos = 0;
|
||||
|
||||
scatterwalk_start(&walk, sg_src);
|
||||
while (assoclen != 0) {
|
||||
unsigned int size = scatterwalk_clamp(&walk, assoclen);
|
||||
unsigned int left = size;
|
||||
void *mapped = scatterwalk_map(&walk);
|
||||
const u8 *src = (const u8 *)mapped;
|
||||
|
||||
if (pos + size >= MORUS1280_BLOCK_SIZE) {
|
||||
if (pos > 0) {
|
||||
unsigned int fill = MORUS1280_BLOCK_SIZE - pos;
|
||||
memcpy(buf.bytes + pos, src, fill);
|
||||
|
||||
crypto_morus1280_load_a(&m, buf.bytes);
|
||||
crypto_morus1280_update(state, &m);
|
||||
|
||||
pos = 0;
|
||||
left -= fill;
|
||||
src += fill;
|
||||
}
|
||||
|
||||
crypto_morus1280_ad(state, src, left);
|
||||
src += left & ~(MORUS1280_BLOCK_SIZE - 1);
|
||||
left &= MORUS1280_BLOCK_SIZE - 1;
|
||||
}
|
||||
|
||||
memcpy(buf.bytes + pos, src, left);
|
||||
|
||||
pos += left;
|
||||
assoclen -= size;
|
||||
scatterwalk_unmap(mapped);
|
||||
scatterwalk_advance(&walk, size);
|
||||
scatterwalk_done(&walk, 0, assoclen);
|
||||
}
|
||||
|
||||
if (pos > 0) {
|
||||
memset(buf.bytes + pos, 0, MORUS1280_BLOCK_SIZE - pos);
|
||||
|
||||
crypto_morus1280_load_a(&m, buf.bytes);
|
||||
crypto_morus1280_update(state, &m);
|
||||
}
|
||||
}
|
||||
|
||||
static void crypto_morus1280_process_crypt(struct morus1280_state *state,
|
||||
struct aead_request *req,
|
||||
const struct morus1280_ops *ops)
|
||||
{
|
||||
struct skcipher_walk walk;
|
||||
|
||||
ops->skcipher_walk_init(&walk, req, false);
|
||||
|
||||
while (walk.nbytes) {
|
||||
unsigned int nbytes = walk.nbytes;
|
||||
|
||||
if (nbytes < walk.total)
|
||||
nbytes = round_down(nbytes, walk.stride);
|
||||
|
||||
ops->crypt_chunk(state, walk.dst.virt.addr, walk.src.virt.addr,
|
||||
nbytes);
|
||||
|
||||
skcipher_walk_done(&walk, walk.nbytes - nbytes);
|
||||
}
|
||||
}
|
||||
|
||||
static void crypto_morus1280_final(struct morus1280_state *state,
|
||||
struct morus1280_block *tag_xor,
|
||||
u64 assoclen, u64 cryptlen)
|
||||
{
|
||||
struct morus1280_block tmp;
|
||||
unsigned int i;
|
||||
|
||||
tmp.words[0] = assoclen * 8;
|
||||
tmp.words[1] = cryptlen * 8;
|
||||
tmp.words[2] = 0;
|
||||
tmp.words[3] = 0;
|
||||
|
||||
for (i = 0; i < MORUS_BLOCK_WORDS; i++)
|
||||
state->s[4].words[i] ^= state->s[0].words[i];
|
||||
|
||||
for (i = 0; i < 10; i++)
|
||||
crypto_morus1280_update(state, &tmp);
|
||||
|
||||
crypto_morus1280_core(state, tag_xor);
|
||||
}
|
||||
|
||||
static int crypto_morus1280_setkey(struct crypto_aead *aead, const u8 *key,
|
||||
unsigned int keylen)
|
||||
{
|
||||
struct morus1280_ctx *ctx = crypto_aead_ctx(aead);
|
||||
union morus1280_block_in tmp;
|
||||
|
||||
if (keylen == MORUS1280_BLOCK_SIZE)
|
||||
crypto_morus1280_load(&ctx->key, key);
|
||||
else if (keylen == MORUS1280_BLOCK_SIZE / 2) {
|
||||
memcpy(tmp.bytes, key, keylen);
|
||||
memcpy(tmp.bytes + keylen, key, keylen);
|
||||
|
||||
crypto_morus1280_load(&ctx->key, tmp.bytes);
|
||||
} else {
|
||||
crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int crypto_morus1280_setauthsize(struct crypto_aead *tfm,
|
||||
unsigned int authsize)
|
||||
{
|
||||
return (authsize <= MORUS_MAX_AUTH_SIZE) ? 0 : -EINVAL;
|
||||
}
|
||||
|
||||
static void crypto_morus1280_crypt(struct aead_request *req,
|
||||
struct morus1280_block *tag_xor,
|
||||
unsigned int cryptlen,
|
||||
const struct morus1280_ops *ops)
|
||||
{
|
||||
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
|
||||
struct morus1280_ctx *ctx = crypto_aead_ctx(tfm);
|
||||
struct morus1280_state state;
|
||||
|
||||
crypto_morus1280_init(&state, &ctx->key, req->iv);
|
||||
crypto_morus1280_process_ad(&state, req->src, req->assoclen);
|
||||
crypto_morus1280_process_crypt(&state, req, ops);
|
||||
crypto_morus1280_final(&state, tag_xor, req->assoclen, cryptlen);
|
||||
}
|
||||
|
||||
static int crypto_morus1280_encrypt(struct aead_request *req)
|
||||
{
|
||||
static const struct morus1280_ops ops = {
|
||||
.skcipher_walk_init = skcipher_walk_aead_encrypt,
|
||||
.crypt_chunk = crypto_morus1280_encrypt_chunk,
|
||||
};
|
||||
|
||||
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
|
||||
struct morus1280_block tag = {};
|
||||
union morus1280_block_in tag_out;
|
||||
unsigned int authsize = crypto_aead_authsize(tfm);
|
||||
unsigned int cryptlen = req->cryptlen;
|
||||
|
||||
crypto_morus1280_crypt(req, &tag, cryptlen, &ops);
|
||||
crypto_morus1280_store(tag_out.bytes, &tag);
|
||||
|
||||
scatterwalk_map_and_copy(tag_out.bytes, req->dst,
|
||||
req->assoclen + cryptlen, authsize, 1);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int crypto_morus1280_decrypt(struct aead_request *req)
|
||||
{
|
||||
static const struct morus1280_ops ops = {
|
||||
.skcipher_walk_init = skcipher_walk_aead_decrypt,
|
||||
.crypt_chunk = crypto_morus1280_decrypt_chunk,
|
||||
};
|
||||
static const u8 zeros[MORUS1280_BLOCK_SIZE] = {};
|
||||
|
||||
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
|
||||
union morus1280_block_in tag_in;
|
||||
struct morus1280_block tag;
|
||||
unsigned int authsize = crypto_aead_authsize(tfm);
|
||||
unsigned int cryptlen = req->cryptlen - authsize;
|
||||
|
||||
scatterwalk_map_and_copy(tag_in.bytes, req->src,
|
||||
req->assoclen + cryptlen, authsize, 0);
|
||||
|
||||
crypto_morus1280_load(&tag, tag_in.bytes);
|
||||
crypto_morus1280_crypt(req, &tag, cryptlen, &ops);
|
||||
crypto_morus1280_store(tag_in.bytes, &tag);
|
||||
|
||||
return crypto_memneq(tag_in.bytes, zeros, authsize) ? -EBADMSG : 0;
|
||||
}
|
||||
|
||||
static int crypto_morus1280_init_tfm(struct crypto_aead *tfm)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void crypto_morus1280_exit_tfm(struct crypto_aead *tfm)
|
||||
{
|
||||
}
|
||||
|
||||
static struct aead_alg crypto_morus1280_alg = {
|
||||
.setkey = crypto_morus1280_setkey,
|
||||
.setauthsize = crypto_morus1280_setauthsize,
|
||||
.encrypt = crypto_morus1280_encrypt,
|
||||
.decrypt = crypto_morus1280_decrypt,
|
||||
.init = crypto_morus1280_init_tfm,
|
||||
.exit = crypto_morus1280_exit_tfm,
|
||||
|
||||
.ivsize = MORUS_NONCE_SIZE,
|
||||
.maxauthsize = MORUS_MAX_AUTH_SIZE,
|
||||
.chunksize = MORUS1280_BLOCK_SIZE,
|
||||
|
||||
.base = {
|
||||
.cra_blocksize = 1,
|
||||
.cra_ctxsize = sizeof(struct morus1280_ctx),
|
||||
.cra_alignmask = 0,
|
||||
|
||||
.cra_priority = 100,
|
||||
|
||||
.cra_name = "morus1280",
|
||||
.cra_driver_name = "morus1280-generic",
|
||||
|
||||
.cra_module = THIS_MODULE,
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
static int __init crypto_morus1280_module_init(void)
|
||||
{
|
||||
return crypto_register_aead(&crypto_morus1280_alg);
|
||||
}
|
||||
|
||||
static void __exit crypto_morus1280_module_exit(void)
|
||||
{
|
||||
crypto_unregister_aead(&crypto_morus1280_alg);
|
||||
}
|
||||
|
||||
subsys_initcall(crypto_morus1280_module_init);
|
||||
module_exit(crypto_morus1280_module_exit);
|
||||
|
||||
MODULE_LICENSE("GPL");
|
||||
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
|
||||
MODULE_DESCRIPTION("MORUS-1280 AEAD algorithm");
|
||||
MODULE_ALIAS_CRYPTO("morus1280");
|
||||
MODULE_ALIAS_CRYPTO("morus1280-generic");
|
||||
@ -1,533 +0,0 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
/*
|
||||
* The MORUS-640 Authenticated-Encryption Algorithm
|
||||
*
|
||||
* Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
|
||||
* Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <asm/unaligned.h>
|
||||
#include <crypto/algapi.h>
|
||||
#include <crypto/internal/aead.h>
|
||||
#include <crypto/internal/skcipher.h>
|
||||
#include <crypto/morus_common.h>
|
||||
#include <crypto/scatterwalk.h>
|
||||
#include <linux/err.h>
|
||||
#include <linux/init.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/scatterlist.h>
|
||||
|
||||
#define MORUS640_WORD_SIZE 4
|
||||
#define MORUS640_BLOCK_SIZE (MORUS_BLOCK_WORDS * MORUS640_WORD_SIZE)
|
||||
#define MORUS640_BLOCK_ALIGN (__alignof__(__le32))
|
||||
#define MORUS640_ALIGNED(p) IS_ALIGNED((uintptr_t)p, MORUS640_BLOCK_ALIGN)
|
||||
|
||||
struct morus640_block {
|
||||
u32 words[MORUS_BLOCK_WORDS];
|
||||
};
|
||||
|
||||
union morus640_block_in {
|
||||
__le32 words[MORUS_BLOCK_WORDS];
|
||||
u8 bytes[MORUS640_BLOCK_SIZE];
|
||||
};
|
||||
|
||||
struct morus640_state {
|
||||
struct morus640_block s[MORUS_STATE_BLOCKS];
|
||||
};
|
||||
|
||||
struct morus640_ctx {
|
||||
struct morus640_block key;
|
||||
};
|
||||
|
||||
struct morus640_ops {
|
||||
int (*skcipher_walk_init)(struct skcipher_walk *walk,
|
||||
struct aead_request *req, bool atomic);
|
||||
|
||||
void (*crypt_chunk)(struct morus640_state *state,
|
||||
u8 *dst, const u8 *src, unsigned int size);
|
||||
};
|
||||
|
||||
static const struct morus640_block crypto_morus640_const[2] = {
|
||||
{ .words = {
|
||||
U32_C(0x02010100),
|
||||
U32_C(0x0d080503),
|
||||
U32_C(0x59372215),
|
||||
U32_C(0x6279e990),
|
||||
} },
|
||||
{ .words = {
|
||||
U32_C(0x55183ddb),
|
||||
U32_C(0xf12fc26d),
|
||||
U32_C(0x42311120),
|
||||
U32_C(0xdd28b573),
|
||||
} },
|
||||
};
|
||||
|
||||
static void crypto_morus640_round(struct morus640_block *b0,
|
||||
struct morus640_block *b1,
|
||||
struct morus640_block *b2,
|
||||
struct morus640_block *b3,
|
||||
struct morus640_block *b4,
|
||||
const struct morus640_block *m,
|
||||
unsigned int b, unsigned int w)
|
||||
{
|
||||
unsigned int i;
|
||||
struct morus640_block tmp;
|
||||
|
||||
for (i = 0; i < MORUS_BLOCK_WORDS; i++) {
|
||||
b0->words[i] ^= b1->words[i] & b2->words[i];
|
||||
b0->words[i] ^= b3->words[i];
|
||||
b0->words[i] ^= m->words[i];
|
||||
b0->words[i] = rol32(b0->words[i], b);
|
||||
}
|
||||
|
||||
tmp = *b3;
|
||||
for (i = 0; i < MORUS_BLOCK_WORDS; i++)
|
||||
b3->words[(i + w) % MORUS_BLOCK_WORDS] = tmp.words[i];
|
||||
}
|
||||
|
||||
static void crypto_morus640_update(struct morus640_state *state,
|
||||
const struct morus640_block *m)
|
||||
{
|
||||
static const struct morus640_block z = {};
|
||||
|
||||
struct morus640_block *s = state->s;
|
||||
|
||||
crypto_morus640_round(&s[0], &s[1], &s[2], &s[3], &s[4], &z, 5, 1);
|
||||
crypto_morus640_round(&s[1], &s[2], &s[3], &s[4], &s[0], m, 31, 2);
|
||||
crypto_morus640_round(&s[2], &s[3], &s[4], &s[0], &s[1], m, 7, 3);
|
||||
crypto_morus640_round(&s[3], &s[4], &s[0], &s[1], &s[2], m, 22, 2);
|
||||
crypto_morus640_round(&s[4], &s[0], &s[1], &s[2], &s[3], m, 13, 1);
|
||||
}
|
||||
|
||||
static void crypto_morus640_load_a(struct morus640_block *dst, const u8 *src)
|
||||
{
|
||||
unsigned int i;
|
||||
for (i = 0; i < MORUS_BLOCK_WORDS; i++) {
|
||||
dst->words[i] = le32_to_cpu(*(const __le32 *)src);
|
||||
src += MORUS640_WORD_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
static void crypto_morus640_load_u(struct morus640_block *dst, const u8 *src)
|
||||
{
|
||||
unsigned int i;
|
||||
for (i = 0; i < MORUS_BLOCK_WORDS; i++) {
|
||||
dst->words[i] = get_unaligned_le32(src);
|
||||
src += MORUS640_WORD_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
static void crypto_morus640_load(struct morus640_block *dst, const u8 *src)
|
||||
{
|
||||
if (MORUS640_ALIGNED(src))
|
||||
crypto_morus640_load_a(dst, src);
|
||||
else
|
||||
crypto_morus640_load_u(dst, src);
|
||||
}
|
||||
|
||||
static void crypto_morus640_store_a(u8 *dst, const struct morus640_block *src)
|
||||
{
|
||||
unsigned int i;
|
||||
for (i = 0; i < MORUS_BLOCK_WORDS; i++) {
|
||||
*(__le32 *)dst = cpu_to_le32(src->words[i]);
|
||||
dst += MORUS640_WORD_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
static void crypto_morus640_store_u(u8 *dst, const struct morus640_block *src)
|
||||
{
|
||||
unsigned int i;
|
||||
for (i = 0; i < MORUS_BLOCK_WORDS; i++) {
|
||||
put_unaligned_le32(src->words[i], dst);
|
||||
dst += MORUS640_WORD_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
static void crypto_morus640_store(u8 *dst, const struct morus640_block *src)
|
||||
{
|
||||
if (MORUS640_ALIGNED(dst))
|
||||
crypto_morus640_store_a(dst, src);
|
||||
else
|
||||
crypto_morus640_store_u(dst, src);
|
||||
}
|
||||
|
||||
static void crypto_morus640_ad(struct morus640_state *state, const u8 *src,
|
||||
unsigned int size)
|
||||
{
|
||||
struct morus640_block m;
|
||||
|
||||
if (MORUS640_ALIGNED(src)) {
|
||||
while (size >= MORUS640_BLOCK_SIZE) {
|
||||
crypto_morus640_load_a(&m, src);
|
||||
crypto_morus640_update(state, &m);
|
||||
|
||||
size -= MORUS640_BLOCK_SIZE;
|
||||
src += MORUS640_BLOCK_SIZE;
|
||||
}
|
||||
} else {
|
||||
while (size >= MORUS640_BLOCK_SIZE) {
|
||||
crypto_morus640_load_u(&m, src);
|
||||
crypto_morus640_update(state, &m);
|
||||
|
||||
size -= MORUS640_BLOCK_SIZE;
|
||||
src += MORUS640_BLOCK_SIZE;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void crypto_morus640_core(const struct morus640_state *state,
|
||||
struct morus640_block *blk)
|
||||
{
|
||||
unsigned int i;
|
||||
|
||||
for (i = 0; i < MORUS_BLOCK_WORDS; i++)
|
||||
blk->words[(i + 3) % MORUS_BLOCK_WORDS] ^= state->s[1].words[i];
|
||||
|
||||
for (i = 0; i < MORUS_BLOCK_WORDS; i++) {
|
||||
blk->words[i] ^= state->s[0].words[i];
|
||||
blk->words[i] ^= state->s[2].words[i] & state->s[3].words[i];
|
||||
}
|
||||
}
|
||||
|
||||
static void crypto_morus640_encrypt_chunk(struct morus640_state *state, u8 *dst,
|
||||
const u8 *src, unsigned int size)
|
||||
{
|
||||
struct morus640_block c, m;
|
||||
|
||||
if (MORUS640_ALIGNED(src) && MORUS640_ALIGNED(dst)) {
|
||||
while (size >= MORUS640_BLOCK_SIZE) {
|
||||
crypto_morus640_load_a(&m, src);
|
||||
c = m;
|
||||
crypto_morus640_core(state, &c);
|
||||
crypto_morus640_store_a(dst, &c);
|
||||
crypto_morus640_update(state, &m);
|
||||
|
||||
src += MORUS640_BLOCK_SIZE;
|
||||
dst += MORUS640_BLOCK_SIZE;
|
||||
size -= MORUS640_BLOCK_SIZE;
|
||||
}
|
||||
} else {
|
||||
while (size >= MORUS640_BLOCK_SIZE) {
|
||||
crypto_morus640_load_u(&m, src);
|
||||
c = m;
|
||||
crypto_morus640_core(state, &c);
|
||||
crypto_morus640_store_u(dst, &c);
|
||||
crypto_morus640_update(state, &m);
|
||||
|
||||
src += MORUS640_BLOCK_SIZE;
|
||||
dst += MORUS640_BLOCK_SIZE;
|
||||
size -= MORUS640_BLOCK_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
if (size > 0) {
|
||||
union morus640_block_in tail;
|
||||
|
||||
memcpy(tail.bytes, src, size);
|
||||
memset(tail.bytes + size, 0, MORUS640_BLOCK_SIZE - size);
|
||||
|
||||
crypto_morus640_load_a(&m, tail.bytes);
|
||||
c = m;
|
||||
crypto_morus640_core(state, &c);
|
||||
crypto_morus640_store_a(tail.bytes, &c);
|
||||
crypto_morus640_update(state, &m);
|
||||
|
||||
memcpy(dst, tail.bytes, size);
|
||||
}
|
||||
}
|
||||
|
||||
static void crypto_morus640_decrypt_chunk(struct morus640_state *state, u8 *dst,
|
||||
const u8 *src, unsigned int size)
|
||||
{
|
||||
struct morus640_block m;
|
||||
|
||||
if (MORUS640_ALIGNED(src) && MORUS640_ALIGNED(dst)) {
|
||||
while (size >= MORUS640_BLOCK_SIZE) {
|
||||
crypto_morus640_load_a(&m, src);
|
||||
crypto_morus640_core(state, &m);
|
||||
crypto_morus640_store_a(dst, &m);
|
||||
crypto_morus640_update(state, &m);
|
||||
|
||||
src += MORUS640_BLOCK_SIZE;
|
||||
dst += MORUS640_BLOCK_SIZE;
|
||||
size -= MORUS640_BLOCK_SIZE;
|
||||
}
|
||||
} else {
|
||||
while (size >= MORUS640_BLOCK_SIZE) {
|
||||
crypto_morus640_load_u(&m, src);
|
||||
crypto_morus640_core(state, &m);
|
||||
crypto_morus640_store_u(dst, &m);
|
||||
crypto_morus640_update(state, &m);
|
||||
|
||||
src += MORUS640_BLOCK_SIZE;
|
||||
dst += MORUS640_BLOCK_SIZE;
|
||||
size -= MORUS640_BLOCK_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
if (size > 0) {
|
||||
union morus640_block_in tail;
|
||||
|
||||
memcpy(tail.bytes, src, size);
|
||||
memset(tail.bytes + size, 0, MORUS640_BLOCK_SIZE - size);
|
||||
|
||||
crypto_morus640_load_a(&m, tail.bytes);
|
||||
crypto_morus640_core(state, &m);
|
||||
crypto_morus640_store_a(tail.bytes, &m);
|
||||
memset(tail.bytes + size, 0, MORUS640_BLOCK_SIZE - size);
|
||||
crypto_morus640_load_a(&m, tail.bytes);
|
||||
crypto_morus640_update(state, &m);
|
||||
|
||||
memcpy(dst, tail.bytes, size);
|
||||
}
|
||||
}
|
||||
|
||||
static void crypto_morus640_init(struct morus640_state *state,
|
||||
const struct morus640_block *key,
|
||||
const u8 *iv)
|
||||
{
|
||||
static const struct morus640_block z = {};
|
||||
|
||||
unsigned int i;
|
||||
|
||||
crypto_morus640_load(&state->s[0], iv);
|
||||
state->s[1] = *key;
|
||||
for (i = 0; i < MORUS_BLOCK_WORDS; i++)
|
||||
state->s[2].words[i] = U32_C(0xFFFFFFFF);
|
||||
state->s[3] = crypto_morus640_const[0];
|
||||
state->s[4] = crypto_morus640_const[1];
|
||||
|
||||
for (i = 0; i < 16; i++)
|
||||
crypto_morus640_update(state, &z);
|
||||
|
||||
for (i = 0; i < MORUS_BLOCK_WORDS; i++)
|
||||
state->s[1].words[i] ^= key->words[i];
|
||||
}
|
||||
|
||||
static void crypto_morus640_process_ad(struct morus640_state *state,
|
||||
struct scatterlist *sg_src,
|
||||
unsigned int assoclen)
|
||||
{
|
||||
struct scatter_walk walk;
|
||||
struct morus640_block m;
|
||||
union morus640_block_in buf;
|
||||
unsigned int pos = 0;
|
||||
|
||||
scatterwalk_start(&walk, sg_src);
|
||||
while (assoclen != 0) {
|
||||
unsigned int size = scatterwalk_clamp(&walk, assoclen);
|
||||
unsigned int left = size;
|
||||
void *mapped = scatterwalk_map(&walk);
|
||||
const u8 *src = (const u8 *)mapped;
|
||||
|
||||
if (pos + size >= MORUS640_BLOCK_SIZE) {
|
||||
if (pos > 0) {
|
||||
unsigned int fill = MORUS640_BLOCK_SIZE - pos;
|
||||
memcpy(buf.bytes + pos, src, fill);
|
||||
|
||||
crypto_morus640_load_a(&m, buf.bytes);
|
||||
crypto_morus640_update(state, &m);
|
||||
|
||||
pos = 0;
|
||||
left -= fill;
|
||||
src += fill;
|
||||
}
|
||||
|
||||
crypto_morus640_ad(state, src, left);
|
||||
src += left & ~(MORUS640_BLOCK_SIZE - 1);
|
||||
left &= MORUS640_BLOCK_SIZE - 1;
|
||||
}
|
||||
|
||||
memcpy(buf.bytes + pos, src, left);
|
||||
|
||||
pos += left;
|
||||
assoclen -= size;
|
||||
scatterwalk_unmap(mapped);
|
||||
scatterwalk_advance(&walk, size);
|
||||
scatterwalk_done(&walk, 0, assoclen);
|
||||
}
|
||||
|
||||
if (pos > 0) {
|
||||
memset(buf.bytes + pos, 0, MORUS640_BLOCK_SIZE - pos);
|
||||
|
||||
crypto_morus640_load_a(&m, buf.bytes);
|
||||
crypto_morus640_update(state, &m);
|
||||
}
|
||||
}
|
||||
|
||||
static void crypto_morus640_process_crypt(struct morus640_state *state,
|
||||
struct aead_request *req,
|
||||
const struct morus640_ops *ops)
|
||||
{
|
||||
struct skcipher_walk walk;
|
||||
|
||||
ops->skcipher_walk_init(&walk, req, false);
|
||||
|
||||
while (walk.nbytes) {
|
||||
unsigned int nbytes = walk.nbytes;
|
||||
|
||||
if (nbytes < walk.total)
|
||||
nbytes = round_down(nbytes, walk.stride);
|
||||
|
||||
ops->crypt_chunk(state, walk.dst.virt.addr, walk.src.virt.addr,
|
||||
nbytes);
|
||||
|
||||
skcipher_walk_done(&walk, walk.nbytes - nbytes);
|
||||
}
|
||||
}
|
||||
|
||||
static void crypto_morus640_final(struct morus640_state *state,
|
||||
struct morus640_block *tag_xor,
|
||||
u64 assoclen, u64 cryptlen)
|
||||
{
|
||||
struct morus640_block tmp;
|
||||
unsigned int i;
|
||||
|
||||
tmp.words[0] = lower_32_bits(assoclen * 8);
|
||||
tmp.words[1] = upper_32_bits(assoclen * 8);
|
||||
tmp.words[2] = lower_32_bits(cryptlen * 8);
|
||||
tmp.words[3] = upper_32_bits(cryptlen * 8);
|
||||
|
||||
for (i = 0; i < MORUS_BLOCK_WORDS; i++)
|
||||
state->s[4].words[i] ^= state->s[0].words[i];
|
||||
|
||||
for (i = 0; i < 10; i++)
|
||||
crypto_morus640_update(state, &tmp);
|
||||
|
||||
crypto_morus640_core(state, tag_xor);
|
||||
}
|
||||
|
||||
static int crypto_morus640_setkey(struct crypto_aead *aead, const u8 *key,
|
||||
unsigned int keylen)
|
||||
{
|
||||
struct morus640_ctx *ctx = crypto_aead_ctx(aead);
|
||||
|
||||
if (keylen != MORUS640_BLOCK_SIZE) {
|
||||
crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
crypto_morus640_load(&ctx->key, key);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int crypto_morus640_setauthsize(struct crypto_aead *tfm,
|
||||
unsigned int authsize)
|
||||
{
|
||||
return (authsize <= MORUS_MAX_AUTH_SIZE) ? 0 : -EINVAL;
|
||||
}
|
||||
|
||||
static void crypto_morus640_crypt(struct aead_request *req,
|
||||
struct morus640_block *tag_xor,
|
||||
unsigned int cryptlen,
|
||||
const struct morus640_ops *ops)
|
||||
{
|
||||
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
|
||||
struct morus640_ctx *ctx = crypto_aead_ctx(tfm);
|
||||
struct morus640_state state;
|
||||
|
||||
crypto_morus640_init(&state, &ctx->key, req->iv);
|
||||
crypto_morus640_process_ad(&state, req->src, req->assoclen);
|
||||
crypto_morus640_process_crypt(&state, req, ops);
|
||||
crypto_morus640_final(&state, tag_xor, req->assoclen, cryptlen);
|
||||
}
|
||||
|
||||
static int crypto_morus640_encrypt(struct aead_request *req)
|
||||
{
|
||||
static const struct morus640_ops ops = {
|
||||
.skcipher_walk_init = skcipher_walk_aead_encrypt,
|
||||
.crypt_chunk = crypto_morus640_encrypt_chunk,
|
||||
};
|
||||
|
||||
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
|
||||
struct morus640_block tag = {};
|
||||
union morus640_block_in tag_out;
|
||||
unsigned int authsize = crypto_aead_authsize(tfm);
|
||||
unsigned int cryptlen = req->cryptlen;
|
||||
|
||||
crypto_morus640_crypt(req, &tag, cryptlen, &ops);
|
||||
crypto_morus640_store(tag_out.bytes, &tag);
|
||||
|
||||
scatterwalk_map_and_copy(tag_out.bytes, req->dst,
|
||||
req->assoclen + cryptlen, authsize, 1);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int crypto_morus640_decrypt(struct aead_request *req)
|
||||
{
|
||||
static const struct morus640_ops ops = {
|
||||
.skcipher_walk_init = skcipher_walk_aead_decrypt,
|
||||
.crypt_chunk = crypto_morus640_decrypt_chunk,
|
||||
};
|
||||
static const u8 zeros[MORUS640_BLOCK_SIZE] = {};
|
||||
|
||||
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
|
||||
union morus640_block_in tag_in;
|
||||
struct morus640_block tag;
|
||||
unsigned int authsize = crypto_aead_authsize(tfm);
|
||||
unsigned int cryptlen = req->cryptlen - authsize;
|
||||
|
||||
scatterwalk_map_and_copy(tag_in.bytes, req->src,
|
||||
req->assoclen + cryptlen, authsize, 0);
|
||||
|
||||
crypto_morus640_load(&tag, tag_in.bytes);
|
||||
crypto_morus640_crypt(req, &tag, cryptlen, &ops);
|
||||
crypto_morus640_store(tag_in.bytes, &tag);
|
||||
|
||||
return crypto_memneq(tag_in.bytes, zeros, authsize) ? -EBADMSG : 0;
|
||||
}
|
||||
|
||||
static int crypto_morus640_init_tfm(struct crypto_aead *tfm)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void crypto_morus640_exit_tfm(struct crypto_aead *tfm)
|
||||
{
|
||||
}
|
||||
|
||||
static struct aead_alg crypto_morus640_alg = {
|
||||
.setkey = crypto_morus640_setkey,
|
||||
.setauthsize = crypto_morus640_setauthsize,
|
||||
.encrypt = crypto_morus640_encrypt,
|
||||
.decrypt = crypto_morus640_decrypt,
|
||||
.init = crypto_morus640_init_tfm,
|
||||
.exit = crypto_morus640_exit_tfm,
|
||||
|
||||
.ivsize = MORUS_NONCE_SIZE,
|
||||
.maxauthsize = MORUS_MAX_AUTH_SIZE,
|
||||
.chunksize = MORUS640_BLOCK_SIZE,
|
||||
|
||||
.base = {
|
||||
.cra_blocksize = 1,
|
||||
.cra_ctxsize = sizeof(struct morus640_ctx),
|
||||
.cra_alignmask = 0,
|
||||
|
||||
.cra_priority = 100,
|
||||
|
||||
.cra_name = "morus640",
|
||||
.cra_driver_name = "morus640-generic",
|
||||
|
||||
.cra_module = THIS_MODULE,
|
||||
}
|
||||
};
|
||||
|
||||
static int __init crypto_morus640_module_init(void)
|
||||
{
|
||||
return crypto_register_aead(&crypto_morus640_alg);
|
||||
}
|
||||
|
||||
static void __exit crypto_morus640_module_exit(void)
|
||||
{
|
||||
crypto_unregister_aead(&crypto_morus640_alg);
|
||||
}
|
||||
|
||||
subsys_initcall(crypto_morus640_module_init);
|
||||
module_exit(crypto_morus640_module_exit);
|
||||
|
||||
MODULE_LICENSE("GPL");
|
||||
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
|
||||
MODULE_DESCRIPTION("MORUS-640 AEAD algorithm");
|
||||
MODULE_ALIAS_CRYPTO("morus640");
|
||||
MODULE_ALIAS_CRYPTO("morus640-generic");
|
||||
@ -4768,18 +4768,6 @@ static const struct alg_test_desc alg_test_descs[] = {
|
||||
.suite = {
|
||||
.hash = __VECS(michael_mic_tv_template)
|
||||
}
|
||||
}, {
|
||||
.alg = "morus1280",
|
||||
.test = alg_test_aead,
|
||||
.suite = {
|
||||
.aead = __VECS(morus1280_tv_template)
|
||||
}
|
||||
}, {
|
||||
.alg = "morus640",
|
||||
.test = alg_test_aead,
|
||||
.suite = {
|
||||
.aead = __VECS(morus640_tv_template)
|
||||
}
|
||||
}, {
|
||||
.alg = "nhpoly1305",
|
||||
.test = alg_test_hash,
|
||||
|
||||
1707
crypto/testmgr.h
1707
crypto/testmgr.h
File diff suppressed because it is too large
Load Diff
@ -1,97 +0,0 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0-or-later */
|
||||
/*
|
||||
* The MORUS-1280 Authenticated-Encryption Algorithm
|
||||
* Common glue skeleton -- header file
|
||||
*
|
||||
* Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
|
||||
* Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef _CRYPTO_MORUS1280_GLUE_H
|
||||
#define _CRYPTO_MORUS1280_GLUE_H
|
||||
|
||||
#include <linux/module.h>
|
||||
#include <linux/types.h>
|
||||
#include <crypto/algapi.h>
|
||||
#include <crypto/aead.h>
|
||||
#include <crypto/morus_common.h>
|
||||
|
||||
#define MORUS1280_WORD_SIZE 8
|
||||
#define MORUS1280_BLOCK_SIZE (MORUS_BLOCK_WORDS * MORUS1280_WORD_SIZE)
|
||||
|
||||
struct morus1280_block {
|
||||
u8 bytes[MORUS1280_BLOCK_SIZE];
|
||||
};
|
||||
|
||||
struct morus1280_glue_ops {
|
||||
void (*init)(void *state, const void *key, const void *iv);
|
||||
void (*ad)(void *state, const void *data, unsigned int length);
|
||||
void (*enc)(void *state, const void *src, void *dst, unsigned int length);
|
||||
void (*dec)(void *state, const void *src, void *dst, unsigned int length);
|
||||
void (*enc_tail)(void *state, const void *src, void *dst, unsigned int length);
|
||||
void (*dec_tail)(void *state, const void *src, void *dst, unsigned int length);
|
||||
void (*final)(void *state, void *tag_xor, u64 assoclen, u64 cryptlen);
|
||||
};
|
||||
|
||||
struct morus1280_ctx {
|
||||
const struct morus1280_glue_ops *ops;
|
||||
struct morus1280_block key;
|
||||
};
|
||||
|
||||
void crypto_morus1280_glue_init_ops(struct crypto_aead *aead,
|
||||
const struct morus1280_glue_ops *ops);
|
||||
int crypto_morus1280_glue_setkey(struct crypto_aead *aead, const u8 *key,
|
||||
unsigned int keylen);
|
||||
int crypto_morus1280_glue_setauthsize(struct crypto_aead *tfm,
|
||||
unsigned int authsize);
|
||||
int crypto_morus1280_glue_encrypt(struct aead_request *req);
|
||||
int crypto_morus1280_glue_decrypt(struct aead_request *req);
|
||||
|
||||
#define MORUS1280_DECLARE_ALG(id, driver_name, priority) \
|
||||
static const struct morus1280_glue_ops crypto_morus1280_##id##_ops = {\
|
||||
.init = crypto_morus1280_##id##_init, \
|
||||
.ad = crypto_morus1280_##id##_ad, \
|
||||
.enc = crypto_morus1280_##id##_enc, \
|
||||
.enc_tail = crypto_morus1280_##id##_enc_tail, \
|
||||
.dec = crypto_morus1280_##id##_dec, \
|
||||
.dec_tail = crypto_morus1280_##id##_dec_tail, \
|
||||
.final = crypto_morus1280_##id##_final, \
|
||||
}; \
|
||||
\
|
||||
static int crypto_morus1280_##id##_init_tfm(struct crypto_aead *tfm) \
|
||||
{ \
|
||||
crypto_morus1280_glue_init_ops(tfm, &crypto_morus1280_##id##_ops); \
|
||||
return 0; \
|
||||
} \
|
||||
\
|
||||
static void crypto_morus1280_##id##_exit_tfm(struct crypto_aead *tfm) \
|
||||
{ \
|
||||
} \
|
||||
\
|
||||
static struct aead_alg crypto_morus1280_##id##_alg = { \
|
||||
.setkey = crypto_morus1280_glue_setkey, \
|
||||
.setauthsize = crypto_morus1280_glue_setauthsize, \
|
||||
.encrypt = crypto_morus1280_glue_encrypt, \
|
||||
.decrypt = crypto_morus1280_glue_decrypt, \
|
||||
.init = crypto_morus1280_##id##_init_tfm, \
|
||||
.exit = crypto_morus1280_##id##_exit_tfm, \
|
||||
\
|
||||
.ivsize = MORUS_NONCE_SIZE, \
|
||||
.maxauthsize = MORUS_MAX_AUTH_SIZE, \
|
||||
.chunksize = MORUS1280_BLOCK_SIZE, \
|
||||
\
|
||||
.base = { \
|
||||
.cra_flags = CRYPTO_ALG_INTERNAL, \
|
||||
.cra_blocksize = 1, \
|
||||
.cra_ctxsize = sizeof(struct morus1280_ctx), \
|
||||
.cra_alignmask = 0, \
|
||||
.cra_priority = priority, \
|
||||
\
|
||||
.cra_name = "__morus1280", \
|
||||
.cra_driver_name = "__"driver_name, \
|
||||
\
|
||||
.cra_module = THIS_MODULE, \
|
||||
} \
|
||||
}
|
||||
|
||||
#endif /* _CRYPTO_MORUS1280_GLUE_H */
|
||||
@ -1,97 +0,0 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0-or-later */
|
||||
/*
|
||||
* The MORUS-640 Authenticated-Encryption Algorithm
|
||||
* Common glue skeleton -- header file
|
||||
*
|
||||
* Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
|
||||
* Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef _CRYPTO_MORUS640_GLUE_H
|
||||
#define _CRYPTO_MORUS640_GLUE_H
|
||||
|
||||
#include <linux/module.h>
|
||||
#include <linux/types.h>
|
||||
#include <crypto/algapi.h>
|
||||
#include <crypto/aead.h>
|
||||
#include <crypto/morus_common.h>
|
||||
|
||||
#define MORUS640_WORD_SIZE 4
|
||||
#define MORUS640_BLOCK_SIZE (MORUS_BLOCK_WORDS * MORUS640_WORD_SIZE)
|
||||
|
||||
struct morus640_block {
|
||||
u8 bytes[MORUS640_BLOCK_SIZE];
|
||||
};
|
||||
|
||||
struct morus640_glue_ops {
|
||||
void (*init)(void *state, const void *key, const void *iv);
|
||||
void (*ad)(void *state, const void *data, unsigned int length);
|
||||
void (*enc)(void *state, const void *src, void *dst, unsigned int length);
|
||||
void (*dec)(void *state, const void *src, void *dst, unsigned int length);
|
||||
void (*enc_tail)(void *state, const void *src, void *dst, unsigned int length);
|
||||
void (*dec_tail)(void *state, const void *src, void *dst, unsigned int length);
|
||||
void (*final)(void *state, void *tag_xor, u64 assoclen, u64 cryptlen);
|
||||
};
|
||||
|
||||
struct morus640_ctx {
|
||||
const struct morus640_glue_ops *ops;
|
||||
struct morus640_block key;
|
||||
};
|
||||
|
||||
void crypto_morus640_glue_init_ops(struct crypto_aead *aead,
|
||||
const struct morus640_glue_ops *ops);
|
||||
int crypto_morus640_glue_setkey(struct crypto_aead *aead, const u8 *key,
|
||||
unsigned int keylen);
|
||||
int crypto_morus640_glue_setauthsize(struct crypto_aead *tfm,
|
||||
unsigned int authsize);
|
||||
int crypto_morus640_glue_encrypt(struct aead_request *req);
|
||||
int crypto_morus640_glue_decrypt(struct aead_request *req);
|
||||
|
||||
#define MORUS640_DECLARE_ALG(id, driver_name, priority) \
|
||||
static const struct morus640_glue_ops crypto_morus640_##id##_ops = {\
|
||||
.init = crypto_morus640_##id##_init, \
|
||||
.ad = crypto_morus640_##id##_ad, \
|
||||
.enc = crypto_morus640_##id##_enc, \
|
||||
.enc_tail = crypto_morus640_##id##_enc_tail, \
|
||||
.dec = crypto_morus640_##id##_dec, \
|
||||
.dec_tail = crypto_morus640_##id##_dec_tail, \
|
||||
.final = crypto_morus640_##id##_final, \
|
||||
}; \
|
||||
\
|
||||
static int crypto_morus640_##id##_init_tfm(struct crypto_aead *tfm) \
|
||||
{ \
|
||||
crypto_morus640_glue_init_ops(tfm, &crypto_morus640_##id##_ops); \
|
||||
return 0; \
|
||||
} \
|
||||
\
|
||||
static void crypto_morus640_##id##_exit_tfm(struct crypto_aead *tfm) \
|
||||
{ \
|
||||
} \
|
||||
\
|
||||
static struct aead_alg crypto_morus640_##id##_alg = {\
|
||||
.setkey = crypto_morus640_glue_setkey, \
|
||||
.setauthsize = crypto_morus640_glue_setauthsize, \
|
||||
.encrypt = crypto_morus640_glue_encrypt, \
|
||||
.decrypt = crypto_morus640_glue_decrypt, \
|
||||
.init = crypto_morus640_##id##_init_tfm, \
|
||||
.exit = crypto_morus640_##id##_exit_tfm, \
|
||||
\
|
||||
.ivsize = MORUS_NONCE_SIZE, \
|
||||
.maxauthsize = MORUS_MAX_AUTH_SIZE, \
|
||||
.chunksize = MORUS640_BLOCK_SIZE, \
|
||||
\
|
||||
.base = { \
|
||||
.cra_flags = CRYPTO_ALG_INTERNAL, \
|
||||
.cra_blocksize = 1, \
|
||||
.cra_ctxsize = sizeof(struct morus640_ctx), \
|
||||
.cra_alignmask = 0, \
|
||||
.cra_priority = priority, \
|
||||
\
|
||||
.cra_name = "__morus640", \
|
||||
.cra_driver_name = "__"driver_name, \
|
||||
\
|
||||
.cra_module = THIS_MODULE, \
|
||||
} \
|
||||
}
|
||||
|
||||
#endif /* _CRYPTO_MORUS640_GLUE_H */
|
||||
@ -1,18 +0,0 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0-or-later */
|
||||
/*
|
||||
* The MORUS Authenticated-Encryption Algorithm
|
||||
* Common definitions
|
||||
*
|
||||
* Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
|
||||
* Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef _CRYPTO_MORUS_COMMON_H
|
||||
#define _CRYPTO_MORUS_COMMON_H
|
||||
|
||||
#define MORUS_BLOCK_WORDS 4
|
||||
#define MORUS_STATE_BLOCKS 5
|
||||
#define MORUS_NONCE_SIZE 16
|
||||
#define MORUS_MAX_AUTH_SIZE 16
|
||||
|
||||
#endif /* _CRYPTO_MORUS_COMMON_H */
|
||||
Loading…
Reference in New Issue
Block a user