Techwizz/kernel-fxtec-pro1x
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

UPSTREAM: crypto: x86/poly1305 - expose existing driver as poly1305 library

Browse source 
Implement the arch init/update/final Poly1305 library routines in the
accelerated SIMD driver for x86 so they are accessible to users of
the Poly1305 library interface as well.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
(cherry picked from commit f0e89bcfbb894e5844cd1bbf6b3cf7c63cb0f5ac)
Bug: 152722841
Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@google.com>
Change-Id: Ib675528205383a4816e2a01f68e89eba2b0abb4d
This commit is contained in:
Ard Biesheuvel 2019-11-08 13:22:23 +01:00 committed by Greg Kroah-Hartman
parent 48c9cbd789
commit 4e42c8cac2
3 changed files with 42 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
arch/x86/crypto/poly1305_glue.c
View file

@ -13,6 +13,7 @@
#include <crypto/internal/hash.h>
#include <crypto/internal/poly1305.h>
#include <linux/crypto.h>
#include <linux/jump_label.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <asm/fpu/api.h>
@ -25,7 +26,8 @@ asmlinkage void poly1305_2block_sse2(u32 *h, const u8 *src, const u32 *r,
asmlinkage void poly1305_4block_avx2(u32 *h, const u8 *src, const u32 *r,
unsigned int blocks, const u32 *u);
static bool poly1305_use_avx2 __ro_after_init;
static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_simd);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx2);
static void poly1305_simd_mult(u32 *a, const u32 *b)
{
@ -68,7 +70,7 @@ static unsigned int poly1305_simd_blocks(struct poly1305_desc_ctx *dctx,
}
if (IS_ENABLED(CONFIG_AS_AVX2) &&
poly1305_use_avx2 &&
static_branch_likely(&poly1305_use_avx2) &&
srclen >= POLY1305_BLOCK_SIZE * 4) {
if (unlikely(dctx->rset < 4)) {
if (dctx->rset < 2) {
@ -107,10 +109,15 @@ static unsigned int poly1305_simd_blocks(struct poly1305_desc_ctx *dctx,
return srclen;
}
static int poly1305_simd_update(struct shash_desc *desc,
const u8 *src, unsigned int srclen)
void poly1305_init_arch(struct poly1305_desc_ctx *desc, const u8 *key)
{
poly1305_init_generic(desc, key);
}
EXPORT_SYMBOL(poly1305_init_arch);
void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
unsigned int srclen)
{
struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
unsigned int bytes;
if (unlikely(dctx->buflen)) {
@ -121,7 +128,8 @@ static int poly1305_simd_update(struct shash_desc *desc,
dctx->buflen += bytes;
if (dctx->buflen == POLY1305_BLOCK_SIZE) {
if (likely(may_use_simd())) {
if (static_branch_likely(&poly1305_use_simd) &&
likely(may_use_simd())) {
kernel_fpu_begin();
poly1305_simd_blocks(dctx, dctx->buf,
POLY1305_BLOCK_SIZE);
@ -135,7 +143,8 @@ static int poly1305_simd_update(struct shash_desc *desc,
}
if (likely(srclen >= POLY1305_BLOCK_SIZE)) {
if (likely(may_use_simd())) {
if (static_branch_likely(&poly1305_use_simd) &&
likely(may_use_simd())) {
kernel_fpu_begin();
bytes = poly1305_simd_blocks(dctx, src, srclen);
kernel_fpu_end();
@ -151,6 +160,13 @@ static int poly1305_simd_update(struct shash_desc *desc,
memcpy(dctx->buf, src, srclen);
}
}
EXPORT_SYMBOL(poly1305_update_arch);
void poly1305_final_arch(struct poly1305_desc_ctx *desc, u8 *digest)
{
poly1305_final_generic(desc, digest);
}
EXPORT_SYMBOL(poly1305_final_arch);
static int crypto_poly1305_init(struct shash_desc *desc)
{
@ -175,6 +191,15 @@ static int crypto_poly1305_final(struct shash_desc *desc, u8 *dst)
return 0;
}
static int poly1305_simd_update(struct shash_desc *desc,
const u8 *src, unsigned int srclen)
{
struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
poly1305_update_arch(dctx, src, srclen);
return 0;
}
static struct shash_alg alg = {
.digestsize = POLY1305_DIGEST_SIZE,
.init = crypto_poly1305_init,
@ -193,15 +218,15 @@ static struct shash_alg alg = {
static int __init poly1305_simd_mod_init(void)
{
if (!boot_cpu_has(X86_FEATURE_XMM2))
return -ENODEV;
return 0;
poly1305_use_avx2 = IS_ENABLED(CONFIG_AS_AVX2) &&
boot_cpu_has(X86_FEATURE_AVX) &&
boot_cpu_has(X86_FEATURE_AVX2) &&
cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL);
alg.descsize = sizeof(struct poly1305_desc_ctx) + 5 * sizeof(u32);
if (poly1305_use_avx2)
alg.descsize += 10 * sizeof(u32);
static_branch_enable(&poly1305_use_simd);
if (IS_ENABLED(CONFIG_AS_AVX2) &&
boot_cpu_has(X86_FEATURE_AVX) &&
boot_cpu_has(X86_FEATURE_AVX2) &&
cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
static_branch_enable(&poly1305_use_avx2);
return crypto_register_shash(&alg);
}

1
crypto/Kconfig
View file

@ -696,6 +696,7 @@ config CRYPTO_POLY1305_X86_64
tristate "Poly1305 authenticator algorithm (x86_64/SSE2/AVX2)"
depends on X86 && 64BIT
select CRYPTO_LIB_POLY1305_GENERIC
select CRYPTO_ARCH_HAVE_LIB_POLY1305
help
Poly1305 authenticator algorithm, RFC7539.

1
lib/crypto/Kconfig
View file

@ -30,6 +30,7 @@ config CRYPTO_LIB_CHACHA
config CRYPTO_LIB_POLY1305_RSIZE
int
default 4 if X86_64
default 1
config CRYPTO_ARCH_HAVE_LIB_POLY1305