Merge branch 'x86-cpufeature-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...

[cascardo/linux.git] / arch / x86 / crypto / aesni-intel_glue.c

diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index 948ad0e..888950f 100644 (file)
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -105,6 +105,9 @@ void crypto_fpu_exit(void);
 #define AVX_GEN4_OPTSIZE 4096
 
 #ifdef CONFIG_X86_64
+
+static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out,
+                             const u8 *in, unsigned int len, u8 *iv);
 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
                              const u8 *in, unsigned int len, u8 *iv);
 
@@ -155,6 +158,12 @@ asmlinkage void aesni_gcm_dec(void *ctx, u8 *out,
 
 
 #ifdef CONFIG_AS_AVX
+asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
+               void *keys, u8 *out, unsigned int num_bytes);
+asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
+               void *keys, u8 *out, unsigned int num_bytes);
+asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
+               void *keys, u8 *out, unsigned int num_bytes);
 /*
  * asmlinkage void aesni_gcm_precomp_avx_gen2()
  * gcm_data *my_ctx_data, context data
@@ -472,6 +481,25 @@ static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
        crypto_inc(ctrblk, AES_BLOCK_SIZE);
 }
 
+#ifdef CONFIG_AS_AVX
+static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
+                             const u8 *in, unsigned int len, u8 *iv)
+{
+       /*
+        * based on key length, override with the by8 version
+        * of ctr mode encryption/decryption for improved performance
+        * aes_set_key_common() ensures that key length is one of
+        * {128,192,256}
+        */
+       if (ctx->key_length == AES_KEYSIZE_128)
+               aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len);
+       else if (ctx->key_length == AES_KEYSIZE_192)
+               aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len);
+       else
+               aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);
+}
+#endif
+
 static int ctr_crypt(struct blkcipher_desc *desc,
                     struct scatterlist *dst, struct scatterlist *src,
                     unsigned int nbytes)
@@ -486,8 +514,8 @@ static int ctr_crypt(struct blkcipher_desc *desc,
 
        kernel_fpu_begin();
        while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
-               aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
-                             nbytes & AES_BLOCK_MASK, walk.iv);
+               aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr,
+                                 nbytes & AES_BLOCK_MASK, walk.iv);
                nbytes &= AES_BLOCK_SIZE - 1;
                err = blkcipher_walk_done(desc, &walk, nbytes);
        }
@@ -1493,6 +1521,14 @@ static int __init aesni_init(void)
                aesni_gcm_enc_tfm = aesni_gcm_enc;
                aesni_gcm_dec_tfm = aesni_gcm_dec;
        }
+       aesni_ctr_enc_tfm = aesni_ctr_enc;
+#ifdef CONFIG_AS_AVX
+       if (cpu_has_avx) {
+               /* optimize performance of ctr mode encryption transform */
+               aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm;
+               pr_info("AES CTR mode by8 optimization enabled\n");
+       }
+#endif
 #endif
 
        err = crypto_fpu_init();