arch/x86/crypto/aesni-intel_glue.c

   1 /*
   2  * Support for Intel AES-NI instructions. This file contains glue
   3  * code, the real AES implementation is in intel-aes_asm.S.
   4  *
   5  * Copyright (C) 2008, Intel Corp.
   6  *    Author: Huang Ying <ying.huang@intel.com>
   7  *
   8  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
   9  * interface for 64-bit kernels.
  10  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
  11  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
  12  *             Tadeusz Struk (tadeusz.struk@intel.com)
  13  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
  14  *    Copyright (c) 2010, Intel Corporation.
  15  *
  16  * This program is free software; you can redistribute it and/or modify
  17  * it under the terms of the GNU General Public License as published by
  18  * the Free Software Foundation; either version 2 of the License, or
  19  * (at your option) any later version.
  20  */
  21
  22 #include <linux/hardirq.h>
  23 #include <linux/types.h>
  24 #include <linux/crypto.h>
  25 #include <linux/module.h>
  26 #include <linux/err.h>
  27 #include <crypto/algapi.h>
  28 #include <crypto/aes.h>
  29 #include <crypto/cryptd.h>
  30 #include <crypto/ctr.h>
  31 #include <crypto/b128ops.h>
  32 #include <crypto/lrw.h>
  33 #include <crypto/xts.h>
  34 #include <asm/cpu_device_id.h>
  35 #include <asm/i387.h>
  36 #include <asm/crypto/aes.h>
  37 #include <crypto/ablk_helper.h>
  38 #include <crypto/scatterwalk.h>
  39 #include <crypto/internal/aead.h>
  40 #include <linux/workqueue.h>
  41 #include <linux/spinlock.h>
  42 #ifdef CONFIG_X86_64
  43 #include <asm/crypto/glue_helper.h>
  44 #endif
  45
  46 /* This data is stored at the end of the crypto_tfm struct.
  47  * It's a type of per "session" data storage location.
  48  * This needs to be 16 byte aligned.
  49  */
  50 struct aesni_rfc4106_gcm_ctx {
  51         u8 hash_subkey[16];
  52         struct crypto_aes_ctx aes_key_expanded;
  53         u8 nonce[4];
  54         struct cryptd_aead *cryptd_tfm;
  55 };
  56
  57 struct aesni_gcm_set_hash_subkey_result {
  58         int err;
  59         struct completion completion;
  60 };
  61
  62 struct aesni_hash_subkey_req_data {
  63         u8 iv[16];
  64         struct aesni_gcm_set_hash_subkey_result result;
  65         struct scatterlist sg;
  66 };
  67
  68 #define AESNI_ALIGN     (16)
  69 #define AES_BLOCK_MASK  (~(AES_BLOCK_SIZE-1))
  70 #define RFC4106_HASH_SUBKEY_SIZE 16
  71
  72 struct aesni_lrw_ctx {
  73         struct lrw_table_ctx lrw_table;
  74         u8 raw_aes_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1];
  75 };
  76
  77 struct aesni_xts_ctx {
  78         u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1];
  79         u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1];
  80 };
  81
  82 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
  83                              unsigned int key_len);
  84 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
  85                           const u8 *in);
  86 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
  87                           const u8 *in);
  88 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
  89                               const u8 *in, unsigned int len);
  90 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
  91                               const u8 *in, unsigned int len);
  92 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
  93                               const u8 *in, unsigned int len, u8 *iv);
  94 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
  95                               const u8 *in, unsigned int len, u8 *iv);
  96
  97 int crypto_fpu_init(void);
  98 void crypto_fpu_exit(void);
  99
 100 #define AVX_GEN2_OPTSIZE 640
 101 #define AVX_GEN4_OPTSIZE 4096
 102
 103 #ifdef CONFIG_X86_64
 104
 105 static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out,
 106                               const u8 *in, unsigned int len, u8 *iv);
 107 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
 108                               const u8 *in, unsigned int len, u8 *iv);
 109
 110 asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out,
 111                                  const u8 *in, bool enc, u8 *iv);
 112
 113 /* asmlinkage void aesni_gcm_enc()
 114  * void *ctx,  AES Key schedule. Starts on a 16 byte boundary.
 115  * u8 *out, Ciphertext output. Encrypt in-place is allowed.
 116  * const u8 *in, Plaintext input
 117  * unsigned long plaintext_len, Length of data in bytes for encryption.
 118  * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
 119  *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
 120  *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
 121  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 122  * const u8 *aad, Additional Authentication Data (AAD)
 123  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this
 124  *          is going to be 8 or 12 bytes
 125  * u8 *auth_tag, Authenticated Tag output.
 126  * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
 127  *          Valid values are 16 (most likely), 12 or 8.
 128  */
 129 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out,
 130                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 131                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 132                         u8 *auth_tag, unsigned long auth_tag_len);
 133
 134 /* asmlinkage void aesni_gcm_dec()
 135  * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
 136  * u8 *out, Plaintext output. Decrypt in-place is allowed.
 137  * const u8 *in, Ciphertext input
 138  * unsigned long ciphertext_len, Length of data in bytes for decryption.
 139  * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
 140  *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
 141  *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
 142  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 143  * const u8 *aad, Additional Authentication Data (AAD)
 144  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
 145  * to be 8 or 12 bytes
 146  * u8 *auth_tag, Authenticated Tag output.
 147  * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
 148  * Valid values are 16 (most likely), 12 or 8.
 149  */
 150 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out,
 151                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 152                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 153                         u8 *auth_tag, unsigned long auth_tag_len);
 154
 155
 156 #ifdef CONFIG_AS_AVX
 157 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
 158                 void *keys, u8 *out, unsigned int num_bytes);
 159 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
 160                 void *keys, u8 *out, unsigned int num_bytes);
 161 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
 162                 void *keys, u8 *out, unsigned int num_bytes);
 163 /*
 164  * asmlinkage void aesni_gcm_precomp_avx_gen2()
 165  * gcm_data *my_ctx_data, context data
 166  * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
 167  */
 168 asmlinkage void aesni_gcm_precomp_avx_gen2(void *my_ctx_data, u8 *hash_subkey);
 169
 170 asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx, u8 *out,
 171                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 172                         const u8 *aad, unsigned long aad_len,
 173                         u8 *auth_tag, unsigned long auth_tag_len);
 174
 175 asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx, u8 *out,
 176                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 177                         const u8 *aad, unsigned long aad_len,
 178                         u8 *auth_tag, unsigned long auth_tag_len);
 179
 180 static void aesni_gcm_enc_avx(void *ctx, u8 *out,
 181                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 182                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 183                         u8 *auth_tag, unsigned long auth_tag_len)
 184 {
 185         if (plaintext_len < AVX_GEN2_OPTSIZE) {
 186                 aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad,
 187                                 aad_len, auth_tag, auth_tag_len);
 188         } else {
 189                 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
 190                 aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad,
 191                                         aad_len, auth_tag, auth_tag_len);
 192         }
 193 }
 194
 195 static void aesni_gcm_dec_avx(void *ctx, u8 *out,
 196                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 197                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 198                         u8 *auth_tag, unsigned long auth_tag_len)
 199 {
 200         if (ciphertext_len < AVX_GEN2_OPTSIZE) {
 201                 aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey, aad,
 202                                 aad_len, auth_tag, auth_tag_len);
 203         } else {
 204                 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
 205                 aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad,
 206                                         aad_len, auth_tag, auth_tag_len);
 207         }
 208 }
 209 #endif
 210
 211 #ifdef CONFIG_AS_AVX2
 212 /*
 213  * asmlinkage void aesni_gcm_precomp_avx_gen4()
 214  * gcm_data *my_ctx_data, context data
 215  * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
 216  */
 217 asmlinkage void aesni_gcm_precomp_avx_gen4(void *my_ctx_data, u8 *hash_subkey);
 218
 219 asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx, u8 *out,
 220                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 221                         const u8 *aad, unsigned long aad_len,
 222                         u8 *auth_tag, unsigned long auth_tag_len);
 223
 224 asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx, u8 *out,
 225                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 226                         const u8 *aad, unsigned long aad_len,
 227                         u8 *auth_tag, unsigned long auth_tag_len);
 228
 229 static void aesni_gcm_enc_avx2(void *ctx, u8 *out,
 230                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 231                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 232                         u8 *auth_tag, unsigned long auth_tag_len)
 233 {
 234         if (plaintext_len < AVX_GEN2_OPTSIZE) {
 235                 aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad,
 236                                 aad_len, auth_tag, auth_tag_len);
 237         } else if (plaintext_len < AVX_GEN4_OPTSIZE) {
 238                 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
 239                 aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad,
 240                                         aad_len, auth_tag, auth_tag_len);
 241         } else {
 242                 aesni_gcm_precomp_avx_gen4(ctx, hash_subkey);
 243                 aesni_gcm_enc_avx_gen4(ctx, out, in, plaintext_len, iv, aad,
 244                                         aad_len, auth_tag, auth_tag_len);
 245         }
 246 }
 247
 248 static void aesni_gcm_dec_avx2(void *ctx, u8 *out,
 249                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 250                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 251                         u8 *auth_tag, unsigned long auth_tag_len)
 252 {
 253         if (ciphertext_len < AVX_GEN2_OPTSIZE) {
 254                 aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey,
 255                                 aad, aad_len, auth_tag, auth_tag_len);
 256         } else if (ciphertext_len < AVX_GEN4_OPTSIZE) {
 257                 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
 258                 aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad,
 259                                         aad_len, auth_tag, auth_tag_len);
 260         } else {
 261                 aesni_gcm_precomp_avx_gen4(ctx, hash_subkey);
 262                 aesni_gcm_dec_avx_gen4(ctx, out, in, ciphertext_len, iv, aad,
 263                                         aad_len, auth_tag, auth_tag_len);
 264         }
 265 }
 266 #endif
 267
 268 static void (*aesni_gcm_enc_tfm)(void *ctx, u8 *out,
 269                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 270                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 271                         u8 *auth_tag, unsigned long auth_tag_len);
 272
 273 static void (*aesni_gcm_dec_tfm)(void *ctx, u8 *out,
 274                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 275                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 276                         u8 *auth_tag, unsigned long auth_tag_len);
 277
 278 static inline struct
 279 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
 280 {
 281         return
 282                 (struct aesni_rfc4106_gcm_ctx *)
 283                 PTR_ALIGN((u8 *)
 284                 crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN);
 285 }
 286 #endif
 287
 288 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
 289 {
 290         unsigned long addr = (unsigned long)raw_ctx;
 291         unsigned long align = AESNI_ALIGN;
 292
 293         if (align <= crypto_tfm_ctx_alignment())
 294                 align = 1;
 295         return (struct crypto_aes_ctx *)ALIGN(addr, align);
 296 }
 297
 298 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
 299                               const u8 *in_key, unsigned int key_len)
 300 {
 301         struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
 302         u32 *flags = &tfm->crt_flags;
 303         int err;
 304
 305         if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
 306             key_len != AES_KEYSIZE_256) {
 307                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 308                 return -EINVAL;
 309         }
 310
 311         if (!irq_fpu_usable())
 312                 err = crypto_aes_expand_key(ctx, in_key, key_len);
 313         else {
 314                 kernel_fpu_begin();
 315                 err = aesni_set_key(ctx, in_key, key_len);
 316                 kernel_fpu_end();
 317         }
 318
 319         return err;
 320 }
 321
 322 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 323                        unsigned int key_len)
 324 {
 325         return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
 326 }
 327
 328 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 329 {
 330         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 331
 332         if (!irq_fpu_usable())
 333                 crypto_aes_encrypt_x86(ctx, dst, src);
 334         else {
 335                 kernel_fpu_begin();
 336                 aesni_enc(ctx, dst, src);
 337                 kernel_fpu_end();
 338         }
 339 }
 340
 341 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 342 {
 343         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 344
 345         if (!irq_fpu_usable())
 346                 crypto_aes_decrypt_x86(ctx, dst, src);
 347         else {
 348                 kernel_fpu_begin();
 349                 aesni_dec(ctx, dst, src);
 350                 kernel_fpu_end();
 351         }
 352 }
 353
 354 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 355 {
 356         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 357
 358         aesni_enc(ctx, dst, src);
 359 }
 360
 361 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 362 {
 363         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 364
 365         aesni_dec(ctx, dst, src);
 366 }
 367
 368 static int ecb_encrypt(struct blkcipher_desc *desc,
 369                        struct scatterlist *dst, struct scatterlist *src,
 370                        unsigned int nbytes)
 371 {
 372         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 373         struct blkcipher_walk walk;
 374         int err;
 375
 376         blkcipher_walk_init(&walk, dst, src, nbytes);
 377         err = blkcipher_walk_virt(desc, &walk);
 378         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 379
 380         kernel_fpu_begin();
 381         while ((nbytes = walk.nbytes)) {
 382                 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 383                               nbytes & AES_BLOCK_MASK);
 384                 nbytes &= AES_BLOCK_SIZE - 1;
 385                 err = blkcipher_walk_done(desc, &walk, nbytes);
 386         }
 387         kernel_fpu_end();
 388
 389         return err;
 390 }
 391
 392 static int ecb_decrypt(struct blkcipher_desc *desc,
 393                        struct scatterlist *dst, struct scatterlist *src,
 394                        unsigned int nbytes)
 395 {
 396         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 397         struct blkcipher_walk walk;
 398         int err;
 399
 400         blkcipher_walk_init(&walk, dst, src, nbytes);
 401         err = blkcipher_walk_virt(desc, &walk);
 402         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 403
 404         kernel_fpu_begin();
 405         while ((nbytes = walk.nbytes)) {
 406                 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 407                               nbytes & AES_BLOCK_MASK);
 408                 nbytes &= AES_BLOCK_SIZE - 1;
 409                 err = blkcipher_walk_done(desc, &walk, nbytes);
 410         }
 411         kernel_fpu_end();
 412
 413         return err;
 414 }
 415
 416 static int cbc_encrypt(struct blkcipher_desc *desc,
 417                        struct scatterlist *dst, struct scatterlist *src,
 418                        unsigned int nbytes)
 419 {
 420         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 421         struct blkcipher_walk walk;
 422         int err;
 423
 424         blkcipher_walk_init(&walk, dst, src, nbytes);
 425         err = blkcipher_walk_virt(desc, &walk);
 426         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 427
 428         kernel_fpu_begin();
 429         while ((nbytes = walk.nbytes)) {
 430                 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 431                               nbytes & AES_BLOCK_MASK, walk.iv);
 432                 nbytes &= AES_BLOCK_SIZE - 1;
 433                 err = blkcipher_walk_done(desc, &walk, nbytes);
 434         }
 435         kernel_fpu_end();
 436
 437         return err;
 438 }
 439
 440 static int cbc_decrypt(struct blkcipher_desc *desc,
 441                        struct scatterlist *dst, struct scatterlist *src,
 442                        unsigned int nbytes)
 443 {
 444         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 445         struct blkcipher_walk walk;
 446         int err;
 447
 448         blkcipher_walk_init(&walk, dst, src, nbytes);
 449         err = blkcipher_walk_virt(desc, &walk);
 450         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 451
 452         kernel_fpu_begin();
 453         while ((nbytes = walk.nbytes)) {
 454                 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 455                               nbytes & AES_BLOCK_MASK, walk.iv);
 456                 nbytes &= AES_BLOCK_SIZE - 1;
 457                 err = blkcipher_walk_done(desc, &walk, nbytes);
 458         }
 459         kernel_fpu_end();
 460
 461         return err;
 462 }
 463
 464 #ifdef CONFIG_X86_64
 465 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
 466                             struct blkcipher_walk *walk)
 467 {
 468         u8 *ctrblk = walk->iv;
 469         u8 keystream[AES_BLOCK_SIZE];
 470         u8 *src = walk->src.virt.addr;
 471         u8 *dst = walk->dst.virt.addr;
 472         unsigned int nbytes = walk->nbytes;
 473
 474         aesni_enc(ctx, keystream, ctrblk);
 475         crypto_xor(keystream, src, nbytes);
 476         memcpy(dst, keystream, nbytes);
 477         crypto_inc(ctrblk, AES_BLOCK_SIZE);
 478 }
 479
 480 #ifdef CONFIG_AS_AVX
 481 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
 482                               const u8 *in, unsigned int len, u8 *iv)
 483 {
 484         /*
 485          * based on key length, override with the by8 version
 486          * of ctr mode encryption/decryption for improved performance
 487          * aes_set_key_common() ensures that key length is one of
 488          * {128,192,256}
 489          */
 490         if (ctx->key_length == AES_KEYSIZE_128)
 491                 aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len);
 492         else if (ctx->key_length == AES_KEYSIZE_192)
 493                 aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len);
 494         else
 495                 aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);
 496 }
 497 #endif
 498
 499 static int ctr_crypt(struct blkcipher_desc *desc,
 500                      struct scatterlist *dst, struct scatterlist *src,
 501                      unsigned int nbytes)
 502 {
 503         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 504         struct blkcipher_walk walk;
 505         int err;
 506
 507         blkcipher_walk_init(&walk, dst, src, nbytes);
 508         err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
 509         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 510
 511         kernel_fpu_begin();
 512         while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
 513                 aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 514                                   nbytes & AES_BLOCK_MASK, walk.iv);
 515                 nbytes &= AES_BLOCK_SIZE - 1;
 516                 err = blkcipher_walk_done(desc, &walk, nbytes);
 517         }
 518         if (walk.nbytes) {
 519                 ctr_crypt_final(ctx, &walk);
 520                 err = blkcipher_walk_done(desc, &walk, 0);
 521         }
 522         kernel_fpu_end();
 523
 524         return err;
 525 }
 526 #endif
 527
 528 static int ablk_ecb_init(struct crypto_tfm *tfm)
 529 {
 530         return ablk_init_common(tfm, "__driver-ecb-aes-aesni");
 531 }
 532
 533 static int ablk_cbc_init(struct crypto_tfm *tfm)
 534 {
 535         return ablk_init_common(tfm, "__driver-cbc-aes-aesni");
 536 }
 537
 538 #ifdef CONFIG_X86_64
 539 static int ablk_ctr_init(struct crypto_tfm *tfm)
 540 {
 541         return ablk_init_common(tfm, "__driver-ctr-aes-aesni");
 542 }
 543
 544 #endif
 545
 546 #if IS_ENABLED(CONFIG_CRYPTO_PCBC)
 547 static int ablk_pcbc_init(struct crypto_tfm *tfm)
 548 {
 549         return ablk_init_common(tfm, "fpu(pcbc(__driver-aes-aesni))");
 550 }
 551 #endif
 552
 553 static void lrw_xts_encrypt_callback(void *ctx, u8 *blks, unsigned int nbytes)
 554 {
 555         aesni_ecb_enc(ctx, blks, blks, nbytes);
 556 }
 557
 558 static void lrw_xts_decrypt_callback(void *ctx, u8 *blks, unsigned int nbytes)
 559 {
 560         aesni_ecb_dec(ctx, blks, blks, nbytes);
 561 }
 562
 563 static int lrw_aesni_setkey(struct crypto_tfm *tfm, const u8 *key,
 564                             unsigned int keylen)
 565 {
 566         struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
 567         int err;
 568
 569         err = aes_set_key_common(tfm, ctx->raw_aes_ctx, key,
 570                                  keylen - AES_BLOCK_SIZE);
 571         if (err)
 572                 return err;
 573
 574         return lrw_init_table(&ctx->lrw_table, key + keylen - AES_BLOCK_SIZE);
 575 }
 576
 577 static void lrw_aesni_exit_tfm(struct crypto_tfm *tfm)
 578 {
 579         struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
 580
 581         lrw_free_table(&ctx->lrw_table);
 582 }
 583
 584 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 585                        struct scatterlist *src, unsigned int nbytes)
 586 {
 587         struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 588         be128 buf[8];
 589         struct lrw_crypt_req req = {
 590                 .tbuf = buf,
 591                 .tbuflen = sizeof(buf),
 592
 593                 .table_ctx = &ctx->lrw_table,
 594                 .crypt_ctx = aes_ctx(ctx->raw_aes_ctx),
 595                 .crypt_fn = lrw_xts_encrypt_callback,
 596         };
 597         int ret;
 598
 599         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 600
 601         kernel_fpu_begin();
 602         ret = lrw_crypt(desc, dst, src, nbytes, &req);
 603         kernel_fpu_end();
 604
 605         return ret;
 606 }
 607
 608 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 609                        struct scatterlist *src, unsigned int nbytes)
 610 {
 611         struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 612         be128 buf[8];
 613         struct lrw_crypt_req req = {
 614                 .tbuf = buf,
 615                 .tbuflen = sizeof(buf),
 616
 617                 .table_ctx = &ctx->lrw_table,
 618                 .crypt_ctx = aes_ctx(ctx->raw_aes_ctx),
 619                 .crypt_fn = lrw_xts_decrypt_callback,
 620         };
 621         int ret;
 622
 623         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 624
 625         kernel_fpu_begin();
 626         ret = lrw_crypt(desc, dst, src, nbytes, &req);
 627         kernel_fpu_end();
 628
 629         return ret;
 630 }
 631
 632 static int xts_aesni_setkey(struct crypto_tfm *tfm, const u8 *key,
 633                             unsigned int keylen)
 634 {
 635         struct aesni_xts_ctx *ctx = crypto_tfm_ctx(tfm);
 636         u32 *flags = &tfm->crt_flags;
 637         int err;
 638
 639         /* key consists of keys of equal size concatenated, therefore
 640          * the length must be even
 641          */
 642         if (keylen % 2) {
 643                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 644                 return -EINVAL;
 645         }
 646
 647         /* first half of xts-key is for crypt */
 648         err = aes_set_key_common(tfm, ctx->raw_crypt_ctx, key, keylen / 2);
 649         if (err)
 650                 return err;
 651
 652         /* second half of xts-key is for tweak */
 653         return aes_set_key_common(tfm, ctx->raw_tweak_ctx, key + keylen / 2,
 654                                   keylen / 2);
 655 }
 656
 657
 658 static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in)
 659 {
 660         aesni_enc(ctx, out, in);
 661 }
 662
 663 #ifdef CONFIG_X86_64
 664
 665 static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 666 {
 667         glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc));
 668 }
 669
 670 static void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 671 {
 672         glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec));
 673 }
 674
 675 static void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 676 {
 677         aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv);
 678 }
 679
 680 static void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 681 {
 682         aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv);
 683 }
 684
 685 static const struct common_glue_ctx aesni_enc_xts = {
 686         .num_funcs = 2,
 687         .fpu_blocks_limit = 1,
 688
 689         .funcs = { {
 690                 .num_blocks = 8,
 691                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc8) }
 692         }, {
 693                 .num_blocks = 1,
 694                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc) }
 695         } }
 696 };
 697
 698 static const struct common_glue_ctx aesni_dec_xts = {
 699         .num_funcs = 2,
 700         .fpu_blocks_limit = 1,
 701
 702         .funcs = { {
 703                 .num_blocks = 8,
 704                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec8) }
 705         }, {
 706                 .num_blocks = 1,
 707                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec) }
 708         } }
 709 };
 710
 711 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 712                        struct scatterlist *src, unsigned int nbytes)
 713 {
 714         struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 715
 716         return glue_xts_crypt_128bit(&aesni_enc_xts, desc, dst, src, nbytes,
 717                                      XTS_TWEAK_CAST(aesni_xts_tweak),
 718                                      aes_ctx(ctx->raw_tweak_ctx),
 719                                      aes_ctx(ctx->raw_crypt_ctx));
 720 }
 721
 722 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 723                        struct scatterlist *src, unsigned int nbytes)
 724 {
 725         struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 726
 727         return glue_xts_crypt_128bit(&aesni_dec_xts, desc, dst, src, nbytes,
 728                                      XTS_TWEAK_CAST(aesni_xts_tweak),
 729                                      aes_ctx(ctx->raw_tweak_ctx),
 730                                      aes_ctx(ctx->raw_crypt_ctx));
 731 }
 732
 733 #else
 734
 735 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 736                        struct scatterlist *src, unsigned int nbytes)
 737 {
 738         struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 739         be128 buf[8];
 740         struct xts_crypt_req req = {
 741                 .tbuf = buf,
 742                 .tbuflen = sizeof(buf),
 743
 744                 .tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),
 745                 .tweak_fn = aesni_xts_tweak,
 746                 .crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),
 747                 .crypt_fn = lrw_xts_encrypt_callback,
 748         };
 749         int ret;
 750
 751         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 752
 753         kernel_fpu_begin();
 754         ret = xts_crypt(desc, dst, src, nbytes, &req);
 755         kernel_fpu_end();
 756
 757         return ret;
 758 }
 759
 760 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 761                        struct scatterlist *src, unsigned int nbytes)
 762 {
 763         struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 764         be128 buf[8];
 765         struct xts_crypt_req req = {
 766                 .tbuf = buf,
 767                 .tbuflen = sizeof(buf),
 768
 769                 .tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),
 770                 .tweak_fn = aesni_xts_tweak,
 771                 .crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),
 772                 .crypt_fn = lrw_xts_decrypt_callback,
 773         };
 774         int ret;
 775
 776         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 777
 778         kernel_fpu_begin();
 779         ret = xts_crypt(desc, dst, src, nbytes, &req);
 780         kernel_fpu_end();
 781
 782         return ret;
 783 }
 784
 785 #endif
 786
 787 #ifdef CONFIG_X86_64
 788 static int rfc4106_init(struct crypto_tfm *tfm)
 789 {
 790         struct cryptd_aead *cryptd_tfm;
 791         struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *)
 792                 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
 793         struct crypto_aead *cryptd_child;
 794         struct aesni_rfc4106_gcm_ctx *child_ctx;
 795         cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0);
 796         if (IS_ERR(cryptd_tfm))
 797                 return PTR_ERR(cryptd_tfm);
 798
 799         cryptd_child = cryptd_aead_child(cryptd_tfm);
 800         child_ctx = aesni_rfc4106_gcm_ctx_get(cryptd_child);
 801         memcpy(child_ctx, ctx, sizeof(*ctx));
 802         ctx->cryptd_tfm = cryptd_tfm;
 803         tfm->crt_aead.reqsize = sizeof(struct aead_request)
 804                 + crypto_aead_reqsize(&cryptd_tfm->base);
 805         return 0;
 806 }
 807
 808 static void rfc4106_exit(struct crypto_tfm *tfm)
 809 {
 810         struct aesni_rfc4106_gcm_ctx *ctx =
 811                 (struct aesni_rfc4106_gcm_ctx *)
 812                 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
 813         if (!IS_ERR(ctx->cryptd_tfm))
 814                 cryptd_free_aead(ctx->cryptd_tfm);
 815         return;
 816 }
 817
 818 static void
 819 rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err)
 820 {
 821         struct aesni_gcm_set_hash_subkey_result *result = req->data;
 822
 823         if (err == -EINPROGRESS)
 824                 return;
 825         result->err = err;
 826         complete(&result->completion);
 827 }
 828
 829 static int
 830 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
 831 {
 832         struct crypto_ablkcipher *ctr_tfm;
 833         struct ablkcipher_request *req;
 834         int ret = -EINVAL;
 835         struct aesni_hash_subkey_req_data *req_data;
 836
 837         ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0);
 838         if (IS_ERR(ctr_tfm))
 839                 return PTR_ERR(ctr_tfm);
 840
 841         crypto_ablkcipher_clear_flags(ctr_tfm, ~0);
 842
 843         ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len);
 844         if (ret)
 845                 goto out_free_ablkcipher;
 846
 847         ret = -ENOMEM;
 848         req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL);
 849         if (!req)
 850                 goto out_free_ablkcipher;
 851
 852         req_data = kmalloc(sizeof(*req_data), GFP_KERNEL);
 853         if (!req_data)
 854                 goto out_free_request;
 855
 856         memset(req_data->iv, 0, sizeof(req_data->iv));
 857
 858         /* Clear the data in the hash sub key container to zero.*/
 859         /* We want to cipher all zeros to create the hash sub key. */
 860         memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
 861
 862         init_completion(&req_data->result.completion);
 863         sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE);
 864         ablkcipher_request_set_tfm(req, ctr_tfm);
 865         ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
 866                                         CRYPTO_TFM_REQ_MAY_BACKLOG,
 867                                         rfc4106_set_hash_subkey_done,
 868                                         &req_data->result);
 869
 870         ablkcipher_request_set_crypt(req, &req_data->sg,
 871                 &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv);
 872
 873         ret = crypto_ablkcipher_encrypt(req);
 874         if (ret == -EINPROGRESS || ret == -EBUSY) {
 875                 ret = wait_for_completion_interruptible
 876                         (&req_data->result.completion);
 877                 if (!ret)
 878                         ret = req_data->result.err;
 879         }
 880         kfree(req_data);
 881 out_free_request:
 882         ablkcipher_request_free(req);
 883 out_free_ablkcipher:
 884         crypto_free_ablkcipher(ctr_tfm);
 885         return ret;
 886 }
 887
 888 static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
 889                                                    unsigned int key_len)
 890 {
 891         int ret = 0;
 892         struct crypto_tfm *tfm = crypto_aead_tfm(parent);
 893         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
 894         struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
 895         struct aesni_rfc4106_gcm_ctx *child_ctx =
 896                                  aesni_rfc4106_gcm_ctx_get(cryptd_child);
 897         u8 *new_key_align, *new_key_mem = NULL;
 898
 899         if (key_len < 4) {
 900                 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 901                 return -EINVAL;
 902         }
 903         /*Account for 4 byte nonce at the end.*/
 904         key_len -= 4;
 905         if (key_len != AES_KEYSIZE_128) {
 906                 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 907                 return -EINVAL;
 908         }
 909
 910         memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
 911         /*This must be on a 16 byte boundary!*/
 912         if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN)
 913                 return -EINVAL;
 914
 915         if ((unsigned long)key % AESNI_ALIGN) {
 916                 /*key is not aligned: use an auxuliar aligned pointer*/
 917                 new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL);
 918                 if (!new_key_mem)
 919                         return -ENOMEM;
 920
 921                 new_key_align = PTR_ALIGN(new_key_mem, AESNI_ALIGN);
 922                 memcpy(new_key_align, key, key_len);
 923                 key = new_key_align;
 924         }
 925
 926         if (!irq_fpu_usable())
 927                 ret = crypto_aes_expand_key(&(ctx->aes_key_expanded),
 928                 key, key_len);
 929         else {
 930                 kernel_fpu_begin();
 931                 ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len);
 932                 kernel_fpu_end();
 933         }
 934         /*This must be on a 16 byte boundary!*/
 935         if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) {
 936                 ret = -EINVAL;
 937                 goto exit;
 938         }
 939         ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
 940         memcpy(child_ctx, ctx, sizeof(*ctx));
 941 exit:
 942         kfree(new_key_mem);
 943         return ret;
 944 }
 945
 946 /* This is the Integrity Check Value (aka the authentication tag length and can
 947  * be 8, 12 or 16 bytes long. */
 948 static int rfc4106_set_authsize(struct crypto_aead *parent,
 949                                 unsigned int authsize)
 950 {
 951         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
 952         struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
 953
 954         switch (authsize) {
 955         case 8:
 956         case 12:
 957         case 16:
 958                 break;
 959         default:
 960                 return -EINVAL;
 961         }
 962         crypto_aead_crt(parent)->authsize = authsize;
 963         crypto_aead_crt(cryptd_child)->authsize = authsize;
 964         return 0;
 965 }
 966
 967 static int rfc4106_encrypt(struct aead_request *req)
 968 {
 969         int ret;
 970         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 971         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
 972
 973         if (!irq_fpu_usable()) {
 974                 struct aead_request *cryptd_req =
 975                         (struct aead_request *) aead_request_ctx(req);
 976                 memcpy(cryptd_req, req, sizeof(*req));
 977                 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
 978                 return crypto_aead_encrypt(cryptd_req);
 979         } else {
 980                 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
 981                 kernel_fpu_begin();
 982                 ret = cryptd_child->base.crt_aead.encrypt(req);
 983                 kernel_fpu_end();
 984                 return ret;
 985         }
 986 }
 987
 988 static int rfc4106_decrypt(struct aead_request *req)
 989 {
 990         int ret;
 991         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 992         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
 993
 994         if (!irq_fpu_usable()) {
 995                 struct aead_request *cryptd_req =
 996                         (struct aead_request *) aead_request_ctx(req);
 997                 memcpy(cryptd_req, req, sizeof(*req));
 998                 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
 999                 return crypto_aead_decrypt(cryptd_req);
1000         } else {
1001                 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
1002                 kernel_fpu_begin();
1003                 ret = cryptd_child->base.crt_aead.decrypt(req);
1004                 kernel_fpu_end();
1005                 return ret;
1006         }
1007 }
1008
1009 static int __driver_rfc4106_encrypt(struct aead_request *req)
1010 {
1011         u8 one_entry_in_sg = 0;
1012         u8 *src, *dst, *assoc;
1013         __be32 counter = cpu_to_be32(1);
1014         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1015         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1016         void *aes_ctx = &(ctx->aes_key_expanded);
1017         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
1018         u8 iv_tab[16+AESNI_ALIGN];
1019         u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN);
1020         struct scatter_walk src_sg_walk;
1021         struct scatter_walk assoc_sg_walk;
1022         struct scatter_walk dst_sg_walk;
1023         unsigned int i;
1024
1025         /* Assuming we are supporting rfc4106 64-bit extended */
1026         /* sequence numbers We need to have the AAD length equal */
1027         /* to 8 or 12 bytes */
1028         if (unlikely(req->assoclen != 8 && req->assoclen != 12))
1029                 return -EINVAL;
1030         /* IV below built */
1031         for (i = 0; i < 4; i++)
1032                 *(iv+i) = ctx->nonce[i];
1033         for (i = 0; i < 8; i++)
1034                 *(iv+4+i) = req->iv[i];
1035         *((__be32 *)(iv+12)) = counter;
1036
1037         if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
1038                 one_entry_in_sg = 1;
1039                 scatterwalk_start(&src_sg_walk, req->src);
1040                 scatterwalk_start(&assoc_sg_walk, req->assoc);
1041                 src = scatterwalk_map(&src_sg_walk);
1042                 assoc = scatterwalk_map(&assoc_sg_walk);
1043                 dst = src;
1044                 if (unlikely(req->src != req->dst)) {
1045                         scatterwalk_start(&dst_sg_walk, req->dst);
1046                         dst = scatterwalk_map(&dst_sg_walk);
1047                 }
1048
1049         } else {
1050                 /* Allocate memory for src, dst, assoc */
1051                 src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
1052                         GFP_ATOMIC);
1053                 if (unlikely(!src))
1054                         return -ENOMEM;
1055                 assoc = (src + req->cryptlen + auth_tag_len);
1056                 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
1057                 scatterwalk_map_and_copy(assoc, req->assoc, 0,
1058                                         req->assoclen, 0);
1059                 dst = src;
1060         }
1061
1062         aesni_gcm_enc_tfm(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv,
1063                 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst
1064                 + ((unsigned long)req->cryptlen), auth_tag_len);
1065
1066         /* The authTag (aka the Integrity Check Value) needs to be written
1067          * back to the packet. */
1068         if (one_entry_in_sg) {
1069                 if (unlikely(req->src != req->dst)) {
1070                         scatterwalk_unmap(dst);
1071                         scatterwalk_done(&dst_sg_walk, 0, 0);
1072                 }
1073                 scatterwalk_unmap(src);
1074                 scatterwalk_unmap(assoc);
1075                 scatterwalk_done(&src_sg_walk, 0, 0);
1076                 scatterwalk_done(&assoc_sg_walk, 0, 0);
1077         } else {
1078                 scatterwalk_map_and_copy(dst, req->dst, 0,
1079                         req->cryptlen + auth_tag_len, 1);
1080                 kfree(src);
1081         }
1082         return 0;
1083 }
1084
1085 static int __driver_rfc4106_decrypt(struct aead_request *req)
1086 {
1087         u8 one_entry_in_sg = 0;
1088         u8 *src, *dst, *assoc;
1089         unsigned long tempCipherLen = 0;
1090         __be32 counter = cpu_to_be32(1);
1091         int retval = 0;
1092         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1093         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1094         void *aes_ctx = &(ctx->aes_key_expanded);
1095         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
1096         u8 iv_and_authTag[32+AESNI_ALIGN];
1097         u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN);
1098         u8 *authTag = iv + 16;
1099         struct scatter_walk src_sg_walk;
1100         struct scatter_walk assoc_sg_walk;
1101         struct scatter_walk dst_sg_walk;
1102         unsigned int i;
1103
1104         if (unlikely((req->cryptlen < auth_tag_len) ||
1105                 (req->assoclen != 8 && req->assoclen != 12)))
1106                 return -EINVAL;
1107         /* Assuming we are supporting rfc4106 64-bit extended */
1108         /* sequence numbers We need to have the AAD length */
1109         /* equal to 8 or 12 bytes */
1110
1111         tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
1112         /* IV below built */
1113         for (i = 0; i < 4; i++)
1114                 *(iv+i) = ctx->nonce[i];
1115         for (i = 0; i < 8; i++)
1116                 *(iv+4+i) = req->iv[i];
1117         *((__be32 *)(iv+12)) = counter;
1118
1119         if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
1120                 one_entry_in_sg = 1;
1121                 scatterwalk_start(&src_sg_walk, req->src);
1122                 scatterwalk_start(&assoc_sg_walk, req->assoc);
1123                 src = scatterwalk_map(&src_sg_walk);
1124                 assoc = scatterwalk_map(&assoc_sg_walk);
1125                 dst = src;
1126                 if (unlikely(req->src != req->dst)) {
1127                         scatterwalk_start(&dst_sg_walk, req->dst);
1128                         dst = scatterwalk_map(&dst_sg_walk);
1129                 }
1130
1131         } else {
1132                 /* Allocate memory for src, dst, assoc */
1133                 src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
1134                 if (!src)
1135                         return -ENOMEM;
1136                 assoc = (src + req->cryptlen + auth_tag_len);
1137                 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
1138                 scatterwalk_map_and_copy(assoc, req->assoc, 0,
1139                         req->assoclen, 0);
1140                 dst = src;
1141         }
1142
1143         aesni_gcm_dec_tfm(aes_ctx, dst, src, tempCipherLen, iv,
1144                 ctx->hash_subkey, assoc, (unsigned long)req->assoclen,
1145                 authTag, auth_tag_len);
1146
1147         /* Compare generated tag with passed in tag. */
1148         retval = crypto_memneq(src + tempCipherLen, authTag, auth_tag_len) ?
1149                 -EBADMSG : 0;
1150
1151         if (one_entry_in_sg) {
1152                 if (unlikely(req->src != req->dst)) {
1153                         scatterwalk_unmap(dst);
1154                         scatterwalk_done(&dst_sg_walk, 0, 0);
1155                 }
1156                 scatterwalk_unmap(src);
1157                 scatterwalk_unmap(assoc);
1158                 scatterwalk_done(&src_sg_walk, 0, 0);
1159                 scatterwalk_done(&assoc_sg_walk, 0, 0);
1160         } else {
1161                 scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1);
1162                 kfree(src);
1163         }
1164         return retval;
1165 }
1166 #endif
1167
1168 static struct crypto_alg aesni_algs[] = { {
1169         .cra_name               = "aes",
1170         .cra_driver_name        = "aes-aesni",
1171         .cra_priority           = 300,
1172         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
1173         .cra_blocksize          = AES_BLOCK_SIZE,
1174         .cra_ctxsize            = sizeof(struct crypto_aes_ctx) +
1175                                   AESNI_ALIGN - 1,
1176         .cra_alignmask          = 0,
1177         .cra_module             = THIS_MODULE,
1178         .cra_u  = {
1179                 .cipher = {
1180                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
1181                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
1182                         .cia_setkey             = aes_set_key,
1183                         .cia_encrypt            = aes_encrypt,
1184                         .cia_decrypt            = aes_decrypt
1185                 }
1186         }
1187 }, {
1188         .cra_name               = "__aes-aesni",
1189         .cra_driver_name        = "__driver-aes-aesni",
1190         .cra_priority           = 0,
1191         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
1192         .cra_blocksize          = AES_BLOCK_SIZE,
1193         .cra_ctxsize            = sizeof(struct crypto_aes_ctx) +
1194                                   AESNI_ALIGN - 1,
1195         .cra_alignmask          = 0,
1196         .cra_module             = THIS_MODULE,
1197         .cra_u  = {
1198                 .cipher = {
1199                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
1200                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
1201                         .cia_setkey             = aes_set_key,
1202                         .cia_encrypt            = __aes_encrypt,
1203                         .cia_decrypt            = __aes_decrypt
1204                 }
1205         }
1206 }, {
1207         .cra_name               = "__ecb-aes-aesni",
1208         .cra_driver_name        = "__driver-ecb-aes-aesni",
1209         .cra_priority           = 0,
1210         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
1211         .cra_blocksize          = AES_BLOCK_SIZE,
1212         .cra_ctxsize            = sizeof(struct crypto_aes_ctx) +
1213                                   AESNI_ALIGN - 1,
1214         .cra_alignmask          = 0,
1215         .cra_type               = &crypto_blkcipher_type,
1216         .cra_module             = THIS_MODULE,
1217         .cra_u = {
1218                 .blkcipher = {
1219                         .min_keysize    = AES_MIN_KEY_SIZE,
1220                         .max_keysize    = AES_MAX_KEY_SIZE,
1221                         .setkey         = aes_set_key,
1222                         .encrypt        = ecb_encrypt,
1223                         .decrypt        = ecb_decrypt,
1224                 },
1225         },
1226 }, {
1227         .cra_name               = "__cbc-aes-aesni",
1228         .cra_driver_name        = "__driver-cbc-aes-aesni",
1229         .cra_priority           = 0,
1230         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
1231         .cra_blocksize          = AES_BLOCK_SIZE,
1232         .cra_ctxsize            = sizeof(struct crypto_aes_ctx) +
1233                                   AESNI_ALIGN - 1,
1234         .cra_alignmask          = 0,
1235         .cra_type               = &crypto_blkcipher_type,
1236         .cra_module             = THIS_MODULE,
1237         .cra_u = {
1238                 .blkcipher = {
1239                         .min_keysize    = AES_MIN_KEY_SIZE,
1240                         .max_keysize    = AES_MAX_KEY_SIZE,
1241                         .setkey         = aes_set_key,
1242                         .encrypt        = cbc_encrypt,
1243                         .decrypt        = cbc_decrypt,
1244                 },
1245         },
1246 }, {
1247         .cra_name               = "ecb(aes)",
1248         .cra_driver_name        = "ecb-aes-aesni",
1249         .cra_priority           = 400,
1250         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1251         .cra_blocksize          = AES_BLOCK_SIZE,
1252         .cra_ctxsize            = sizeof(struct async_helper_ctx),
1253         .cra_alignmask          = 0,
1254         .cra_type               = &crypto_ablkcipher_type,
1255         .cra_module             = THIS_MODULE,
1256         .cra_init               = ablk_ecb_init,
1257         .cra_exit               = ablk_exit,
1258         .cra_u = {
1259                 .ablkcipher = {
1260                         .min_keysize    = AES_MIN_KEY_SIZE,
1261                         .max_keysize    = AES_MAX_KEY_SIZE,
1262                         .setkey         = ablk_set_key,
1263                         .encrypt        = ablk_encrypt,
1264                         .decrypt        = ablk_decrypt,
1265                 },
1266         },
1267 }, {
1268         .cra_name               = "cbc(aes)",
1269         .cra_driver_name        = "cbc-aes-aesni",
1270         .cra_priority           = 400,
1271         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1272         .cra_blocksize          = AES_BLOCK_SIZE,
1273         .cra_ctxsize            = sizeof(struct async_helper_ctx),
1274         .cra_alignmask          = 0,
1275         .cra_type               = &crypto_ablkcipher_type,
1276         .cra_module             = THIS_MODULE,
1277         .cra_init               = ablk_cbc_init,
1278         .cra_exit               = ablk_exit,
1279         .cra_u = {
1280                 .ablkcipher = {
1281                         .min_keysize    = AES_MIN_KEY_SIZE,
1282                         .max_keysize    = AES_MAX_KEY_SIZE,
1283                         .ivsize         = AES_BLOCK_SIZE,
1284                         .setkey         = ablk_set_key,
1285                         .encrypt        = ablk_encrypt,
1286                         .decrypt        = ablk_decrypt,
1287                 },
1288         },
1289 #ifdef CONFIG_X86_64
1290 }, {
1291         .cra_name               = "__ctr-aes-aesni",
1292         .cra_driver_name        = "__driver-ctr-aes-aesni",
1293         .cra_priority           = 0,
1294         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
1295         .cra_blocksize          = 1,
1296         .cra_ctxsize            = sizeof(struct crypto_aes_ctx) +
1297                                   AESNI_ALIGN - 1,
1298         .cra_alignmask          = 0,
1299         .cra_type               = &crypto_blkcipher_type,
1300         .cra_module             = THIS_MODULE,
1301         .cra_u = {
1302                 .blkcipher = {
1303                         .min_keysize    = AES_MIN_KEY_SIZE,
1304                         .max_keysize    = AES_MAX_KEY_SIZE,
1305                         .ivsize         = AES_BLOCK_SIZE,
1306                         .setkey         = aes_set_key,
1307                         .encrypt        = ctr_crypt,
1308                         .decrypt        = ctr_crypt,
1309                 },
1310         },
1311 }, {
1312         .cra_name               = "ctr(aes)",
1313         .cra_driver_name        = "ctr-aes-aesni",
1314         .cra_priority           = 400,
1315         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1316         .cra_blocksize          = 1,
1317         .cra_ctxsize            = sizeof(struct async_helper_ctx),
1318         .cra_alignmask          = 0,
1319         .cra_type               = &crypto_ablkcipher_type,
1320         .cra_module             = THIS_MODULE,
1321         .cra_init               = ablk_ctr_init,
1322         .cra_exit               = ablk_exit,
1323         .cra_u = {
1324                 .ablkcipher = {
1325                         .min_keysize    = AES_MIN_KEY_SIZE,
1326                         .max_keysize    = AES_MAX_KEY_SIZE,
1327                         .ivsize         = AES_BLOCK_SIZE,
1328                         .setkey         = ablk_set_key,
1329                         .encrypt        = ablk_encrypt,
1330                         .decrypt        = ablk_encrypt,
1331                         .geniv          = "chainiv",
1332                 },
1333         },
1334 }, {
1335         .cra_name               = "__gcm-aes-aesni",
1336         .cra_driver_name        = "__driver-gcm-aes-aesni",
1337         .cra_priority           = 0,
1338         .cra_flags              = CRYPTO_ALG_TYPE_AEAD,
1339         .cra_blocksize          = 1,
1340         .cra_ctxsize            = sizeof(struct aesni_rfc4106_gcm_ctx) +
1341                                   AESNI_ALIGN,
1342         .cra_alignmask          = 0,
1343         .cra_type               = &crypto_aead_type,
1344         .cra_module             = THIS_MODULE,
1345         .cra_u = {
1346                 .aead = {
1347                         .encrypt        = __driver_rfc4106_encrypt,
1348                         .decrypt        = __driver_rfc4106_decrypt,
1349                 },
1350         },
1351 }, {
1352         .cra_name               = "rfc4106(gcm(aes))",
1353         .cra_driver_name        = "rfc4106-gcm-aesni",
1354         .cra_priority           = 400,
1355         .cra_flags              = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1356         .cra_blocksize          = 1,
1357         .cra_ctxsize            = sizeof(struct aesni_rfc4106_gcm_ctx) +
1358                                   AESNI_ALIGN,
1359         .cra_alignmask          = 0,
1360         .cra_type               = &crypto_nivaead_type,
1361         .cra_module             = THIS_MODULE,
1362         .cra_init               = rfc4106_init,
1363         .cra_exit               = rfc4106_exit,
1364         .cra_u = {
1365                 .aead = {
1366                         .setkey         = rfc4106_set_key,
1367                         .setauthsize    = rfc4106_set_authsize,
1368                         .encrypt        = rfc4106_encrypt,
1369                         .decrypt        = rfc4106_decrypt,
1370                         .geniv          = "seqiv",
1371                         .ivsize         = 8,
1372                         .maxauthsize    = 16,
1373                 },
1374         },
1375 #endif
1376 #if IS_ENABLED(CONFIG_CRYPTO_PCBC)
1377 }, {
1378         .cra_name               = "pcbc(aes)",
1379         .cra_driver_name        = "pcbc-aes-aesni",
1380         .cra_priority           = 400,
1381         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1382         .cra_blocksize          = AES_BLOCK_SIZE,
1383         .cra_ctxsize            = sizeof(struct async_helper_ctx),
1384         .cra_alignmask          = 0,
1385         .cra_type               = &crypto_ablkcipher_type,
1386         .cra_module             = THIS_MODULE,
1387         .cra_init               = ablk_pcbc_init,
1388         .cra_exit               = ablk_exit,
1389         .cra_u = {
1390                 .ablkcipher = {
1391                         .min_keysize    = AES_MIN_KEY_SIZE,
1392                         .max_keysize    = AES_MAX_KEY_SIZE,
1393                         .ivsize         = AES_BLOCK_SIZE,
1394                         .setkey         = ablk_set_key,
1395                         .encrypt        = ablk_encrypt,
1396                         .decrypt        = ablk_decrypt,
1397                 },
1398         },
1399 #endif
1400 }, {
1401         .cra_name               = "__lrw-aes-aesni",
1402         .cra_driver_name        = "__driver-lrw-aes-aesni",
1403         .cra_priority           = 0,
1404         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
1405         .cra_blocksize          = AES_BLOCK_SIZE,
1406         .cra_ctxsize            = sizeof(struct aesni_lrw_ctx),
1407         .cra_alignmask          = 0,
1408         .cra_type               = &crypto_blkcipher_type,
1409         .cra_module             = THIS_MODULE,
1410         .cra_exit               = lrw_aesni_exit_tfm,
1411         .cra_u = {
1412                 .blkcipher = {
1413                         .min_keysize    = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
1414                         .max_keysize    = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
1415                         .ivsize         = AES_BLOCK_SIZE,
1416                         .setkey         = lrw_aesni_setkey,
1417                         .encrypt        = lrw_encrypt,
1418                         .decrypt        = lrw_decrypt,
1419                 },
1420         },
1421 }, {
1422         .cra_name               = "__xts-aes-aesni",
1423         .cra_driver_name        = "__driver-xts-aes-aesni",
1424         .cra_priority           = 0,
1425         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
1426         .cra_blocksize          = AES_BLOCK_SIZE,
1427         .cra_ctxsize            = sizeof(struct aesni_xts_ctx),
1428         .cra_alignmask          = 0,
1429         .cra_type               = &crypto_blkcipher_type,
1430         .cra_module             = THIS_MODULE,
1431         .cra_u = {
1432                 .blkcipher = {
1433                         .min_keysize    = 2 * AES_MIN_KEY_SIZE,
1434                         .max_keysize    = 2 * AES_MAX_KEY_SIZE,
1435                         .ivsize         = AES_BLOCK_SIZE,
1436                         .setkey         = xts_aesni_setkey,
1437                         .encrypt        = xts_encrypt,
1438                         .decrypt        = xts_decrypt,
1439                 },
1440         },
1441 }, {
1442         .cra_name               = "lrw(aes)",
1443         .cra_driver_name        = "lrw-aes-aesni",
1444         .cra_priority           = 400,
1445         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1446         .cra_blocksize          = AES_BLOCK_SIZE,
1447         .cra_ctxsize            = sizeof(struct async_helper_ctx),
1448         .cra_alignmask          = 0,
1449         .cra_type               = &crypto_ablkcipher_type,
1450         .cra_module             = THIS_MODULE,
1451         .cra_init               = ablk_init,
1452         .cra_exit               = ablk_exit,
1453         .cra_u = {
1454                 .ablkcipher = {
1455                         .min_keysize    = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
1456                         .max_keysize    = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
1457                         .ivsize         = AES_BLOCK_SIZE,
1458                         .setkey         = ablk_set_key,
1459                         .encrypt        = ablk_encrypt,
1460                         .decrypt        = ablk_decrypt,
1461                 },
1462         },
1463 }, {
1464         .cra_name               = "xts(aes)",
1465         .cra_driver_name        = "xts-aes-aesni",
1466         .cra_priority           = 400,
1467         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1468         .cra_blocksize          = AES_BLOCK_SIZE,
1469         .cra_ctxsize            = sizeof(struct async_helper_ctx),
1470         .cra_alignmask          = 0,
1471         .cra_type               = &crypto_ablkcipher_type,
1472         .cra_module             = THIS_MODULE,
1473         .cra_init               = ablk_init,
1474         .cra_exit               = ablk_exit,
1475         .cra_u = {
1476                 .ablkcipher = {
1477                         .min_keysize    = 2 * AES_MIN_KEY_SIZE,
1478                         .max_keysize    = 2 * AES_MAX_KEY_SIZE,
1479                         .ivsize         = AES_BLOCK_SIZE,
1480                         .setkey         = ablk_set_key,
1481                         .encrypt        = ablk_encrypt,
1482                         .decrypt        = ablk_decrypt,
1483                 },
1484         },
1485 } };
1486
1487
1488 static const struct x86_cpu_id aesni_cpu_id[] = {
1489         X86_FEATURE_MATCH(X86_FEATURE_AES),
1490         {}
1491 };
1492 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1493
1494 static int __init aesni_init(void)
1495 {
1496         int err;
1497
1498         if (!x86_match_cpu(aesni_cpu_id))
1499                 return -ENODEV;
1500 #ifdef CONFIG_X86_64
1501 #ifdef CONFIG_AS_AVX2
1502         if (boot_cpu_has(X86_FEATURE_AVX2)) {
1503                 pr_info("AVX2 version of gcm_enc/dec engaged.\n");
1504                 aesni_gcm_enc_tfm = aesni_gcm_enc_avx2;
1505                 aesni_gcm_dec_tfm = aesni_gcm_dec_avx2;
1506         } else
1507 #endif
1508 #ifdef CONFIG_AS_AVX
1509         if (boot_cpu_has(X86_FEATURE_AVX)) {
1510                 pr_info("AVX version of gcm_enc/dec engaged.\n");
1511                 aesni_gcm_enc_tfm = aesni_gcm_enc_avx;
1512                 aesni_gcm_dec_tfm = aesni_gcm_dec_avx;
1513         } else
1514 #endif
1515         {
1516                 pr_info("SSE version of gcm_enc/dec engaged.\n");
1517                 aesni_gcm_enc_tfm = aesni_gcm_enc;
1518                 aesni_gcm_dec_tfm = aesni_gcm_dec;
1519         }
1520         aesni_ctr_enc_tfm = aesni_ctr_enc;
1521 #ifdef CONFIG_AS_AVX
1522         if (cpu_has_avx) {
1523                 /* optimize performance of ctr mode encryption transform */
1524                 aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm;
1525                 pr_info("AES CTR mode by8 optimization enabled\n");
1526         }
1527 #endif
1528 #endif
1529
1530         err = crypto_fpu_init();
1531         if (err)
1532                 return err;
1533
1534         return crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1535 }
1536
1537 static void __exit aesni_exit(void)
1538 {
1539         crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1540
1541         crypto_fpu_exit();
1542 }
1543
1544 module_init(aesni_init);
1545 module_exit(aesni_exit);
1546
1547 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1548 MODULE_LICENSE("GPL");
1549 MODULE_ALIAS_CRYPTO("aes");