drivers/crypto/caam/caamalg.c

   1 /*
   2  * caam - Freescale FSL CAAM support for crypto API
   3  *
   4  * Copyright 2008-2011 Freescale Semiconductor, Inc.
   5  *
   6  * Based on talitos crypto API driver.
   7  *
   8  * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008):
   9  *
  10  * ---------------                     ---------------
  11  * | JobDesc #1  |-------------------->|  ShareDesc  |
  12  * | *(packet 1) |                     |   (PDB)     |
  13  * ---------------      |------------->|  (hashKey)  |
  14  *       .              |              | (cipherKey) |
  15  *       .              |    |-------->| (operation) |
  16  * ---------------      |    |         ---------------
  17  * | JobDesc #2  |------|    |
  18  * | *(packet 2) |           |
  19  * ---------------           |
  20  *       .                   |
  21  *       .                   |
  22  * ---------------           |
  23  * | JobDesc #3  |------------
  24  * | *(packet 3) |
  25  * ---------------
  26  *
  27  * The SharedDesc never changes for a connection unless rekeyed, but
  28  * each packet will likely be in a different place. So all we need
  29  * to know to process the packet is where the input is, where the
  30  * output goes, and what context we want to process with. Context is
  31  * in the SharedDesc, packet references in the JobDesc.
  32  *
  33  * So, a job desc looks like:
  34  *
  35  * ---------------------
  36  * | Header            |
  37  * | ShareDesc Pointer |
  38  * | SEQ_OUT_PTR       |
  39  * | (output buffer)   |
  40  * | (output length)   |
  41  * | SEQ_IN_PTR        |
  42  * | (input buffer)    |
  43  * | (input length)    |
  44  * ---------------------
  45  */
  46
  47 #include "compat.h"
  48
  49 #include "regs.h"
  50 #include "intern.h"
  51 #include "desc_constr.h"
  52 #include "jr.h"
  53 #include "error.h"
  54 #include "sg_sw_sec4.h"
  55 #include "key_gen.h"
  56
  57 /*
  58  * crypto alg
  59  */
  60 #define CAAM_CRA_PRIORITY               3000
  61 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
  62 #define CAAM_MAX_KEY_SIZE               (AES_MAX_KEY_SIZE + \
  63                                          CTR_RFC3686_NONCE_SIZE + \
  64                                          SHA512_DIGEST_SIZE * 2)
  65 /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
  66 #define CAAM_MAX_IV_LENGTH              16
  67
  68 #define AEAD_DESC_JOB_IO_LEN            (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2)
  69 #define GCM_DESC_JOB_IO_LEN             (AEAD_DESC_JOB_IO_LEN + \
  70                                          CAAM_CMD_SZ * 4)
  71 #define AUTHENC_DESC_JOB_IO_LEN         (AEAD_DESC_JOB_IO_LEN + \
  72                                          CAAM_CMD_SZ * 5)
  73
  74 /* length of descriptors text */
  75 #define DESC_AEAD_BASE                  (4 * CAAM_CMD_SZ)
  76 #define DESC_AEAD_ENC_LEN               (DESC_AEAD_BASE + 11 * CAAM_CMD_SZ)
  77 #define DESC_AEAD_DEC_LEN               (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ)
  78 #define DESC_AEAD_GIVENC_LEN            (DESC_AEAD_ENC_LEN + 9 * CAAM_CMD_SZ)
  79
  80 /* Note: Nonce is counted in enckeylen */
  81 #define DESC_AEAD_CTR_RFC3686_LEN       (4 * CAAM_CMD_SZ)
  82
  83 #define DESC_AEAD_NULL_BASE             (3 * CAAM_CMD_SZ)
  84 #define DESC_AEAD_NULL_ENC_LEN          (DESC_AEAD_NULL_BASE + 11 * CAAM_CMD_SZ)
  85 #define DESC_AEAD_NULL_DEC_LEN          (DESC_AEAD_NULL_BASE + 13 * CAAM_CMD_SZ)
  86
  87 #define DESC_GCM_BASE                   (3 * CAAM_CMD_SZ)
  88 #define DESC_GCM_ENC_LEN                (DESC_GCM_BASE + 16 * CAAM_CMD_SZ)
  89 #define DESC_GCM_DEC_LEN                (DESC_GCM_BASE + 12 * CAAM_CMD_SZ)
  90
  91 #define DESC_RFC4106_BASE               (3 * CAAM_CMD_SZ)
  92 #define DESC_RFC4106_ENC_LEN            (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ)
  93 #define DESC_RFC4106_DEC_LEN            (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ)
  94
  95 #define DESC_RFC4543_BASE               (3 * CAAM_CMD_SZ)
  96 #define DESC_RFC4543_ENC_LEN            (DESC_RFC4543_BASE + 11 * CAAM_CMD_SZ)
  97 #define DESC_RFC4543_DEC_LEN            (DESC_RFC4543_BASE + 12 * CAAM_CMD_SZ)
  98
  99 #define DESC_ABLKCIPHER_BASE            (3 * CAAM_CMD_SZ)
 100 #define DESC_ABLKCIPHER_ENC_LEN         (DESC_ABLKCIPHER_BASE + \
 101                                          20 * CAAM_CMD_SZ)
 102 #define DESC_ABLKCIPHER_DEC_LEN         (DESC_ABLKCIPHER_BASE + \
 103                                          15 * CAAM_CMD_SZ)
 104
 105 #define DESC_MAX_USED_BYTES             (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN)
 106 #define DESC_MAX_USED_LEN               (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
 107
 108 #ifdef DEBUG
 109 /* for print_hex_dumps with line references */
 110 #define debug(format, arg...) printk(format, arg)
 111 #else
 112 #define debug(format, arg...)
 113 #endif
 114 static struct list_head alg_list;
 115
 116 struct caam_alg_entry {
 117         int class1_alg_type;
 118         int class2_alg_type;
 119         int alg_op;
 120         bool rfc3686;
 121         bool geniv;
 122 };
 123
 124 struct caam_aead_alg {
 125         struct aead_alg aead;
 126         struct caam_alg_entry caam;
 127         bool registered;
 128 };
 129
 130 /* Set DK bit in class 1 operation if shared */
 131 static inline void append_dec_op1(u32 *desc, u32 type)
 132 {
 133         u32 *jump_cmd, *uncond_jump_cmd;
 134
 135         /* DK bit is valid only for AES */
 136         if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) {
 137                 append_operation(desc, type | OP_ALG_AS_INITFINAL |
 138                                  OP_ALG_DECRYPT);
 139                 return;
 140         }
 141
 142         jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
 143         append_operation(desc, type | OP_ALG_AS_INITFINAL |
 144                          OP_ALG_DECRYPT);
 145         uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
 146         set_jump_tgt_here(desc, jump_cmd);
 147         append_operation(desc, type | OP_ALG_AS_INITFINAL |
 148                          OP_ALG_DECRYPT | OP_ALG_AAI_DK);
 149         set_jump_tgt_here(desc, uncond_jump_cmd);
 150 }
 151
 152 /*
 153  * For aead functions, read payload and write payload,
 154  * both of which are specified in req->src and req->dst
 155  */
 156 static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
 157 {
 158         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
 159         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
 160                              KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
 161 }
 162
 163 /*
 164  * For ablkcipher encrypt and decrypt, read from req->src and
 165  * write to req->dst
 166  */
 167 static inline void ablkcipher_append_src_dst(u32 *desc)
 168 {
 169         append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 170         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 171         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
 172                              KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
 173         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
 174 }
 175
 176 /*
 177  * per-session context
 178  */
 179 struct caam_ctx {
 180         struct device *jrdev;
 181         u32 sh_desc_enc[DESC_MAX_USED_LEN];
 182         u32 sh_desc_dec[DESC_MAX_USED_LEN];
 183         u32 sh_desc_givenc[DESC_MAX_USED_LEN];
 184         dma_addr_t sh_desc_enc_dma;
 185         dma_addr_t sh_desc_dec_dma;
 186         dma_addr_t sh_desc_givenc_dma;
 187         u32 class1_alg_type;
 188         u32 class2_alg_type;
 189         u32 alg_op;
 190         u8 key[CAAM_MAX_KEY_SIZE];
 191         dma_addr_t key_dma;
 192         unsigned int enckeylen;
 193         unsigned int split_key_len;
 194         unsigned int split_key_pad_len;
 195         unsigned int authsize;
 196 };
 197
 198 static void append_key_aead(u32 *desc, struct caam_ctx *ctx,
 199                             int keys_fit_inline, bool is_rfc3686)
 200 {
 201         u32 *nonce;
 202         unsigned int enckeylen = ctx->enckeylen;
 203
 204         /*
 205          * RFC3686 specific:
 206          *      | ctx->key = {AUTH_KEY, ENC_KEY, NONCE}
 207          *      | enckeylen = encryption key size + nonce size
 208          */
 209         if (is_rfc3686)
 210                 enckeylen -= CTR_RFC3686_NONCE_SIZE;
 211
 212         if (keys_fit_inline) {
 213                 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
 214                                   ctx->split_key_len, CLASS_2 |
 215                                   KEY_DEST_MDHA_SPLIT | KEY_ENC);
 216                 append_key_as_imm(desc, (void *)ctx->key +
 217                                   ctx->split_key_pad_len, enckeylen,
 218                                   enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
 219         } else {
 220                 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
 221                            KEY_DEST_MDHA_SPLIT | KEY_ENC);
 222                 append_key(desc, ctx->key_dma + ctx->split_key_pad_len,
 223                            enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
 224         }
 225
 226         /* Load Counter into CONTEXT1 reg */
 227         if (is_rfc3686) {
 228                 nonce = (u32 *)((void *)ctx->key + ctx->split_key_pad_len +
 229                                enckeylen);
 230                 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
 231                                     LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
 232                 append_move(desc,
 233                             MOVE_SRC_OUTFIFO |
 234                             MOVE_DEST_CLASS1CTX |
 235                             (16 << MOVE_OFFSET_SHIFT) |
 236                             (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
 237         }
 238 }
 239
 240 static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx,
 241                                   int keys_fit_inline, bool is_rfc3686)
 242 {
 243         u32 *key_jump_cmd;
 244
 245         /* Note: Context registers are saved. */
 246         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
 247
 248         /* Skip if already shared */
 249         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 250                                    JUMP_COND_SHRD);
 251
 252         append_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
 253
 254         set_jump_tgt_here(desc, key_jump_cmd);
 255 }
 256
 257 static int aead_null_set_sh_desc(struct crypto_aead *aead)
 258 {
 259         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 260         struct device *jrdev = ctx->jrdev;
 261         bool keys_fit_inline = false;
 262         u32 *key_jump_cmd, *jump_cmd, *read_move_cmd, *write_move_cmd;
 263         u32 *desc;
 264
 265         /*
 266          * Job Descriptor and Shared Descriptors
 267          * must all fit into the 64-word Descriptor h/w Buffer
 268          */
 269         if (DESC_AEAD_NULL_ENC_LEN + AEAD_DESC_JOB_IO_LEN +
 270             ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX)
 271                 keys_fit_inline = true;
 272
 273         /* aead_encrypt shared descriptor */
 274         desc = ctx->sh_desc_enc;
 275
 276         init_sh_desc(desc, HDR_SHARE_SERIAL);
 277
 278         /* Skip if already shared */
 279         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 280                                    JUMP_COND_SHRD);
 281         if (keys_fit_inline)
 282                 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
 283                                   ctx->split_key_len, CLASS_2 |
 284                                   KEY_DEST_MDHA_SPLIT | KEY_ENC);
 285         else
 286                 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
 287                            KEY_DEST_MDHA_SPLIT | KEY_ENC);
 288         set_jump_tgt_here(desc, key_jump_cmd);
 289
 290         /* assoclen + cryptlen = seqinlen */
 291         append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
 292
 293         /* Prepare to read and write cryptlen + assoclen bytes */
 294         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 295         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 296
 297         /*
 298          * MOVE_LEN opcode is not available in all SEC HW revisions,
 299          * thus need to do some magic, i.e. self-patch the descriptor
 300          * buffer.
 301          */
 302         read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
 303                                     MOVE_DEST_MATH3 |
 304                                     (0x6 << MOVE_LEN_SHIFT));
 305         write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
 306                                      MOVE_DEST_DESCBUF |
 307                                      MOVE_WAITCOMP |
 308                                      (0x8 << MOVE_LEN_SHIFT));
 309
 310         /* Class 2 operation */
 311         append_operation(desc, ctx->class2_alg_type |
 312                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 313
 314         /* Read and write cryptlen bytes */
 315         aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
 316
 317         set_move_tgt_here(desc, read_move_cmd);
 318         set_move_tgt_here(desc, write_move_cmd);
 319         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
 320         append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
 321                     MOVE_AUX_LS);
 322
 323         /* Write ICV */
 324         append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
 325                          LDST_SRCDST_BYTE_CONTEXT);
 326
 327         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 328                                               desc_bytes(desc),
 329                                               DMA_TO_DEVICE);
 330         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 331                 dev_err(jrdev, "unable to map shared descriptor\n");
 332                 return -ENOMEM;
 333         }
 334 #ifdef DEBUG
 335         print_hex_dump(KERN_ERR,
 336                        "aead null enc shdesc@"__stringify(__LINE__)": ",
 337                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 338                        desc_bytes(desc), 1);
 339 #endif
 340
 341         /*
 342          * Job Descriptor and Shared Descriptors
 343          * must all fit into the 64-word Descriptor h/w Buffer
 344          */
 345         keys_fit_inline = false;
 346         if (DESC_AEAD_NULL_DEC_LEN + DESC_JOB_IO_LEN +
 347             ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX)
 348                 keys_fit_inline = true;
 349
 350         desc = ctx->sh_desc_dec;
 351
 352         /* aead_decrypt shared descriptor */
 353         init_sh_desc(desc, HDR_SHARE_SERIAL);
 354
 355         /* Skip if already shared */
 356         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 357                                    JUMP_COND_SHRD);
 358         if (keys_fit_inline)
 359                 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
 360                                   ctx->split_key_len, CLASS_2 |
 361                                   KEY_DEST_MDHA_SPLIT | KEY_ENC);
 362         else
 363                 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
 364                            KEY_DEST_MDHA_SPLIT | KEY_ENC);
 365         set_jump_tgt_here(desc, key_jump_cmd);
 366
 367         /* Class 2 operation */
 368         append_operation(desc, ctx->class2_alg_type |
 369                          OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
 370
 371         /* assoclen + cryptlen = seqoutlen */
 372         append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 373
 374         /* Prepare to read and write cryptlen + assoclen bytes */
 375         append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
 376         append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
 377
 378         /*
 379          * MOVE_LEN opcode is not available in all SEC HW revisions,
 380          * thus need to do some magic, i.e. self-patch the descriptor
 381          * buffer.
 382          */
 383         read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
 384                                     MOVE_DEST_MATH2 |
 385                                     (0x6 << MOVE_LEN_SHIFT));
 386         write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
 387                                      MOVE_DEST_DESCBUF |
 388                                      MOVE_WAITCOMP |
 389                                      (0x8 << MOVE_LEN_SHIFT));
 390
 391         /* Read and write cryptlen bytes */
 392         aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
 393
 394         /*
 395          * Insert a NOP here, since we need at least 4 instructions between
 396          * code patching the descriptor buffer and the location being patched.
 397          */
 398         jump_cmd = append_jump(desc, JUMP_TEST_ALL);
 399         set_jump_tgt_here(desc, jump_cmd);
 400
 401         set_move_tgt_here(desc, read_move_cmd);
 402         set_move_tgt_here(desc, write_move_cmd);
 403         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
 404         append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
 405                     MOVE_AUX_LS);
 406         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
 407
 408         /* Load ICV */
 409         append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 |
 410                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
 411
 412         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 413                                               desc_bytes(desc),
 414                                               DMA_TO_DEVICE);
 415         if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
 416                 dev_err(jrdev, "unable to map shared descriptor\n");
 417                 return -ENOMEM;
 418         }
 419 #ifdef DEBUG
 420         print_hex_dump(KERN_ERR,
 421                        "aead null dec shdesc@"__stringify(__LINE__)": ",
 422                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 423                        desc_bytes(desc), 1);
 424 #endif
 425
 426         return 0;
 427 }
 428
 429 static int aead_set_sh_desc(struct crypto_aead *aead)
 430 {
 431         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
 432                                                  struct caam_aead_alg, aead);
 433         unsigned int ivsize = crypto_aead_ivsize(aead);
 434         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 435         struct device *jrdev = ctx->jrdev;
 436         bool keys_fit_inline;
 437         u32 geniv, moveiv;
 438         u32 ctx1_iv_off = 0;
 439         u32 *desc;
 440         const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) ==
 441                                OP_ALG_AAI_CTR_MOD128);
 442         const bool is_rfc3686 = alg->caam.rfc3686;
 443
 444         if (!ctx->authsize)
 445                 return 0;
 446
 447         /* NULL encryption / decryption */
 448         if (!ctx->enckeylen)
 449                 return aead_null_set_sh_desc(aead);
 450
 451         /*
 452          * AES-CTR needs to load IV in CONTEXT1 reg
 453          * at an offset of 128bits (16bytes)
 454          * CONTEXT1[255:128] = IV
 455          */
 456         if (ctr_mode)
 457                 ctx1_iv_off = 16;
 458
 459         /*
 460          * RFC3686 specific:
 461          *      CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 462          */
 463         if (is_rfc3686)
 464                 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
 465
 466         if (alg->caam.geniv)
 467                 goto skip_enc;
 468
 469         /*
 470          * Job Descriptor and Shared Descriptors
 471          * must all fit into the 64-word Descriptor h/w Buffer
 472          */
 473         keys_fit_inline = false;
 474         if (DESC_AEAD_ENC_LEN + AUTHENC_DESC_JOB_IO_LEN +
 475             ctx->split_key_pad_len + ctx->enckeylen +
 476             (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
 477             CAAM_DESC_BYTES_MAX)
 478                 keys_fit_inline = true;
 479
 480         /* aead_encrypt shared descriptor */
 481         desc = ctx->sh_desc_enc;
 482
 483         /* Note: Context registers are saved. */
 484         init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
 485
 486         /* Class 2 operation */
 487         append_operation(desc, ctx->class2_alg_type |
 488                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 489
 490         /* Read and write assoclen bytes */
 491         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 492         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 493
 494         /* Skip assoc data */
 495         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 496
 497         /* read assoc before reading payload */
 498         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 499                                       FIFOLDST_VLF);
 500
 501         /* Load Counter into CONTEXT1 reg */
 502         if (is_rfc3686)
 503                 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
 504                                     LDST_CLASS_1_CCB |
 505                                     LDST_SRCDST_BYTE_CONTEXT |
 506                                     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
 507                                      LDST_OFFSET_SHIFT));
 508
 509         /* Class 1 operation */
 510         append_operation(desc, ctx->class1_alg_type |
 511                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 512
 513         /* Read and write cryptlen bytes */
 514         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 515         append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 516         aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
 517
 518         /* Write ICV */
 519         append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
 520                          LDST_SRCDST_BYTE_CONTEXT);
 521
 522         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 523                                               desc_bytes(desc),
 524                                               DMA_TO_DEVICE);
 525         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 526                 dev_err(jrdev, "unable to map shared descriptor\n");
 527                 return -ENOMEM;
 528         }
 529 #ifdef DEBUG
 530         print_hex_dump(KERN_ERR, "aead enc shdesc@"__stringify(__LINE__)": ",
 531                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 532                        desc_bytes(desc), 1);
 533 #endif
 534
 535 skip_enc:
 536         /*
 537          * Job Descriptor and Shared Descriptors
 538          * must all fit into the 64-word Descriptor h/w Buffer
 539          */
 540         keys_fit_inline = false;
 541         if (DESC_AEAD_DEC_LEN + AUTHENC_DESC_JOB_IO_LEN +
 542             ctx->split_key_pad_len + ctx->enckeylen +
 543             (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
 544             CAAM_DESC_BYTES_MAX)
 545                 keys_fit_inline = true;
 546
 547         /* aead_decrypt shared descriptor */
 548         desc = ctx->sh_desc_dec;
 549
 550         /* Note: Context registers are saved. */
 551         init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
 552
 553         /* Class 2 operation */
 554         append_operation(desc, ctx->class2_alg_type |
 555                          OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
 556
 557         /* Read and write assoclen bytes */
 558         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 559         if (alg->caam.geniv)
 560                 append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM, ivsize);
 561         else
 562                 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 563
 564         /* Skip assoc data */
 565         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 566
 567         /* read assoc before reading payload */
 568         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 569                              KEY_VLF);
 570
 571         if (alg->caam.geniv) {
 572                 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
 573                                 LDST_SRCDST_BYTE_CONTEXT |
 574                                 (ctx1_iv_off << LDST_OFFSET_SHIFT));
 575                 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO |
 576                             (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize);
 577         }
 578
 579         /* Load Counter into CONTEXT1 reg */
 580         if (is_rfc3686)
 581                 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
 582                                     LDST_CLASS_1_CCB |
 583                                     LDST_SRCDST_BYTE_CONTEXT |
 584                                     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
 585                                      LDST_OFFSET_SHIFT));
 586
 587         /* Choose operation */
 588         if (ctr_mode)
 589                 append_operation(desc, ctx->class1_alg_type |
 590                                  OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT);
 591         else
 592                 append_dec_op1(desc, ctx->class1_alg_type);
 593
 594         /* Read and write cryptlen bytes */
 595         append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 596         append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 597         aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
 598
 599         /* Load ICV */
 600         append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 |
 601                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
 602
 603         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 604                                               desc_bytes(desc),
 605                                               DMA_TO_DEVICE);
 606         if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
 607                 dev_err(jrdev, "unable to map shared descriptor\n");
 608                 return -ENOMEM;
 609         }
 610 #ifdef DEBUG
 611         print_hex_dump(KERN_ERR, "aead dec shdesc@"__stringify(__LINE__)": ",
 612                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 613                        desc_bytes(desc), 1);
 614 #endif
 615
 616         if (!alg->caam.geniv)
 617                 goto skip_givenc;
 618
 619         /*
 620          * Job Descriptor and Shared Descriptors
 621          * must all fit into the 64-word Descriptor h/w Buffer
 622          */
 623         keys_fit_inline = false;
 624         if (DESC_AEAD_GIVENC_LEN + AUTHENC_DESC_JOB_IO_LEN +
 625             ctx->split_key_pad_len + ctx->enckeylen +
 626             (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
 627             CAAM_DESC_BYTES_MAX)
 628                 keys_fit_inline = true;
 629
 630         /* aead_givencrypt shared descriptor */
 631         desc = ctx->sh_desc_enc;
 632
 633         /* Note: Context registers are saved. */
 634         init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
 635
 636         if (is_rfc3686)
 637                 goto copy_iv;
 638
 639         /* Generate IV */
 640         geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
 641                 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
 642                 NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT);
 643         append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
 644                             LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
 645         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
 646         append_move(desc, MOVE_WAITCOMP |
 647                     MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
 648                     (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
 649                     (ivsize << MOVE_LEN_SHIFT));
 650         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
 651
 652 copy_iv:
 653         /* Copy IV to class 1 context */
 654         append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
 655                     (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
 656                     (ivsize << MOVE_LEN_SHIFT));
 657
 658         /* Return to encryption */
 659         append_operation(desc, ctx->class2_alg_type |
 660                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 661
 662         /* Read and write assoclen bytes */
 663         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 664         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 665
 666         /* ivsize + cryptlen = seqoutlen - authsize */
 667         append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
 668
 669         /* Skip assoc data */
 670         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 671
 672         /* read assoc before reading payload */
 673         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 674                              KEY_VLF);
 675
 676         /* Copy iv from outfifo to class 2 fifo */
 677         moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
 678                  NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT);
 679         append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
 680                             LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
 681         append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB |
 682                             LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
 683
 684         /* Load Counter into CONTEXT1 reg */
 685         if (is_rfc3686)
 686                 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
 687                                     LDST_CLASS_1_CCB |
 688                                     LDST_SRCDST_BYTE_CONTEXT |
 689                                     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
 690                                      LDST_OFFSET_SHIFT));
 691
 692         /* Class 1 operation */
 693         append_operation(desc, ctx->class1_alg_type |
 694                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 695
 696         /* Will write ivsize + cryptlen */
 697         append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 698
 699         /* Not need to reload iv */
 700         append_seq_fifo_load(desc, ivsize,
 701                              FIFOLD_CLASS_SKIP);
 702
 703         /* Will read cryptlen */
 704         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 705         aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
 706
 707         /* Write ICV */
 708         append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
 709                          LDST_SRCDST_BYTE_CONTEXT);
 710
 711         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 712                                               desc_bytes(desc),
 713                                               DMA_TO_DEVICE);
 714         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 715                 dev_err(jrdev, "unable to map shared descriptor\n");
 716                 return -ENOMEM;
 717         }
 718 #ifdef DEBUG
 719         print_hex_dump(KERN_ERR, "aead givenc shdesc@"__stringify(__LINE__)": ",
 720                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 721                        desc_bytes(desc), 1);
 722 #endif
 723
 724 skip_givenc:
 725         return 0;
 726 }
 727
 728 static int aead_setauthsize(struct crypto_aead *authenc,
 729                                     unsigned int authsize)
 730 {
 731         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 732
 733         ctx->authsize = authsize;
 734         aead_set_sh_desc(authenc);
 735
 736         return 0;
 737 }
 738
 739 static int gcm_set_sh_desc(struct crypto_aead *aead)
 740 {
 741         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 742         struct device *jrdev = ctx->jrdev;
 743         bool keys_fit_inline = false;
 744         u32 *key_jump_cmd, *zero_payload_jump_cmd,
 745             *zero_assoc_jump_cmd1, *zero_assoc_jump_cmd2;
 746         u32 *desc;
 747
 748         if (!ctx->enckeylen || !ctx->authsize)
 749                 return 0;
 750
 751         /*
 752          * AES GCM encrypt shared descriptor
 753          * Job Descriptor and Shared Descriptor
 754          * must fit into the 64-word Descriptor h/w Buffer
 755          */
 756         if (DESC_GCM_ENC_LEN + GCM_DESC_JOB_IO_LEN +
 757             ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
 758                 keys_fit_inline = true;
 759
 760         desc = ctx->sh_desc_enc;
 761
 762         init_sh_desc(desc, HDR_SHARE_SERIAL);
 763
 764         /* skip key loading if they are loaded due to sharing */
 765         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 766                                    JUMP_COND_SHRD | JUMP_COND_SELF);
 767         if (keys_fit_inline)
 768                 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
 769                                   ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
 770         else
 771                 append_key(desc, ctx->key_dma, ctx->enckeylen,
 772                            CLASS_1 | KEY_DEST_CLASS_REG);
 773         set_jump_tgt_here(desc, key_jump_cmd);
 774
 775         /* class 1 operation */
 776         append_operation(desc, ctx->class1_alg_type |
 777                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 778
 779         /* if assoclen + cryptlen is ZERO, skip to ICV write */
 780         append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 781         zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
 782                                                  JUMP_COND_MATH_Z);
 783
 784         /* if assoclen is ZERO, skip reading the assoc data */
 785         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 786         zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
 787                                                  JUMP_COND_MATH_Z);
 788
 789         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 790
 791         /* skip assoc data */
 792         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 793
 794         /* cryptlen = seqinlen - assoclen */
 795         append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
 796
 797         /* if cryptlen is ZERO jump to zero-payload commands */
 798         zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
 799                                             JUMP_COND_MATH_Z);
 800
 801         /* read assoc data */
 802         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 803                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
 804         set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
 805
 806         append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 807
 808         /* write encrypted data */
 809         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
 810
 811         /* read payload data */
 812         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 813                              FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
 814
 815         /* jump the zero-payload commands */
 816         append_jump(desc, JUMP_TEST_ALL | 2);
 817
 818         /* zero-payload commands */
 819         set_jump_tgt_here(desc, zero_payload_jump_cmd);
 820
 821         /* read assoc data */
 822         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 823                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
 824
 825         /* There is no input data */
 826         set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
 827
 828         /* write ICV */
 829         append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
 830                          LDST_SRCDST_BYTE_CONTEXT);
 831
 832         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 833                                               desc_bytes(desc),
 834                                               DMA_TO_DEVICE);
 835         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 836                 dev_err(jrdev, "unable to map shared descriptor\n");
 837                 return -ENOMEM;
 838         }
 839 #ifdef DEBUG
 840         print_hex_dump(KERN_ERR, "gcm enc shdesc@"__stringify(__LINE__)": ",
 841                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 842                        desc_bytes(desc), 1);
 843 #endif
 844
 845         /*
 846          * Job Descriptor and Shared Descriptors
 847          * must all fit into the 64-word Descriptor h/w Buffer
 848          */
 849         keys_fit_inline = false;
 850         if (DESC_GCM_DEC_LEN + GCM_DESC_JOB_IO_LEN +
 851             ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
 852                 keys_fit_inline = true;
 853
 854         desc = ctx->sh_desc_dec;
 855
 856         init_sh_desc(desc, HDR_SHARE_SERIAL);
 857
 858         /* skip key loading if they are loaded due to sharing */
 859         key_jump_cmd = append_jump(desc, JUMP_JSL |
 860                                    JUMP_TEST_ALL | JUMP_COND_SHRD |
 861                                    JUMP_COND_SELF);
 862         if (keys_fit_inline)
 863                 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
 864                                   ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
 865         else
 866                 append_key(desc, ctx->key_dma, ctx->enckeylen,
 867                            CLASS_1 | KEY_DEST_CLASS_REG);
 868         set_jump_tgt_here(desc, key_jump_cmd);
 869
 870         /* class 1 operation */
 871         append_operation(desc, ctx->class1_alg_type |
 872                          OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
 873
 874         /* if assoclen is ZERO, skip reading the assoc data */
 875         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 876         zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
 877                                                  JUMP_COND_MATH_Z);
 878
 879         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 880
 881         /* skip assoc data */
 882         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 883
 884         /* read assoc data */
 885         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 886                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
 887
 888         set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
 889
 890         /* cryptlen = seqoutlen - assoclen */
 891         append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 892
 893         /* jump to zero-payload command if cryptlen is zero */
 894         zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
 895                                             JUMP_COND_MATH_Z);
 896
 897         append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 898
 899         /* store encrypted data */
 900         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
 901
 902         /* read payload data */
 903         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 904                              FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
 905
 906         /* zero-payload command */
 907         set_jump_tgt_here(desc, zero_payload_jump_cmd);
 908
 909         /* read ICV */
 910         append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
 911                              FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
 912
 913         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 914                                               desc_bytes(desc),
 915                                               DMA_TO_DEVICE);
 916         if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
 917                 dev_err(jrdev, "unable to map shared descriptor\n");
 918                 return -ENOMEM;
 919         }
 920 #ifdef DEBUG
 921         print_hex_dump(KERN_ERR, "gcm dec shdesc@"__stringify(__LINE__)": ",
 922                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 923                        desc_bytes(desc), 1);
 924 #endif
 925
 926         return 0;
 927 }
 928
 929 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
 930 {
 931         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 932
 933         ctx->authsize = authsize;
 934         gcm_set_sh_desc(authenc);
 935
 936         return 0;
 937 }
 938
 939 static int rfc4106_set_sh_desc(struct crypto_aead *aead)
 940 {
 941         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 942         struct device *jrdev = ctx->jrdev;
 943         bool keys_fit_inline = false;
 944         u32 *key_jump_cmd;
 945         u32 *desc;
 946
 947         if (!ctx->enckeylen || !ctx->authsize)
 948                 return 0;
 949
 950         /*
 951          * RFC4106 encrypt shared descriptor
 952          * Job Descriptor and Shared Descriptor
 953          * must fit into the 64-word Descriptor h/w Buffer
 954          */
 955         if (DESC_RFC4106_ENC_LEN + GCM_DESC_JOB_IO_LEN +
 956             ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
 957                 keys_fit_inline = true;
 958
 959         desc = ctx->sh_desc_enc;
 960
 961         init_sh_desc(desc, HDR_SHARE_SERIAL);
 962
 963         /* Skip key loading if it is loaded due to sharing */
 964         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 965                                    JUMP_COND_SHRD);
 966         if (keys_fit_inline)
 967                 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
 968                                   ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
 969         else
 970                 append_key(desc, ctx->key_dma, ctx->enckeylen,
 971                            CLASS_1 | KEY_DEST_CLASS_REG);
 972         set_jump_tgt_here(desc, key_jump_cmd);
 973
 974         /* Class 1 operation */
 975         append_operation(desc, ctx->class1_alg_type |
 976                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 977
 978         append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8);
 979         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 980
 981         /* Read assoc data */
 982         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 983                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
 984
 985         /* Skip IV */
 986         append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
 987
 988         /* Will read cryptlen bytes */
 989         append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 990
 991         /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
 992         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
 993
 994         /* Skip assoc data */
 995         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 996
 997         /* cryptlen = seqoutlen - assoclen */
 998         append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ);
 999
1000         /* Write encrypted data */
1001         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1002
1003         /* Read payload data */
1004         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1005                              FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
1006
1007         /* Write ICV */
1008         append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
1009                          LDST_SRCDST_BYTE_CONTEXT);
1010
1011         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
1012                                               desc_bytes(desc),
1013                                               DMA_TO_DEVICE);
1014         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
1015                 dev_err(jrdev, "unable to map shared descriptor\n");
1016                 return -ENOMEM;
1017         }
1018 #ifdef DEBUG
1019         print_hex_dump(KERN_ERR, "rfc4106 enc shdesc@"__stringify(__LINE__)": ",
1020                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
1021                        desc_bytes(desc), 1);
1022 #endif
1023
1024         /*
1025          * Job Descriptor and Shared Descriptors
1026          * must all fit into the 64-word Descriptor h/w Buffer
1027          */
1028         keys_fit_inline = false;
1029         if (DESC_RFC4106_DEC_LEN + DESC_JOB_IO_LEN +
1030             ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
1031                 keys_fit_inline = true;
1032
1033         desc = ctx->sh_desc_dec;
1034
1035         init_sh_desc(desc, HDR_SHARE_SERIAL);
1036
1037         /* Skip key loading if it is loaded due to sharing */
1038         key_jump_cmd = append_jump(desc, JUMP_JSL |
1039                                    JUMP_TEST_ALL | JUMP_COND_SHRD);
1040         if (keys_fit_inline)
1041                 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1042                                   ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1043         else
1044                 append_key(desc, ctx->key_dma, ctx->enckeylen,
1045                            CLASS_1 | KEY_DEST_CLASS_REG);
1046         set_jump_tgt_here(desc, key_jump_cmd);
1047
1048         /* Class 1 operation */
1049         append_operation(desc, ctx->class1_alg_type |
1050                          OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1051
1052         append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8);
1053         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1054
1055         /* Read assoc data */
1056         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1057                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1058
1059         /* Skip IV */
1060         append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1061
1062         /* Will read cryptlen bytes */
1063         append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ);
1064
1065         /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
1066         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
1067
1068         /* Skip assoc data */
1069         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
1070
1071         /* Will write cryptlen bytes */
1072         append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1073
1074         /* Store payload data */
1075         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1076
1077         /* Read encrypted data */
1078         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1079                              FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
1080
1081         /* Read ICV */
1082         append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
1083                              FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1084
1085         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
1086                                               desc_bytes(desc),
1087                                               DMA_TO_DEVICE);
1088         if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
1089                 dev_err(jrdev, "unable to map shared descriptor\n");
1090                 return -ENOMEM;
1091         }
1092 #ifdef DEBUG
1093         print_hex_dump(KERN_ERR, "rfc4106 dec shdesc@"__stringify(__LINE__)": ",
1094                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
1095                        desc_bytes(desc), 1);
1096 #endif
1097
1098         return 0;
1099 }
1100
1101 static int rfc4106_setauthsize(struct crypto_aead *authenc,
1102                                unsigned int authsize)
1103 {
1104         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
1105
1106         ctx->authsize = authsize;
1107         rfc4106_set_sh_desc(authenc);
1108
1109         return 0;
1110 }
1111
1112 static int rfc4543_set_sh_desc(struct crypto_aead *aead)
1113 {
1114         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1115         struct device *jrdev = ctx->jrdev;
1116         bool keys_fit_inline = false;
1117         u32 *key_jump_cmd;
1118         u32 *read_move_cmd, *write_move_cmd;
1119         u32 *desc;
1120
1121         if (!ctx->enckeylen || !ctx->authsize)
1122                 return 0;
1123
1124         /*
1125          * RFC4543 encrypt shared descriptor
1126          * Job Descriptor and Shared Descriptor
1127          * must fit into the 64-word Descriptor h/w Buffer
1128          */
1129         if (DESC_RFC4543_ENC_LEN + GCM_DESC_JOB_IO_LEN +
1130             ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
1131                 keys_fit_inline = true;
1132
1133         desc = ctx->sh_desc_enc;
1134
1135         init_sh_desc(desc, HDR_SHARE_SERIAL);
1136
1137         /* Skip key loading if it is loaded due to sharing */
1138         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1139                                    JUMP_COND_SHRD);
1140         if (keys_fit_inline)
1141                 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1142                                   ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1143         else
1144                 append_key(desc, ctx->key_dma, ctx->enckeylen,
1145                            CLASS_1 | KEY_DEST_CLASS_REG);
1146         set_jump_tgt_here(desc, key_jump_cmd);
1147
1148         /* Class 1 operation */
1149         append_operation(desc, ctx->class1_alg_type |
1150                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1151
1152         /* assoclen + cryptlen = seqinlen */
1153         append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
1154
1155         /*
1156          * MOVE_LEN opcode is not available in all SEC HW revisions,
1157          * thus need to do some magic, i.e. self-patch the descriptor
1158          * buffer.
1159          */
1160         read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1161                                     (0x6 << MOVE_LEN_SHIFT));
1162         write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1163                                      (0x8 << MOVE_LEN_SHIFT));
1164
1165         /* Will read assoclen + cryptlen bytes */
1166         append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1167
1168         /* Will write assoclen + cryptlen bytes */
1169         append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1170
1171         /* Read and write assoclen + cryptlen bytes */
1172         aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
1173
1174         set_move_tgt_here(desc, read_move_cmd);
1175         set_move_tgt_here(desc, write_move_cmd);
1176         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1177         /* Move payload data to OFIFO */
1178         append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1179
1180         /* Write ICV */
1181         append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
1182                          LDST_SRCDST_BYTE_CONTEXT);
1183
1184         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
1185                                               desc_bytes(desc),
1186                                               DMA_TO_DEVICE);
1187         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
1188                 dev_err(jrdev, "unable to map shared descriptor\n");
1189                 return -ENOMEM;
1190         }
1191 #ifdef DEBUG
1192         print_hex_dump(KERN_ERR, "rfc4543 enc shdesc@"__stringify(__LINE__)": ",
1193                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
1194                        desc_bytes(desc), 1);
1195 #endif
1196
1197         /*
1198          * Job Descriptor and Shared Descriptors
1199          * must all fit into the 64-word Descriptor h/w Buffer
1200          */
1201         keys_fit_inline = false;
1202         if (DESC_RFC4543_DEC_LEN + GCM_DESC_JOB_IO_LEN +
1203             ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
1204                 keys_fit_inline = true;
1205
1206         desc = ctx->sh_desc_dec;
1207
1208         init_sh_desc(desc, HDR_SHARE_SERIAL);
1209
1210         /* Skip key loading if it is loaded due to sharing */
1211         key_jump_cmd = append_jump(desc, JUMP_JSL |
1212                                    JUMP_TEST_ALL | JUMP_COND_SHRD);
1213         if (keys_fit_inline)
1214                 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1215                                   ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1216         else
1217                 append_key(desc, ctx->key_dma, ctx->enckeylen,
1218                            CLASS_1 | KEY_DEST_CLASS_REG);
1219         set_jump_tgt_here(desc, key_jump_cmd);
1220
1221         /* Class 1 operation */
1222         append_operation(desc, ctx->class1_alg_type |
1223                          OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1224
1225         /* assoclen + cryptlen = seqoutlen */
1226         append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1227
1228         /*
1229          * MOVE_LEN opcode is not available in all SEC HW revisions,
1230          * thus need to do some magic, i.e. self-patch the descriptor
1231          * buffer.
1232          */
1233         read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1234                                     (0x6 << MOVE_LEN_SHIFT));
1235         write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1236                                      (0x8 << MOVE_LEN_SHIFT));
1237
1238         /* Will read assoclen + cryptlen bytes */
1239         append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1240
1241         /* Will write assoclen + cryptlen bytes */
1242         append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1243
1244         /* Store payload data */
1245         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1246
1247         /* In-snoop assoclen + cryptlen data */
1248         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF |
1249                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1);
1250
1251         set_move_tgt_here(desc, read_move_cmd);
1252         set_move_tgt_here(desc, write_move_cmd);
1253         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1254         /* Move payload data to OFIFO */
1255         append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1256         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1257
1258         /* Read ICV */
1259         append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
1260                              FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1261
1262         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
1263                                               desc_bytes(desc),
1264                                               DMA_TO_DEVICE);
1265         if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
1266                 dev_err(jrdev, "unable to map shared descriptor\n");
1267                 return -ENOMEM;
1268         }
1269 #ifdef DEBUG
1270         print_hex_dump(KERN_ERR, "rfc4543 dec shdesc@"__stringify(__LINE__)": ",
1271                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
1272                        desc_bytes(desc), 1);
1273 #endif
1274
1275         return 0;
1276 }
1277
1278 static int rfc4543_setauthsize(struct crypto_aead *authenc,
1279                                unsigned int authsize)
1280 {
1281         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
1282
1283         ctx->authsize = authsize;
1284         rfc4543_set_sh_desc(authenc);
1285
1286         return 0;
1287 }
1288
1289 static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in,
1290                               u32 authkeylen)
1291 {
1292         return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len,
1293                                ctx->split_key_pad_len, key_in, authkeylen,
1294                                ctx->alg_op);
1295 }
1296
1297 static int aead_setkey(struct crypto_aead *aead,
1298                                const u8 *key, unsigned int keylen)
1299 {
1300         /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
1301         static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
1302         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1303         struct device *jrdev = ctx->jrdev;
1304         struct crypto_authenc_keys keys;
1305         int ret = 0;
1306
1307         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
1308                 goto badkey;
1309
1310         /* Pick class 2 key length from algorithm submask */
1311         ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >>
1312                                       OP_ALG_ALGSEL_SHIFT] * 2;
1313         ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16);
1314
1315         if (ctx->split_key_pad_len + keys.enckeylen > CAAM_MAX_KEY_SIZE)
1316                 goto badkey;
1317
1318 #ifdef DEBUG
1319         printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n",
1320                keys.authkeylen + keys.enckeylen, keys.enckeylen,
1321                keys.authkeylen);
1322         printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n",
1323                ctx->split_key_len, ctx->split_key_pad_len);
1324         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1325                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1326 #endif
1327
1328         ret = gen_split_aead_key(ctx, keys.authkey, keys.authkeylen);
1329         if (ret) {
1330                 goto badkey;
1331         }
1332
1333         /* postpend encryption key to auth split key */
1334         memcpy(ctx->key + ctx->split_key_pad_len, keys.enckey, keys.enckeylen);
1335
1336         ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len +
1337                                       keys.enckeylen, DMA_TO_DEVICE);
1338         if (dma_mapping_error(jrdev, ctx->key_dma)) {
1339                 dev_err(jrdev, "unable to map key i/o memory\n");
1340                 return -ENOMEM;
1341         }
1342 #ifdef DEBUG
1343         print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
1344                        DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
1345                        ctx->split_key_pad_len + keys.enckeylen, 1);
1346 #endif
1347
1348         ctx->enckeylen = keys.enckeylen;
1349
1350         ret = aead_set_sh_desc(aead);
1351         if (ret) {
1352                 dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len +
1353                                  keys.enckeylen, DMA_TO_DEVICE);
1354         }
1355
1356         return ret;
1357 badkey:
1358         crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
1359         return -EINVAL;
1360 }
1361
1362 static int gcm_setkey(struct crypto_aead *aead,
1363                       const u8 *key, unsigned int keylen)
1364 {
1365         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1366         struct device *jrdev = ctx->jrdev;
1367         int ret = 0;
1368
1369 #ifdef DEBUG
1370         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1371                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1372 #endif
1373
1374         memcpy(ctx->key, key, keylen);
1375         ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen,
1376                                       DMA_TO_DEVICE);
1377         if (dma_mapping_error(jrdev, ctx->key_dma)) {
1378                 dev_err(jrdev, "unable to map key i/o memory\n");
1379                 return -ENOMEM;
1380         }
1381         ctx->enckeylen = keylen;
1382
1383         ret = gcm_set_sh_desc(aead);
1384         if (ret) {
1385                 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen,
1386                                  DMA_TO_DEVICE);
1387         }
1388
1389         return ret;
1390 }
1391
1392 static int rfc4106_setkey(struct crypto_aead *aead,
1393                           const u8 *key, unsigned int keylen)
1394 {
1395         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1396         struct device *jrdev = ctx->jrdev;
1397         int ret = 0;
1398
1399         if (keylen < 4)
1400                 return -EINVAL;
1401
1402 #ifdef DEBUG
1403         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1404                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1405 #endif
1406
1407         memcpy(ctx->key, key, keylen);
1408
1409         /*
1410          * The last four bytes of the key material are used as the salt value
1411          * in the nonce. Update the AES key length.
1412          */
1413         ctx->enckeylen = keylen - 4;
1414
1415         ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen,
1416                                       DMA_TO_DEVICE);
1417         if (dma_mapping_error(jrdev, ctx->key_dma)) {
1418                 dev_err(jrdev, "unable to map key i/o memory\n");
1419                 return -ENOMEM;
1420         }
1421
1422         ret = rfc4106_set_sh_desc(aead);
1423         if (ret) {
1424                 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen,
1425                                  DMA_TO_DEVICE);
1426         }
1427
1428         return ret;
1429 }
1430
1431 static int rfc4543_setkey(struct crypto_aead *aead,
1432                           const u8 *key, unsigned int keylen)
1433 {
1434         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1435         struct device *jrdev = ctx->jrdev;
1436         int ret = 0;
1437
1438         if (keylen < 4)
1439                 return -EINVAL;
1440
1441 #ifdef DEBUG
1442         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1443                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1444 #endif
1445
1446         memcpy(ctx->key, key, keylen);
1447
1448         /*
1449          * The last four bytes of the key material are used as the salt value
1450          * in the nonce. Update the AES key length.
1451          */
1452         ctx->enckeylen = keylen - 4;
1453
1454         ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen,
1455                                       DMA_TO_DEVICE);
1456         if (dma_mapping_error(jrdev, ctx->key_dma)) {
1457                 dev_err(jrdev, "unable to map key i/o memory\n");
1458                 return -ENOMEM;
1459         }
1460
1461         ret = rfc4543_set_sh_desc(aead);
1462         if (ret) {
1463                 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen,
1464                                  DMA_TO_DEVICE);
1465         }
1466
1467         return ret;
1468 }
1469
1470 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
1471                              const u8 *key, unsigned int keylen)
1472 {
1473         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1474         struct ablkcipher_tfm *crt = &ablkcipher->base.crt_ablkcipher;
1475         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher);
1476         const char *alg_name = crypto_tfm_alg_name(tfm);
1477         struct device *jrdev = ctx->jrdev;
1478         int ret = 0;
1479         u32 *key_jump_cmd;
1480         u32 *desc;
1481         u32 *nonce;
1482         u32 geniv;
1483         u32 ctx1_iv_off = 0;
1484         const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) ==
1485                                OP_ALG_AAI_CTR_MOD128);
1486         const bool is_rfc3686 = (ctr_mode &&
1487                                  (strstr(alg_name, "rfc3686") != NULL));
1488
1489 #ifdef DEBUG
1490         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1491                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1492 #endif
1493         /*
1494          * AES-CTR needs to load IV in CONTEXT1 reg
1495          * at an offset of 128bits (16bytes)
1496          * CONTEXT1[255:128] = IV
1497          */
1498         if (ctr_mode)
1499                 ctx1_iv_off = 16;
1500
1501         /*
1502          * RFC3686 specific:
1503          *      | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
1504          *      | *key = {KEY, NONCE}
1505          */
1506         if (is_rfc3686) {
1507                 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
1508                 keylen -= CTR_RFC3686_NONCE_SIZE;
1509         }
1510
1511         memcpy(ctx->key, key, keylen);
1512         ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen,
1513                                       DMA_TO_DEVICE);
1514         if (dma_mapping_error(jrdev, ctx->key_dma)) {
1515                 dev_err(jrdev, "unable to map key i/o memory\n");
1516                 return -ENOMEM;
1517         }
1518         ctx->enckeylen = keylen;
1519
1520         /* ablkcipher_encrypt shared descriptor */
1521         desc = ctx->sh_desc_enc;
1522         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1523         /* Skip if already shared */
1524         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1525                                    JUMP_COND_SHRD);
1526
1527         /* Load class1 key only */
1528         append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1529                           ctx->enckeylen, CLASS_1 |
1530                           KEY_DEST_CLASS_REG);
1531
1532         /* Load nonce into CONTEXT1 reg */
1533         if (is_rfc3686) {
1534                 nonce = (u32 *)(key + keylen);
1535                 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
1536                                     LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1537                 append_move(desc, MOVE_WAITCOMP |
1538                             MOVE_SRC_OUTFIFO |
1539                             MOVE_DEST_CLASS1CTX |
1540                             (16 << MOVE_OFFSET_SHIFT) |
1541                             (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1542         }
1543
1544         set_jump_tgt_here(desc, key_jump_cmd);
1545
1546         /* Load iv */
1547         append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT |
1548                         LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1549
1550         /* Load counter into CONTEXT1 reg */
1551         if (is_rfc3686)
1552                 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
1553                                     LDST_CLASS_1_CCB |
1554                                     LDST_SRCDST_BYTE_CONTEXT |
1555                                     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1556                                      LDST_OFFSET_SHIFT));
1557
1558         /* Load operation */
1559         append_operation(desc, ctx->class1_alg_type |
1560                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1561
1562         /* Perform operation */
1563         ablkcipher_append_src_dst(desc);
1564
1565         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
1566                                               desc_bytes(desc),
1567                                               DMA_TO_DEVICE);
1568         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
1569                 dev_err(jrdev, "unable to map shared descriptor\n");
1570                 return -ENOMEM;
1571         }
1572 #ifdef DEBUG
1573         print_hex_dump(KERN_ERR,
1574                        "ablkcipher enc shdesc@"__stringify(__LINE__)": ",
1575                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
1576                        desc_bytes(desc), 1);
1577 #endif
1578         /* ablkcipher_decrypt shared descriptor */
1579         desc = ctx->sh_desc_dec;
1580
1581         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1582         /* Skip if already shared */
1583         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1584                                    JUMP_COND_SHRD);
1585
1586         /* Load class1 key only */
1587         append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1588                           ctx->enckeylen, CLASS_1 |
1589                           KEY_DEST_CLASS_REG);
1590
1591         /* Load nonce into CONTEXT1 reg */
1592         if (is_rfc3686) {
1593                 nonce = (u32 *)(key + keylen);
1594                 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
1595                                     LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1596                 append_move(desc, MOVE_WAITCOMP |
1597                             MOVE_SRC_OUTFIFO |
1598                             MOVE_DEST_CLASS1CTX |
1599                             (16 << MOVE_OFFSET_SHIFT) |
1600                             (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1601         }
1602
1603         set_jump_tgt_here(desc, key_jump_cmd);
1604
1605         /* load IV */
1606         append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT |
1607                         LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1608
1609         /* Load counter into CONTEXT1 reg */
1610         if (is_rfc3686)
1611                 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
1612                                     LDST_CLASS_1_CCB |
1613                                     LDST_SRCDST_BYTE_CONTEXT |
1614                                     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1615                                      LDST_OFFSET_SHIFT));
1616
1617         /* Choose operation */
1618         if (ctr_mode)
1619                 append_operation(desc, ctx->class1_alg_type |
1620                                  OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT);
1621         else
1622                 append_dec_op1(desc, ctx->class1_alg_type);
1623
1624         /* Perform operation */
1625         ablkcipher_append_src_dst(desc);
1626
1627         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
1628                                               desc_bytes(desc),
1629                                               DMA_TO_DEVICE);
1630         if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
1631                 dev_err(jrdev, "unable to map shared descriptor\n");
1632                 return -ENOMEM;
1633         }
1634
1635 #ifdef DEBUG
1636         print_hex_dump(KERN_ERR,
1637                        "ablkcipher dec shdesc@"__stringify(__LINE__)": ",
1638                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
1639                        desc_bytes(desc), 1);
1640 #endif
1641         /* ablkcipher_givencrypt shared descriptor */
1642         desc = ctx->sh_desc_givenc;
1643
1644         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1645         /* Skip if already shared */
1646         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1647                                    JUMP_COND_SHRD);
1648
1649         /* Load class1 key only */
1650         append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1651                           ctx->enckeylen, CLASS_1 |
1652                           KEY_DEST_CLASS_REG);
1653
1654         /* Load Nonce into CONTEXT1 reg */
1655         if (is_rfc3686) {
1656                 nonce = (u32 *)(key + keylen);
1657                 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
1658                                     LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1659                 append_move(desc, MOVE_WAITCOMP |
1660                             MOVE_SRC_OUTFIFO |
1661                             MOVE_DEST_CLASS1CTX |
1662                             (16 << MOVE_OFFSET_SHIFT) |
1663                             (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1664         }
1665         set_jump_tgt_here(desc, key_jump_cmd);
1666
1667         /* Generate IV */
1668         geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
1669                 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
1670                 NFIFOENTRY_PTYPE_RND | (crt->ivsize << NFIFOENTRY_DLEN_SHIFT);
1671         append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
1672                             LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
1673         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1674         append_move(desc, MOVE_WAITCOMP |
1675                     MOVE_SRC_INFIFO |
1676                     MOVE_DEST_CLASS1CTX |
1677                     (crt->ivsize << MOVE_LEN_SHIFT) |
1678                     (ctx1_iv_off << MOVE_OFFSET_SHIFT));
1679         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1680
1681         /* Copy generated IV to memory */
1682         append_seq_store(desc, crt->ivsize,
1683                          LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1684                          (ctx1_iv_off << LDST_OFFSET_SHIFT));
1685
1686         /* Load Counter into CONTEXT1 reg */
1687         if (is_rfc3686)
1688                 append_load_imm_u32(desc, (u32)1, LDST_IMM |
1689                                     LDST_CLASS_1_CCB |
1690                                     LDST_SRCDST_BYTE_CONTEXT |
1691                                     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1692                                      LDST_OFFSET_SHIFT));
1693
1694         if (ctx1_iv_off)
1695                 append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP |
1696                             (1 << JUMP_OFFSET_SHIFT));
1697
1698         /* Load operation */
1699         append_operation(desc, ctx->class1_alg_type |
1700                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1701
1702         /* Perform operation */
1703         ablkcipher_append_src_dst(desc);
1704
1705         ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc,
1706                                                  desc_bytes(desc),
1707                                                  DMA_TO_DEVICE);
1708         if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
1709                 dev_err(jrdev, "unable to map shared descriptor\n");
1710                 return -ENOMEM;
1711         }
1712 #ifdef DEBUG
1713         print_hex_dump(KERN_ERR,
1714                        "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ",
1715                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
1716                        desc_bytes(desc), 1);
1717 #endif
1718
1719         return ret;
1720 }
1721
1722 static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
1723                                  const u8 *key, unsigned int keylen)
1724 {
1725         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1726         struct device *jrdev = ctx->jrdev;
1727         u32 *key_jump_cmd, *desc;
1728         __be64 sector_size = cpu_to_be64(512);
1729
1730         if (keylen != 2 * AES_MIN_KEY_SIZE  && keylen != 2 * AES_MAX_KEY_SIZE) {
1731                 crypto_ablkcipher_set_flags(ablkcipher,
1732                                             CRYPTO_TFM_RES_BAD_KEY_LEN);
1733                 dev_err(jrdev, "key size mismatch\n");
1734                 return -EINVAL;
1735         }
1736
1737         memcpy(ctx->key, key, keylen);
1738         ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, DMA_TO_DEVICE);
1739         if (dma_mapping_error(jrdev, ctx->key_dma)) {
1740                 dev_err(jrdev, "unable to map key i/o memory\n");
1741                 return -ENOMEM;
1742         }
1743         ctx->enckeylen = keylen;
1744
1745         /* xts_ablkcipher_encrypt shared descriptor */
1746         desc = ctx->sh_desc_enc;
1747         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1748         /* Skip if already shared */
1749         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1750                                    JUMP_COND_SHRD);
1751
1752         /* Load class1 keys only */
1753         append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1754                           ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1755
1756         /* Load sector size with index 40 bytes (0x28) */
1757         append_cmd(desc, CMD_LOAD | IMMEDIATE | LDST_SRCDST_BYTE_CONTEXT |
1758                    LDST_CLASS_1_CCB | (0x28 << LDST_OFFSET_SHIFT) | 8);
1759         append_data(desc, (void *)&sector_size, 8);
1760
1761         set_jump_tgt_here(desc, key_jump_cmd);
1762
1763         /*
1764          * create sequence for loading the sector index
1765          * Upper 8B of IV - will be used as sector index
1766          * Lower 8B of IV - will be discarded
1767          */
1768         append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
1769                    LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT) | 8);
1770         append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1771
1772         /* Load operation */
1773         append_operation(desc, ctx->class1_alg_type | OP_ALG_AS_INITFINAL |
1774                          OP_ALG_ENCRYPT);
1775
1776         /* Perform operation */
1777         ablkcipher_append_src_dst(desc);
1778
1779         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, desc_bytes(desc),
1780                                               DMA_TO_DEVICE);
1781         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
1782                 dev_err(jrdev, "unable to map shared descriptor\n");
1783                 return -ENOMEM;
1784         }
1785 #ifdef DEBUG
1786         print_hex_dump(KERN_ERR,
1787                        "xts ablkcipher enc shdesc@" __stringify(__LINE__) ": ",
1788                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1789 #endif
1790
1791         /* xts_ablkcipher_decrypt shared descriptor */
1792         desc = ctx->sh_desc_dec;
1793
1794         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1795         /* Skip if already shared */
1796         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1797                                    JUMP_COND_SHRD);
1798
1799         /* Load class1 key only */
1800         append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1801                           ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1802
1803         /* Load sector size with index 40 bytes (0x28) */
1804         append_cmd(desc, CMD_LOAD | IMMEDIATE | LDST_SRCDST_BYTE_CONTEXT |
1805                    LDST_CLASS_1_CCB | (0x28 << LDST_OFFSET_SHIFT) | 8);
1806         append_data(desc, (void *)&sector_size, 8);
1807
1808         set_jump_tgt_here(desc, key_jump_cmd);
1809
1810         /*
1811          * create sequence for loading the sector index
1812          * Upper 8B of IV - will be used as sector index
1813          * Lower 8B of IV - will be discarded
1814          */
1815         append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
1816                    LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT) | 8);
1817         append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1818
1819         /* Load operation */
1820         append_dec_op1(desc, ctx->class1_alg_type);
1821
1822         /* Perform operation */
1823         ablkcipher_append_src_dst(desc);
1824
1825         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, desc_bytes(desc),
1826                                               DMA_TO_DEVICE);
1827         if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
1828                 dma_unmap_single(jrdev, ctx->sh_desc_enc_dma,
1829                                  desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE);
1830                 dev_err(jrdev, "unable to map shared descriptor\n");
1831                 return -ENOMEM;
1832         }
1833 #ifdef DEBUG
1834         print_hex_dump(KERN_ERR,
1835                        "xts ablkcipher dec shdesc@" __stringify(__LINE__) ": ",
1836                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1837 #endif
1838
1839         return 0;
1840 }
1841
1842 /*
1843  * aead_edesc - s/w-extended aead descriptor
1844  * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist
1845  * @src_nents: number of segments in input scatterlist
1846  * @dst_nents: number of segments in output scatterlist
1847  * @iv_dma: dma address of iv for checking continuity and link table
1848  * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
1849  * @sec4_sg_bytes: length of dma mapped sec4_sg space
1850  * @sec4_sg_dma: bus physical mapped address of h/w link table
1851  * @hw_desc: the h/w job descriptor followed by any referenced link tables
1852  */
1853 struct aead_edesc {
1854         int assoc_nents;
1855         int src_nents;
1856         int dst_nents;
1857         dma_addr_t iv_dma;
1858         int sec4_sg_bytes;
1859         dma_addr_t sec4_sg_dma;
1860         struct sec4_sg_entry *sec4_sg;
1861         u32 hw_desc[];
1862 };
1863
1864 /*
1865  * ablkcipher_edesc - s/w-extended ablkcipher descriptor
1866  * @src_nents: number of segments in input scatterlist
1867  * @dst_nents: number of segments in output scatterlist
1868  * @iv_dma: dma address of iv for checking continuity and link table
1869  * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
1870  * @sec4_sg_bytes: length of dma mapped sec4_sg space
1871  * @sec4_sg_dma: bus physical mapped address of h/w link table
1872  * @hw_desc: the h/w job descriptor followed by any referenced link tables
1873  */
1874 struct ablkcipher_edesc {
1875         int src_nents;
1876         int dst_nents;
1877         dma_addr_t iv_dma;
1878         int sec4_sg_bytes;
1879         dma_addr_t sec4_sg_dma;
1880         struct sec4_sg_entry *sec4_sg;
1881         u32 hw_desc[0];
1882 };
1883
1884 static void caam_unmap(struct device *dev, struct scatterlist *src,
1885                        struct scatterlist *dst, int src_nents,
1886                        int dst_nents,
1887                        dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma,
1888                        int sec4_sg_bytes)
1889 {
1890         if (dst != src) {
1891                 dma_unmap_sg(dev, src, src_nents ? : 1, DMA_TO_DEVICE);
1892                 dma_unmap_sg(dev, dst, dst_nents ? : 1, DMA_FROM_DEVICE);
1893         } else {
1894                 dma_unmap_sg(dev, src, src_nents ? : 1, DMA_BIDIRECTIONAL);
1895         }
1896
1897         if (iv_dma)
1898                 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
1899         if (sec4_sg_bytes)
1900                 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes,
1901                                  DMA_TO_DEVICE);
1902 }
1903
1904 static void aead_unmap(struct device *dev,
1905                        struct aead_edesc *edesc,
1906                        struct aead_request *req)
1907 {
1908         caam_unmap(dev, req->src, req->dst,
1909                    edesc->src_nents, edesc->dst_nents, 0, 0,
1910                    edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
1911 }
1912
1913 static void ablkcipher_unmap(struct device *dev,
1914                              struct ablkcipher_edesc *edesc,
1915                              struct ablkcipher_request *req)
1916 {
1917         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1918         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1919
1920         caam_unmap(dev, req->src, req->dst,
1921                    edesc->src_nents, edesc->dst_nents,
1922                    edesc->iv_dma, ivsize,
1923                    edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
1924 }
1925
1926 static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
1927                                    void *context)
1928 {
1929         struct aead_request *req = context;
1930         struct aead_edesc *edesc;
1931
1932 #ifdef DEBUG
1933         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
1934 #endif
1935
1936         edesc = container_of(desc, struct aead_edesc, hw_desc[0]);
1937
1938         if (err)
1939                 caam_jr_strstatus(jrdev, err);
1940
1941         aead_unmap(jrdev, edesc, req);
1942
1943         kfree(edesc);
1944
1945         aead_request_complete(req, err);
1946 }
1947
1948 static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
1949                                    void *context)
1950 {
1951         struct aead_request *req = context;
1952         struct aead_edesc *edesc;
1953
1954 #ifdef DEBUG
1955         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
1956 #endif
1957
1958         edesc = container_of(desc, struct aead_edesc, hw_desc[0]);
1959
1960         if (err)
1961                 caam_jr_strstatus(jrdev, err);
1962
1963         aead_unmap(jrdev, edesc, req);
1964
1965         /*
1966          * verify hw auth check passed else return -EBADMSG
1967          */
1968         if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK)
1969                 err = -EBADMSG;
1970
1971         kfree(edesc);
1972
1973         aead_request_complete(req, err);
1974 }
1975
1976 static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
1977                                    void *context)
1978 {
1979         struct ablkcipher_request *req = context;
1980         struct ablkcipher_edesc *edesc;
1981 #ifdef DEBUG
1982         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1983         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1984
1985         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
1986 #endif
1987
1988         edesc = (struct ablkcipher_edesc *)((char *)desc -
1989                  offsetof(struct ablkcipher_edesc, hw_desc));
1990
1991         if (err)
1992                 caam_jr_strstatus(jrdev, err);
1993
1994 #ifdef DEBUG
1995         print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
1996                        DUMP_PREFIX_ADDRESS, 16, 4, req->info,
1997                        edesc->src_nents > 1 ? 100 : ivsize, 1);
1998         print_hex_dump(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
1999                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2000                        edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
2001 #endif
2002
2003         ablkcipher_unmap(jrdev, edesc, req);
2004         kfree(edesc);
2005
2006         ablkcipher_request_complete(req, err);
2007 }
2008
2009 static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
2010                                     void *context)
2011 {
2012         struct ablkcipher_request *req = context;
2013         struct ablkcipher_edesc *edesc;
2014 #ifdef DEBUG
2015         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2016         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2017
2018         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
2019 #endif
2020
2021         edesc = (struct ablkcipher_edesc *)((char *)desc -
2022                  offsetof(struct ablkcipher_edesc, hw_desc));
2023         if (err)
2024                 caam_jr_strstatus(jrdev, err);
2025
2026 #ifdef DEBUG
2027         print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
2028                        DUMP_PREFIX_ADDRESS, 16, 4, req->info,
2029                        ivsize, 1);
2030         print_hex_dump(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
2031                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2032                        edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
2033 #endif
2034
2035         ablkcipher_unmap(jrdev, edesc, req);
2036         kfree(edesc);
2037
2038         ablkcipher_request_complete(req, err);
2039 }
2040
2041 /*
2042  * Fill in aead job descriptor
2043  */
2044 static void init_aead_job(struct aead_request *req,
2045                           struct aead_edesc *edesc,
2046                           bool all_contig, bool encrypt)
2047 {
2048         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2049         struct caam_ctx *ctx = crypto_aead_ctx(aead);
2050         int authsize = ctx->authsize;
2051         u32 *desc = edesc->hw_desc;
2052         u32 out_options, in_options;
2053         dma_addr_t dst_dma, src_dma;
2054         int len, sec4_sg_index = 0;
2055         dma_addr_t ptr;
2056         u32 *sh_desc;
2057
2058         sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec;
2059         ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma;
2060
2061         len = desc_len(sh_desc);
2062         init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
2063
2064         if (all_contig) {
2065                 src_dma = sg_dma_address(req->src);
2066                 in_options = 0;
2067         } else {
2068                 src_dma = edesc->sec4_sg_dma;
2069                 sec4_sg_index += edesc->src_nents;
2070                 in_options = LDST_SGF;
2071         }
2072
2073         append_seq_in_ptr(desc, src_dma, req->assoclen + req->cryptlen,
2074                           in_options);
2075
2076         dst_dma = src_dma;
2077         out_options = in_options;
2078
2079         if (unlikely(req->src != req->dst)) {
2080                 if (!edesc->dst_nents) {
2081                         dst_dma = sg_dma_address(req->dst);
2082                 } else {
2083                         dst_dma = edesc->sec4_sg_dma +
2084                                   sec4_sg_index *
2085                                   sizeof(struct sec4_sg_entry);
2086                         out_options = LDST_SGF;
2087                 }
2088         }
2089
2090         if (encrypt)
2091                 append_seq_out_ptr(desc, dst_dma,
2092                                    req->assoclen + req->cryptlen + authsize,
2093                                    out_options);
2094         else
2095                 append_seq_out_ptr(desc, dst_dma,
2096                                    req->assoclen + req->cryptlen - authsize,
2097                                    out_options);
2098
2099         /* REG3 = assoclen */
2100         append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
2101 }
2102
2103 static void init_gcm_job(struct aead_request *req,
2104                          struct aead_edesc *edesc,
2105                          bool all_contig, bool encrypt)
2106 {
2107         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2108         struct caam_ctx *ctx = crypto_aead_ctx(aead);
2109         unsigned int ivsize = crypto_aead_ivsize(aead);
2110         u32 *desc = edesc->hw_desc;
2111         bool generic_gcm = (ivsize == 12);
2112         unsigned int last;
2113
2114         init_aead_job(req, edesc, all_contig, encrypt);
2115
2116         /* BUG This should not be specific to generic GCM. */
2117         last = 0;
2118         if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen))
2119                 last = FIFOLD_TYPE_LAST1;
2120
2121         /* Read GCM IV */
2122         append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
2123                          FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | 12 | last);
2124         /* Append Salt */
2125         if (!generic_gcm)
2126                 append_data(desc, ctx->key + ctx->enckeylen, 4);
2127         /* Append IV */
2128         append_data(desc, req->iv, ivsize);
2129         /* End of blank commands */
2130 }
2131
2132 static void init_authenc_job(struct aead_request *req,
2133                              struct aead_edesc *edesc,
2134                              bool all_contig, bool encrypt)
2135 {
2136         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2137         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
2138                                                  struct caam_aead_alg, aead);
2139         unsigned int ivsize = crypto_aead_ivsize(aead);
2140         struct caam_ctx *ctx = crypto_aead_ctx(aead);
2141         const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) ==
2142                                OP_ALG_AAI_CTR_MOD128);
2143         const bool is_rfc3686 = alg->caam.rfc3686;
2144         u32 *desc = edesc->hw_desc;
2145         u32 ivoffset = 0;
2146
2147         /*
2148          * AES-CTR needs to load IV in CONTEXT1 reg
2149          * at an offset of 128bits (16bytes)
2150          * CONTEXT1[255:128] = IV
2151          */
2152         if (ctr_mode)
2153                 ivoffset = 16;
2154
2155         /*
2156          * RFC3686 specific:
2157          *      CONTEXT1[255:128] = {NONCE, IV, COUNTER}
2158          */
2159         if (is_rfc3686)
2160                 ivoffset = 16 + CTR_RFC3686_NONCE_SIZE;
2161
2162         init_aead_job(req, edesc, all_contig, encrypt);
2163
2164         if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv))
2165                 append_load_as_imm(desc, req->iv, ivsize,
2166                                    LDST_CLASS_1_CCB |
2167                                    LDST_SRCDST_BYTE_CONTEXT |
2168                                    (ivoffset << LDST_OFFSET_SHIFT));
2169 }
2170
2171 /*
2172  * Fill in ablkcipher job descriptor
2173  */
2174 static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
2175                                 struct ablkcipher_edesc *edesc,
2176                                 struct ablkcipher_request *req,
2177                                 bool iv_contig)
2178 {
2179         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2180         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2181         u32 *desc = edesc->hw_desc;
2182         u32 out_options = 0, in_options;
2183         dma_addr_t dst_dma, src_dma;
2184         int len, sec4_sg_index = 0;
2185
2186 #ifdef DEBUG
2187         print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
2188                        DUMP_PREFIX_ADDRESS, 16, 4, req->info,
2189                        ivsize, 1);
2190         print_hex_dump(KERN_ERR, "src    @"__stringify(__LINE__)": ",
2191                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2192                        edesc->src_nents ? 100 : req->nbytes, 1);
2193 #endif
2194
2195         len = desc_len(sh_desc);
2196         init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
2197
2198         if (iv_contig) {
2199                 src_dma = edesc->iv_dma;
2200                 in_options = 0;
2201         } else {
2202                 src_dma = edesc->sec4_sg_dma;
2203                 sec4_sg_index += edesc->src_nents + 1;
2204                 in_options = LDST_SGF;
2205         }
2206         append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options);
2207
2208         if (likely(req->src == req->dst)) {
2209                 if (!edesc->src_nents && iv_contig) {
2210                         dst_dma = sg_dma_address(req->src);
2211                 } else {
2212                         dst_dma = edesc->sec4_sg_dma +
2213                                 sizeof(struct sec4_sg_entry);
2214                         out_options = LDST_SGF;
2215                 }
2216         } else {
2217                 if (!edesc->dst_nents) {
2218                         dst_dma = sg_dma_address(req->dst);
2219                 } else {
2220                         dst_dma = edesc->sec4_sg_dma +
2221                                 sec4_sg_index * sizeof(struct sec4_sg_entry);
2222                         out_options = LDST_SGF;
2223                 }
2224         }
2225         append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options);
2226 }
2227
2228 /*
2229  * Fill in ablkcipher givencrypt job descriptor
2230  */
2231 static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr,
2232                                     struct ablkcipher_edesc *edesc,
2233                                     struct ablkcipher_request *req,
2234                                     bool iv_contig)
2235 {
2236         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2237         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2238         u32 *desc = edesc->hw_desc;
2239         u32 out_options, in_options;
2240         dma_addr_t dst_dma, src_dma;
2241         int len, sec4_sg_index = 0;
2242
2243 #ifdef DEBUG
2244         print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ",
2245                        DUMP_PREFIX_ADDRESS, 16, 4, req->info,
2246                        ivsize, 1);
2247         print_hex_dump(KERN_ERR, "src    @" __stringify(__LINE__) ": ",
2248                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2249                        edesc->src_nents ? 100 : req->nbytes, 1);
2250 #endif
2251
2252         len = desc_len(sh_desc);
2253         init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
2254
2255         if (!edesc->src_nents) {
2256                 src_dma = sg_dma_address(req->src);
2257                 in_options = 0;
2258         } else {
2259                 src_dma = edesc->sec4_sg_dma;
2260                 sec4_sg_index += edesc->src_nents;
2261                 in_options = LDST_SGF;
2262         }
2263         append_seq_in_ptr(desc, src_dma, req->nbytes, in_options);
2264
2265         if (iv_contig) {
2266                 dst_dma = edesc->iv_dma;
2267                 out_options = 0;
2268         } else {
2269                 dst_dma = edesc->sec4_sg_dma +
2270                           sec4_sg_index * sizeof(struct sec4_sg_entry);
2271                 out_options = LDST_SGF;
2272         }
2273         append_seq_out_ptr(desc, dst_dma, req->nbytes + ivsize, out_options);
2274 }
2275
2276 /*
2277  * allocate and map the aead extended descriptor
2278  */
2279 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
2280                                            int desc_bytes, bool *all_contig_ptr,
2281                                            bool encrypt)
2282 {
2283         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2284         struct caam_ctx *ctx = crypto_aead_ctx(aead);
2285         struct device *jrdev = ctx->jrdev;
2286         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2287                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2288         int src_nents, dst_nents = 0;
2289         struct aead_edesc *edesc;
2290         int sgc;
2291         bool all_contig = true;
2292         int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
2293         unsigned int authsize = ctx->authsize;
2294
2295         if (unlikely(req->dst != req->src)) {
2296                 src_nents = sg_count(req->src, req->assoclen + req->cryptlen);
2297                 dst_nents = sg_count(req->dst,
2298                                      req->assoclen + req->cryptlen +
2299                                         (encrypt ? authsize : (-authsize)));
2300         } else {
2301                 src_nents = sg_count(req->src,
2302                                      req->assoclen + req->cryptlen +
2303                                         (encrypt ? authsize : 0));
2304         }
2305
2306         /* Check if data are contiguous. */
2307         all_contig = !src_nents;
2308         if (!all_contig) {
2309                 src_nents = src_nents ? : 1;
2310                 sec4_sg_len = src_nents;
2311         }
2312
2313         sec4_sg_len += dst_nents;
2314
2315         sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
2316
2317         /* allocate space for base edesc and hw desc commands, link tables */
2318         edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
2319                         GFP_DMA | flags);
2320         if (!edesc) {
2321                 dev_err(jrdev, "could not allocate extended descriptor\n");
2322                 return ERR_PTR(-ENOMEM);
2323         }
2324
2325         if (likely(req->src == req->dst)) {
2326                 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
2327                                  DMA_BIDIRECTIONAL);
2328                 if (unlikely(!sgc)) {
2329                         dev_err(jrdev, "unable to map source\n");
2330                         kfree(edesc);
2331                         return ERR_PTR(-ENOMEM);
2332                 }
2333         } else {
2334                 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
2335                                  DMA_TO_DEVICE);
2336                 if (unlikely(!sgc)) {
2337                         dev_err(jrdev, "unable to map source\n");
2338                         kfree(edesc);
2339                         return ERR_PTR(-ENOMEM);
2340                 }
2341
2342                 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
2343                                  DMA_FROM_DEVICE);
2344                 if (unlikely(!sgc)) {
2345                         dev_err(jrdev, "unable to map destination\n");
2346                         dma_unmap_sg(jrdev, req->src, src_nents ? : 1,
2347                                      DMA_TO_DEVICE);
2348                         kfree(edesc);
2349                         return ERR_PTR(-ENOMEM);
2350                 }
2351         }
2352
2353         edesc->src_nents = src_nents;
2354         edesc->dst_nents = dst_nents;
2355         edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
2356                          desc_bytes;
2357         *all_contig_ptr = all_contig;
2358
2359         sec4_sg_index = 0;
2360         if (!all_contig) {
2361                 sg_to_sec4_sg_last(req->src, src_nents,
2362                               edesc->sec4_sg + sec4_sg_index, 0);
2363                 sec4_sg_index += src_nents;
2364         }
2365         if (dst_nents) {
2366                 sg_to_sec4_sg_last(req->dst, dst_nents,
2367                                    edesc->sec4_sg + sec4_sg_index, 0);
2368         }
2369
2370         if (!sec4_sg_bytes)
2371                 return edesc;
2372
2373         edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
2374                                             sec4_sg_bytes, DMA_TO_DEVICE);
2375         if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
2376                 dev_err(jrdev, "unable to map S/G table\n");
2377                 aead_unmap(jrdev, edesc, req);
2378                 kfree(edesc);
2379                 return ERR_PTR(-ENOMEM);
2380         }
2381
2382         edesc->sec4_sg_bytes = sec4_sg_bytes;
2383
2384         return edesc;
2385 }
2386
2387 static int gcm_encrypt(struct aead_request *req)
2388 {
2389         struct aead_edesc *edesc;
2390         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2391         struct caam_ctx *ctx = crypto_aead_ctx(aead);
2392         struct device *jrdev = ctx->jrdev;
2393         bool all_contig;
2394         u32 *desc;
2395         int ret = 0;
2396
2397         /* allocate extended descriptor */
2398         edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, true);
2399         if (IS_ERR(edesc))
2400                 return PTR_ERR(edesc);
2401
2402         /* Create and submit job descriptor */
2403         init_gcm_job(req, edesc, all_contig, true);
2404 #ifdef DEBUG
2405         print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
2406                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2407                        desc_bytes(edesc->hw_desc), 1);
2408 #endif
2409
2410         desc = edesc->hw_desc;
2411         ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
2412         if (!ret) {
2413                 ret = -EINPROGRESS;
2414         } else {
2415                 aead_unmap(jrdev, edesc, req);
2416                 kfree(edesc);
2417         }
2418
2419         return ret;
2420 }
2421
2422 static int ipsec_gcm_encrypt(struct aead_request *req)
2423 {
2424         if (req->assoclen < 8)
2425                 return -EINVAL;
2426
2427         return gcm_encrypt(req);
2428 }
2429
2430 static int aead_encrypt(struct aead_request *req)
2431 {
2432         struct aead_edesc *edesc;
2433         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2434         struct caam_ctx *ctx = crypto_aead_ctx(aead);
2435         struct device *jrdev = ctx->jrdev;
2436         bool all_contig;
2437         u32 *desc;
2438         int ret = 0;
2439
2440         /* allocate extended descriptor */
2441         edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN,
2442                                  &all_contig, true);
2443         if (IS_ERR(edesc))
2444                 return PTR_ERR(edesc);
2445
2446         /* Create and submit job descriptor */
2447         init_authenc_job(req, edesc, all_contig, true);
2448 #ifdef DEBUG
2449         print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
2450                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2451                        desc_bytes(edesc->hw_desc), 1);
2452 #endif
2453
2454         desc = edesc->hw_desc;
2455         ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
2456         if (!ret) {
2457                 ret = -EINPROGRESS;
2458         } else {
2459                 aead_unmap(jrdev, edesc, req);
2460                 kfree(edesc);
2461         }
2462
2463         return ret;
2464 }
2465
2466 static int gcm_decrypt(struct aead_request *req)
2467 {
2468         struct aead_edesc *edesc;
2469         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2470         struct caam_ctx *ctx = crypto_aead_ctx(aead);
2471         struct device *jrdev = ctx->jrdev;
2472         bool all_contig;
2473         u32 *desc;
2474         int ret = 0;
2475
2476         /* allocate extended descriptor */
2477         edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, false);
2478         if (IS_ERR(edesc))
2479                 return PTR_ERR(edesc);
2480
2481         /* Create and submit job descriptor*/
2482         init_gcm_job(req, edesc, all_contig, false);
2483 #ifdef DEBUG
2484         print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
2485                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2486                        desc_bytes(edesc->hw_desc), 1);
2487 #endif
2488
2489         desc = edesc->hw_desc;
2490         ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
2491         if (!ret) {
2492                 ret = -EINPROGRESS;
2493         } else {
2494                 aead_unmap(jrdev, edesc, req);
2495                 kfree(edesc);
2496         }
2497
2498         return ret;
2499 }
2500
2501 static int ipsec_gcm_decrypt(struct aead_request *req)
2502 {
2503         if (req->assoclen < 8)
2504                 return -EINVAL;
2505
2506         return gcm_decrypt(req);
2507 }
2508
2509 static int aead_decrypt(struct aead_request *req)
2510 {
2511         struct aead_edesc *edesc;
2512         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2513         struct caam_ctx *ctx = crypto_aead_ctx(aead);
2514         struct device *jrdev = ctx->jrdev;
2515         bool all_contig;
2516         u32 *desc;
2517         int ret = 0;
2518
2519         /* allocate extended descriptor */
2520         edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN,
2521                                  &all_contig, false);
2522         if (IS_ERR(edesc))
2523                 return PTR_ERR(edesc);
2524
2525 #ifdef DEBUG
2526         print_hex_dump(KERN_ERR, "dec src@"__stringify(__LINE__)": ",
2527                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2528                        req->assoclen + req->cryptlen, 1);
2529 #endif
2530
2531         /* Create and submit job descriptor*/
2532         init_authenc_job(req, edesc, all_contig, false);
2533 #ifdef DEBUG
2534         print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
2535                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2536                        desc_bytes(edesc->hw_desc), 1);
2537 #endif
2538
2539         desc = edesc->hw_desc;
2540         ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
2541         if (!ret) {
2542                 ret = -EINPROGRESS;
2543         } else {
2544                 aead_unmap(jrdev, edesc, req);
2545                 kfree(edesc);
2546         }
2547
2548         return ret;
2549 }
2550
2551 /*
2552  * allocate and map the ablkcipher extended descriptor for ablkcipher
2553  */
2554 static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
2555                                                        *req, int desc_bytes,
2556                                                        bool *iv_contig_out)
2557 {
2558         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2559         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
2560         struct device *jrdev = ctx->jrdev;
2561         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2562                                           CRYPTO_TFM_REQ_MAY_SLEEP)) ?
2563                        GFP_KERNEL : GFP_ATOMIC;
2564         int src_nents, dst_nents = 0, sec4_sg_bytes;
2565         struct ablkcipher_edesc *edesc;
2566         dma_addr_t iv_dma = 0;
2567         bool iv_contig = false;
2568         int sgc;
2569         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2570         int sec4_sg_index;
2571
2572         src_nents = sg_count(req->src, req->nbytes);
2573
2574         if (req->dst != req->src)
2575                 dst_nents = sg_count(req->dst, req->nbytes);
2576
2577         if (likely(req->src == req->dst)) {
2578                 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
2579                                  DMA_BIDIRECTIONAL);
2580         } else {
2581                 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
2582                                  DMA_TO_DEVICE);
2583                 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
2584                                  DMA_FROM_DEVICE);
2585         }
2586
2587         iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE);
2588         if (dma_mapping_error(jrdev, iv_dma)) {
2589                 dev_err(jrdev, "unable to map IV\n");
2590                 return ERR_PTR(-ENOMEM);
2591         }
2592
2593         /*
2594          * Check if iv can be contiguous with source and destination.
2595          * If so, include it. If not, create scatterlist.
2596          */
2597         if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src))
2598                 iv_contig = true;
2599         else
2600                 src_nents = src_nents ? : 1;
2601         sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) *
2602                         sizeof(struct sec4_sg_entry);
2603
2604         /* allocate space for base edesc and hw desc commands, link tables */
2605         edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
2606                         GFP_DMA | flags);
2607         if (!edesc) {
2608                 dev_err(jrdev, "could not allocate extended descriptor\n");
2609                 return ERR_PTR(-ENOMEM);
2610         }
2611
2612         edesc->src_nents = src_nents;
2613         edesc->dst_nents = dst_nents;
2614         edesc->sec4_sg_bytes = sec4_sg_bytes;
2615         edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +
2616                          desc_bytes;
2617
2618         sec4_sg_index = 0;
2619         if (!iv_contig) {
2620                 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);
2621                 sg_to_sec4_sg_last(req->src, src_nents,
2622                                    edesc->sec4_sg + 1, 0);
2623                 sec4_sg_index += 1 + src_nents;
2624         }
2625
2626         if (dst_nents) {
2627                 sg_to_sec4_sg_last(req->dst, dst_nents,
2628                         edesc->sec4_sg + sec4_sg_index, 0);
2629         }
2630
2631         edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
2632                                             sec4_sg_bytes, DMA_TO_DEVICE);
2633         if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
2634                 dev_err(jrdev, "unable to map S/G table\n");
2635                 return ERR_PTR(-ENOMEM);
2636         }
2637
2638         edesc->iv_dma = iv_dma;
2639
2640 #ifdef DEBUG
2641         print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"__stringify(__LINE__)": ",
2642                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
2643                        sec4_sg_bytes, 1);
2644 #endif
2645
2646         *iv_contig_out = iv_contig;
2647         return edesc;
2648 }
2649
2650 static int ablkcipher_encrypt(struct ablkcipher_request *req)
2651 {
2652         struct ablkcipher_edesc *edesc;
2653         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2654         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
2655         struct device *jrdev = ctx->jrdev;
2656         bool iv_contig;
2657         u32 *desc;
2658         int ret = 0;
2659
2660         /* allocate extended descriptor */
2661         edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
2662                                        CAAM_CMD_SZ, &iv_contig);
2663         if (IS_ERR(edesc))
2664                 return PTR_ERR(edesc);
2665
2666         /* Create and submit job descriptor*/
2667         init_ablkcipher_job(ctx->sh_desc_enc,
2668                 ctx->sh_desc_enc_dma, edesc, req, iv_contig);
2669 #ifdef DEBUG
2670         print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
2671                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2672                        desc_bytes(edesc->hw_desc), 1);
2673 #endif
2674         desc = edesc->hw_desc;
2675         ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
2676
2677         if (!ret) {
2678                 ret = -EINPROGRESS;
2679         } else {
2680                 ablkcipher_unmap(jrdev, edesc, req);
2681                 kfree(edesc);
2682         }
2683
2684         return ret;
2685 }
2686
2687 static int ablkcipher_decrypt(struct ablkcipher_request *req)
2688 {
2689         struct ablkcipher_edesc *edesc;
2690         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2691         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
2692         struct device *jrdev = ctx->jrdev;
2693         bool iv_contig;
2694         u32 *desc;
2695         int ret = 0;
2696
2697         /* allocate extended descriptor */
2698         edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
2699                                        CAAM_CMD_SZ, &iv_contig);
2700         if (IS_ERR(edesc))
2701                 return PTR_ERR(edesc);
2702
2703         /* Create and submit job descriptor*/
2704         init_ablkcipher_job(ctx->sh_desc_dec,
2705                 ctx->sh_desc_dec_dma, edesc, req, iv_contig);
2706         desc = edesc->hw_desc;
2707 #ifdef DEBUG
2708         print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
2709                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2710                        desc_bytes(edesc->hw_desc), 1);
2711 #endif
2712
2713         ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req);
2714         if (!ret) {
2715                 ret = -EINPROGRESS;
2716         } else {
2717                 ablkcipher_unmap(jrdev, edesc, req);
2718                 kfree(edesc);
2719         }
2720
2721         return ret;
2722 }
2723
2724 /*
2725  * allocate and map the ablkcipher extended descriptor
2726  * for ablkcipher givencrypt
2727  */
2728 static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
2729                                 struct skcipher_givcrypt_request *greq,
2730                                 int desc_bytes,
2731                                 bool *iv_contig_out)
2732 {
2733         struct ablkcipher_request *req = &greq->creq;
2734         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2735         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
2736         struct device *jrdev = ctx->jrdev;
2737         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2738                                           CRYPTO_TFM_REQ_MAY_SLEEP)) ?
2739                        GFP_KERNEL : GFP_ATOMIC;
2740         int src_nents, dst_nents = 0, sec4_sg_bytes;
2741         struct ablkcipher_edesc *edesc;
2742         dma_addr_t iv_dma = 0;
2743         bool iv_contig = false;
2744         int sgc;
2745         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2746         int sec4_sg_index;
2747
2748         src_nents = sg_count(req->src, req->nbytes);
2749
2750         if (unlikely(req->dst != req->src))
2751                 dst_nents = sg_count(req->dst, req->nbytes);
2752
2753         if (likely(req->src == req->dst)) {
2754                 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
2755                                  DMA_BIDIRECTIONAL);
2756         } else {
2757                 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
2758                                  DMA_TO_DEVICE);
2759                 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
2760                                  DMA_FROM_DEVICE);
2761         }
2762
2763         /*
2764          * Check if iv can be contiguous with source and destination.
2765          * If so, include it. If not, create scatterlist.
2766          */
2767         iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE);
2768         if (dma_mapping_error(jrdev, iv_dma)) {
2769                 dev_err(jrdev, "unable to map IV\n");
2770                 return ERR_PTR(-ENOMEM);
2771         }
2772
2773         if (!dst_nents && iv_dma + ivsize == sg_dma_address(req->dst))
2774                 iv_contig = true;
2775         else
2776                 dst_nents = dst_nents ? : 1;
2777         sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) *
2778                         sizeof(struct sec4_sg_entry);
2779
2780         /* allocate space for base edesc and hw desc commands, link tables */
2781         edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
2782                         GFP_DMA | flags);
2783         if (!edesc) {
2784                 dev_err(jrdev, "could not allocate extended descriptor\n");
2785                 return ERR_PTR(-ENOMEM);
2786         }
2787
2788         edesc->src_nents = src_nents;
2789         edesc->dst_nents = dst_nents;
2790         edesc->sec4_sg_bytes = sec4_sg_bytes;
2791         edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +
2792                          desc_bytes;
2793
2794         sec4_sg_index = 0;
2795         if (src_nents) {
2796                 sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0);
2797                 sec4_sg_index += src_nents;
2798         }
2799
2800         if (!iv_contig) {
2801                 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
2802                                    iv_dma, ivsize, 0);
2803                 sec4_sg_index += 1;
2804                 sg_to_sec4_sg_last(req->dst, dst_nents,
2805                                    edesc->sec4_sg + sec4_sg_index, 0);
2806         }
2807
2808         edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
2809                                             sec4_sg_bytes, DMA_TO_DEVICE);
2810         if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
2811                 dev_err(jrdev, "unable to map S/G table\n");
2812                 return ERR_PTR(-ENOMEM);
2813         }
2814         edesc->iv_dma = iv_dma;
2815
2816 #ifdef DEBUG
2817         print_hex_dump(KERN_ERR,
2818                        "ablkcipher sec4_sg@" __stringify(__LINE__) ": ",
2819                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
2820                        sec4_sg_bytes, 1);
2821 #endif
2822
2823         *iv_contig_out = iv_contig;
2824         return edesc;
2825 }
2826
2827 static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq)
2828 {
2829         struct ablkcipher_request *req = &creq->creq;
2830         struct ablkcipher_edesc *edesc;
2831         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2832         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
2833         struct device *jrdev = ctx->jrdev;
2834         bool iv_contig;
2835         u32 *desc;
2836         int ret = 0;
2837
2838         /* allocate extended descriptor */
2839         edesc = ablkcipher_giv_edesc_alloc(creq, DESC_JOB_IO_LEN *
2840                                        CAAM_CMD_SZ, &iv_contig);
2841         if (IS_ERR(edesc))
2842                 return PTR_ERR(edesc);
2843
2844         /* Create and submit job descriptor*/
2845         init_ablkcipher_giv_job(ctx->sh_desc_givenc, ctx->sh_desc_givenc_dma,
2846                                 edesc, req, iv_contig);
2847 #ifdef DEBUG
2848         print_hex_dump(KERN_ERR,
2849                        "ablkcipher jobdesc@" __stringify(__LINE__) ": ",
2850                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2851                        desc_bytes(edesc->hw_desc), 1);
2852 #endif
2853         desc = edesc->hw_desc;
2854         ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
2855
2856         if (!ret) {
2857                 ret = -EINPROGRESS;
2858         } else {
2859                 ablkcipher_unmap(jrdev, edesc, req);
2860                 kfree(edesc);
2861         }
2862
2863         return ret;
2864 }
2865
2866 #define template_aead           template_u.aead
2867 #define template_ablkcipher     template_u.ablkcipher
2868 struct caam_alg_template {
2869         char name[CRYPTO_MAX_ALG_NAME];
2870         char driver_name[CRYPTO_MAX_ALG_NAME];
2871         unsigned int blocksize;
2872         u32 type;
2873         union {
2874                 struct ablkcipher_alg ablkcipher;
2875         } template_u;
2876         u32 class1_alg_type;
2877         u32 class2_alg_type;
2878         u32 alg_op;
2879 };
2880
2881 static struct caam_alg_template driver_algs[] = {
2882         /* ablkcipher descriptor */
2883         {
2884                 .name = "cbc(aes)",
2885                 .driver_name = "cbc-aes-caam",
2886                 .blocksize = AES_BLOCK_SIZE,
2887                 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
2888                 .template_ablkcipher = {
2889                         .setkey = ablkcipher_setkey,
2890                         .encrypt = ablkcipher_encrypt,
2891                         .decrypt = ablkcipher_decrypt,
2892                         .givencrypt = ablkcipher_givencrypt,
2893                         .geniv = "<built-in>",
2894                         .min_keysize = AES_MIN_KEY_SIZE,
2895                         .max_keysize = AES_MAX_KEY_SIZE,
2896                         .ivsize = AES_BLOCK_SIZE,
2897                         },
2898                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2899         },
2900         {
2901                 .name = "cbc(des3_ede)",
2902                 .driver_name = "cbc-3des-caam",
2903                 .blocksize = DES3_EDE_BLOCK_SIZE,
2904                 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
2905                 .template_ablkcipher = {
2906                         .setkey = ablkcipher_setkey,
2907                         .encrypt = ablkcipher_encrypt,
2908                         .decrypt = ablkcipher_decrypt,
2909                         .givencrypt = ablkcipher_givencrypt,
2910                         .geniv = "<built-in>",
2911                         .min_keysize = DES3_EDE_KEY_SIZE,
2912                         .max_keysize = DES3_EDE_KEY_SIZE,
2913                         .ivsize = DES3_EDE_BLOCK_SIZE,
2914                         },
2915                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2916         },
2917         {
2918                 .name = "cbc(des)",
2919                 .driver_name = "cbc-des-caam",
2920                 .blocksize = DES_BLOCK_SIZE,
2921                 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
2922                 .template_ablkcipher = {
2923                         .setkey = ablkcipher_setkey,
2924                         .encrypt = ablkcipher_encrypt,
2925                         .decrypt = ablkcipher_decrypt,
2926                         .givencrypt = ablkcipher_givencrypt,
2927                         .geniv = "<built-in>",
2928                         .min_keysize = DES_KEY_SIZE,
2929                         .max_keysize = DES_KEY_SIZE,
2930                         .ivsize = DES_BLOCK_SIZE,
2931                         },
2932                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2933         },
2934         {
2935                 .name = "ctr(aes)",
2936                 .driver_name = "ctr-aes-caam",
2937                 .blocksize = 1,
2938                 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2939                 .template_ablkcipher = {
2940                         .setkey = ablkcipher_setkey,
2941                         .encrypt = ablkcipher_encrypt,
2942                         .decrypt = ablkcipher_decrypt,
2943                         .geniv = "chainiv",
2944                         .min_keysize = AES_MIN_KEY_SIZE,
2945                         .max_keysize = AES_MAX_KEY_SIZE,
2946                         .ivsize = AES_BLOCK_SIZE,
2947                         },
2948                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
2949         },
2950         {
2951                 .name = "rfc3686(ctr(aes))",
2952                 .driver_name = "rfc3686-ctr-aes-caam",
2953                 .blocksize = 1,
2954                 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
2955                 .template_ablkcipher = {
2956                         .setkey = ablkcipher_setkey,
2957                         .encrypt = ablkcipher_encrypt,
2958                         .decrypt = ablkcipher_decrypt,
2959                         .givencrypt = ablkcipher_givencrypt,
2960                         .geniv = "<built-in>",
2961                         .min_keysize = AES_MIN_KEY_SIZE +
2962                                        CTR_RFC3686_NONCE_SIZE,
2963                         .max_keysize = AES_MAX_KEY_SIZE +
2964                                        CTR_RFC3686_NONCE_SIZE,
2965                         .ivsize = CTR_RFC3686_IV_SIZE,
2966                         },
2967                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
2968         },
2969         {
2970                 .name = "xts(aes)",
2971                 .driver_name = "xts-aes-caam",
2972                 .blocksize = AES_BLOCK_SIZE,
2973                 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2974                 .template_ablkcipher = {
2975                         .setkey = xts_ablkcipher_setkey,
2976                         .encrypt = ablkcipher_encrypt,
2977                         .decrypt = ablkcipher_decrypt,
2978                         .geniv = "eseqiv",
2979                         .min_keysize = 2 * AES_MIN_KEY_SIZE,
2980                         .max_keysize = 2 * AES_MAX_KEY_SIZE,
2981                         .ivsize = AES_BLOCK_SIZE,
2982                         },
2983                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
2984         },
2985 };
2986
2987 static struct caam_aead_alg driver_aeads[] = {
2988         {
2989                 .aead = {
2990                         .base = {
2991                                 .cra_name = "rfc4106(gcm(aes))",
2992                                 .cra_driver_name = "rfc4106-gcm-aes-caam",
2993                                 .cra_blocksize = 1,
2994                         },
2995                         .setkey = rfc4106_setkey,
2996                         .setauthsize = rfc4106_setauthsize,
2997                         .encrypt = ipsec_gcm_encrypt,
2998                         .decrypt = ipsec_gcm_decrypt,
2999                         .ivsize = 8,
3000                         .maxauthsize = AES_BLOCK_SIZE,
3001                 },
3002                 .caam = {
3003                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
3004                 },
3005         },
3006         {
3007                 .aead = {
3008                         .base = {
3009                                 .cra_name = "rfc4543(gcm(aes))",
3010                                 .cra_driver_name = "rfc4543-gcm-aes-caam",
3011                                 .cra_blocksize = 1,
3012                         },
3013                         .setkey = rfc4543_setkey,
3014                         .setauthsize = rfc4543_setauthsize,
3015                         .encrypt = ipsec_gcm_encrypt,
3016                         .decrypt = ipsec_gcm_decrypt,
3017                         .ivsize = 8,
3018                         .maxauthsize = AES_BLOCK_SIZE,
3019                 },
3020                 .caam = {
3021                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
3022                 },
3023         },
3024         /* Galois Counter Mode */
3025         {
3026                 .aead = {
3027                         .base = {
3028                                 .cra_name = "gcm(aes)",
3029                                 .cra_driver_name = "gcm-aes-caam",
3030                                 .cra_blocksize = 1,
3031                         },
3032                         .setkey = gcm_setkey,
3033                         .setauthsize = gcm_setauthsize,
3034                         .encrypt = gcm_encrypt,
3035                         .decrypt = gcm_decrypt,
3036                         .ivsize = 12,
3037                         .maxauthsize = AES_BLOCK_SIZE,
3038                 },
3039                 .caam = {
3040                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
3041                 },
3042         },
3043         /* single-pass ipsec_esp descriptor */
3044         {
3045                 .aead = {
3046                         .base = {
3047                                 .cra_name = "authenc(hmac(md5),"
3048                                             "ecb(cipher_null))",
3049                                 .cra_driver_name = "authenc-hmac-md5-"
3050                                                    "ecb-cipher_null-caam",
3051                                 .cra_blocksize = NULL_BLOCK_SIZE,
3052                         },
3053                         .setkey = aead_setkey,
3054                         .setauthsize = aead_setauthsize,
3055                         .encrypt = aead_encrypt,
3056                         .decrypt = aead_decrypt,
3057                         .ivsize = NULL_IV_SIZE,
3058                         .maxauthsize = MD5_DIGEST_SIZE,
3059                 },
3060                 .caam = {
3061                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3062                                            OP_ALG_AAI_HMAC_PRECOMP,
3063                         .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3064                 },
3065         },
3066         {
3067                 .aead = {
3068                         .base = {
3069                                 .cra_name = "authenc(hmac(sha1),"
3070                                             "ecb(cipher_null))",
3071                                 .cra_driver_name = "authenc-hmac-sha1-"
3072                                                    "ecb-cipher_null-caam",
3073                                 .cra_blocksize = NULL_BLOCK_SIZE,
3074                         },
3075                         .setkey = aead_setkey,
3076                         .setauthsize = aead_setauthsize,
3077                         .encrypt = aead_encrypt,
3078                         .decrypt = aead_decrypt,
3079                         .ivsize = NULL_IV_SIZE,
3080                         .maxauthsize = SHA1_DIGEST_SIZE,
3081                 },
3082                 .caam = {
3083                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3084                                            OP_ALG_AAI_HMAC_PRECOMP,
3085                         .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3086                 },
3087         },
3088         {
3089                 .aead = {
3090                         .base = {
3091                                 .cra_name = "authenc(hmac(sha224),"
3092                                             "ecb(cipher_null))",
3093                                 .cra_driver_name = "authenc-hmac-sha224-"
3094                                                    "ecb-cipher_null-caam",
3095                                 .cra_blocksize = NULL_BLOCK_SIZE,
3096                         },
3097                         .setkey = aead_setkey,
3098                         .setauthsize = aead_setauthsize,
3099                         .encrypt = aead_encrypt,
3100                         .decrypt = aead_decrypt,
3101                         .ivsize = NULL_IV_SIZE,
3102                         .maxauthsize = SHA224_DIGEST_SIZE,
3103                 },
3104                 .caam = {
3105                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3106                                            OP_ALG_AAI_HMAC_PRECOMP,
3107                         .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3108                 },
3109         },
3110         {
3111                 .aead = {
3112                         .base = {
3113                                 .cra_name = "authenc(hmac(sha256),"
3114                                             "ecb(cipher_null))",
3115                                 .cra_driver_name = "authenc-hmac-sha256-"
3116                                                    "ecb-cipher_null-caam",
3117                                 .cra_blocksize = NULL_BLOCK_SIZE,
3118                         },
3119                         .setkey = aead_setkey,
3120                         .setauthsize = aead_setauthsize,
3121                         .encrypt = aead_encrypt,
3122                         .decrypt = aead_decrypt,
3123                         .ivsize = NULL_IV_SIZE,
3124                         .maxauthsize = SHA256_DIGEST_SIZE,
3125                 },
3126                 .caam = {
3127                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3128                                            OP_ALG_AAI_HMAC_PRECOMP,
3129                         .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3130                 },
3131         },
3132         {
3133                 .aead = {
3134                         .base = {
3135                                 .cra_name = "authenc(hmac(sha384),"
3136                                             "ecb(cipher_null))",
3137                                 .cra_driver_name = "authenc-hmac-sha384-"
3138                                                    "ecb-cipher_null-caam",
3139                                 .cra_blocksize = NULL_BLOCK_SIZE,
3140                         },
3141                         .setkey = aead_setkey,
3142                         .setauthsize = aead_setauthsize,
3143                         .encrypt = aead_encrypt,
3144                         .decrypt = aead_decrypt,
3145                         .ivsize = NULL_IV_SIZE,
3146                         .maxauthsize = SHA384_DIGEST_SIZE,
3147                 },
3148                 .caam = {
3149                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3150                                            OP_ALG_AAI_HMAC_PRECOMP,
3151                         .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3152                 },
3153         },
3154         {
3155                 .aead = {
3156                         .base = {
3157                                 .cra_name = "authenc(hmac(sha512),"
3158                                             "ecb(cipher_null))",
3159                                 .cra_driver_name = "authenc-hmac-sha512-"
3160                                                    "ecb-cipher_null-caam",
3161                                 .cra_blocksize = NULL_BLOCK_SIZE,
3162                         },
3163                         .setkey = aead_setkey,
3164                         .setauthsize = aead_setauthsize,
3165                         .encrypt = aead_encrypt,
3166                         .decrypt = aead_decrypt,
3167                         .ivsize = NULL_IV_SIZE,
3168                         .maxauthsize = SHA512_DIGEST_SIZE,
3169                 },
3170                 .caam = {
3171                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3172                                            OP_ALG_AAI_HMAC_PRECOMP,
3173                         .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3174                 },
3175         },
3176         {
3177                 .aead = {
3178                         .base = {
3179                                 .cra_name = "authenc(hmac(md5),cbc(aes))",
3180                                 .cra_driver_name = "authenc-hmac-md5-"
3181                                                    "cbc-aes-caam",
3182                                 .cra_blocksize = AES_BLOCK_SIZE,
3183                         },
3184                         .setkey = aead_setkey,
3185                         .setauthsize = aead_setauthsize,
3186                         .encrypt = aead_encrypt,
3187                         .decrypt = aead_decrypt,
3188                         .ivsize = AES_BLOCK_SIZE,
3189                         .maxauthsize = MD5_DIGEST_SIZE,
3190                 },
3191                 .caam = {
3192                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3193                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3194                                            OP_ALG_AAI_HMAC_PRECOMP,
3195                         .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3196                 },
3197         },
3198         {
3199                 .aead = {
3200                         .base = {
3201                                 .cra_name = "echainiv(authenc(hmac(md5),"
3202                                             "cbc(aes)))",
3203                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
3204                                                    "cbc-aes-caam",
3205                                 .cra_blocksize = AES_BLOCK_SIZE,
3206                         },
3207                         .setkey = aead_setkey,
3208                         .setauthsize = aead_setauthsize,
3209                         .encrypt = aead_encrypt,
3210                         .decrypt = aead_decrypt,
3211                         .ivsize = AES_BLOCK_SIZE,
3212                         .maxauthsize = MD5_DIGEST_SIZE,
3213                 },
3214                 .caam = {
3215                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3216                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3217                                            OP_ALG_AAI_HMAC_PRECOMP,
3218                         .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3219                         .geniv = true,
3220                 },
3221         },
3222         {
3223                 .aead = {
3224                         .base = {
3225                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
3226                                 .cra_driver_name = "authenc-hmac-sha1-"
3227                                                    "cbc-aes-caam",
3228                                 .cra_blocksize = AES_BLOCK_SIZE,
3229                         },
3230                         .setkey = aead_setkey,
3231                         .setauthsize = aead_setauthsize,
3232                         .encrypt = aead_encrypt,
3233                         .decrypt = aead_decrypt,
3234                         .ivsize = AES_BLOCK_SIZE,
3235                         .maxauthsize = SHA1_DIGEST_SIZE,
3236                 },
3237                 .caam = {
3238                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3239                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3240                                            OP_ALG_AAI_HMAC_PRECOMP,
3241                         .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3242                 },
3243         },
3244         {
3245                 .aead = {
3246                         .base = {
3247                                 .cra_name = "echainiv(authenc(hmac(sha1),"
3248                                             "cbc(aes)))",
3249                                 .cra_driver_name = "echainiv-authenc-"
3250                                                    "hmac-sha1-cbc-aes-caam",
3251                                 .cra_blocksize = AES_BLOCK_SIZE,
3252                         },
3253                         .setkey = aead_setkey,
3254                         .setauthsize = aead_setauthsize,
3255                         .encrypt = aead_encrypt,
3256                         .decrypt = aead_decrypt,
3257                         .ivsize = AES_BLOCK_SIZE,
3258                         .maxauthsize = SHA1_DIGEST_SIZE,
3259                 },
3260                 .caam = {
3261                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3262                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3263                                            OP_ALG_AAI_HMAC_PRECOMP,
3264                         .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3265                         .geniv = true,
3266                 },
3267         },
3268         {
3269                 .aead = {
3270                         .base = {
3271                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
3272                                 .cra_driver_name = "authenc-hmac-sha224-"
3273                                                    "cbc-aes-caam",
3274                                 .cra_blocksize = AES_BLOCK_SIZE,
3275                         },
3276                         .setkey = aead_setkey,
3277                         .setauthsize = aead_setauthsize,
3278                         .encrypt = aead_encrypt,
3279                         .decrypt = aead_decrypt,
3280                         .ivsize = AES_BLOCK_SIZE,
3281                         .maxauthsize = SHA224_DIGEST_SIZE,
3282                 },
3283                 .caam = {
3284                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3285                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3286                                            OP_ALG_AAI_HMAC_PRECOMP,
3287                         .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3288                 },
3289         },
3290         {
3291                 .aead = {
3292                         .base = {
3293                                 .cra_name = "echainiv(authenc(hmac(sha224),"
3294                                             "cbc(aes)))",
3295                                 .cra_driver_name = "echainiv-authenc-"
3296                                                    "hmac-sha224-cbc-aes-caam",
3297                                 .cra_blocksize = AES_BLOCK_SIZE,
3298                         },
3299                         .setkey = aead_setkey,
3300                         .setauthsize = aead_setauthsize,
3301                         .encrypt = aead_encrypt,
3302                         .decrypt = aead_decrypt,
3303                         .ivsize = AES_BLOCK_SIZE,
3304                         .maxauthsize = SHA224_DIGEST_SIZE,
3305                 },
3306                 .caam = {
3307                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3308                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3309                                            OP_ALG_AAI_HMAC_PRECOMP,
3310                         .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3311                         .geniv = true,
3312                 },
3313         },
3314         {
3315                 .aead = {
3316                         .base = {
3317                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
3318                                 .cra_driver_name = "authenc-hmac-sha256-"
3319                                                    "cbc-aes-caam",
3320                                 .cra_blocksize = AES_BLOCK_SIZE,
3321                         },
3322                         .setkey = aead_setkey,
3323                         .setauthsize = aead_setauthsize,
3324                         .encrypt = aead_encrypt,
3325                         .decrypt = aead_decrypt,
3326                         .ivsize = AES_BLOCK_SIZE,
3327                         .maxauthsize = SHA256_DIGEST_SIZE,
3328                 },
3329                 .caam = {
3330                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3331                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3332                                            OP_ALG_AAI_HMAC_PRECOMP,
3333                         .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3334                 },
3335         },
3336         {
3337                 .aead = {
3338                         .base = {
3339                                 .cra_name = "echainiv(authenc(hmac(sha256),"
3340                                             "cbc(aes)))",
3341                                 .cra_driver_name = "echainiv-authenc-"
3342                                                    "hmac-sha256-cbc-aes-caam",
3343                                 .cra_blocksize = AES_BLOCK_SIZE,
3344                         },
3345                         .setkey = aead_setkey,
3346                         .setauthsize = aead_setauthsize,
3347                         .encrypt = aead_encrypt,
3348                         .decrypt = aead_decrypt,
3349                         .ivsize = AES_BLOCK_SIZE,
3350                         .maxauthsize = SHA256_DIGEST_SIZE,
3351                 },
3352                 .caam = {
3353                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3354                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3355                                            OP_ALG_AAI_HMAC_PRECOMP,
3356                         .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3357                         .geniv = true,
3358                 },
3359         },
3360         {
3361                 .aead = {
3362                         .base = {
3363                                 .cra_name = "authenc(hmac(sha384),cbc(aes))",
3364                                 .cra_driver_name = "authenc-hmac-sha384-"
3365                                                    "cbc-aes-caam",
3366                                 .cra_blocksize = AES_BLOCK_SIZE,
3367                         },
3368                         .setkey = aead_setkey,
3369                         .setauthsize = aead_setauthsize,
3370                         .encrypt = aead_encrypt,
3371                         .decrypt = aead_decrypt,
3372                         .ivsize = AES_BLOCK_SIZE,
3373                         .maxauthsize = SHA384_DIGEST_SIZE,
3374                 },
3375                 .caam = {
3376                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3377                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3378                                            OP_ALG_AAI_HMAC_PRECOMP,
3379                         .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3380                 },
3381         },
3382         {
3383                 .aead = {
3384                         .base = {
3385                                 .cra_name = "echainiv(authenc(hmac(sha384),"
3386                                             "cbc(aes)))",
3387                                 .cra_driver_name = "echainiv-authenc-"
3388                                                    "hmac-sha384-cbc-aes-caam",
3389                                 .cra_blocksize = AES_BLOCK_SIZE,
3390                         },
3391                         .setkey = aead_setkey,
3392                         .setauthsize = aead_setauthsize,
3393                         .encrypt = aead_encrypt,
3394                         .decrypt = aead_decrypt,
3395                         .ivsize = AES_BLOCK_SIZE,
3396                         .maxauthsize = SHA384_DIGEST_SIZE,
3397                 },
3398                 .caam = {
3399                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3400                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3401                                            OP_ALG_AAI_HMAC_PRECOMP,
3402                         .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3403                         .geniv = true,
3404                 },
3405         },
3406         {
3407                 .aead = {
3408                         .base = {
3409                                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
3410                                 .cra_driver_name = "authenc-hmac-sha512-"
3411                                                    "cbc-aes-caam",
3412                                 .cra_blocksize = AES_BLOCK_SIZE,
3413                         },
3414                         .setkey = aead_setkey,
3415                         .setauthsize = aead_setauthsize,
3416                         .encrypt = aead_encrypt,
3417                         .decrypt = aead_decrypt,
3418                         .ivsize = AES_BLOCK_SIZE,
3419                         .maxauthsize = SHA512_DIGEST_SIZE,
3420                 },
3421                 .caam = {
3422                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3423                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3424                                            OP_ALG_AAI_HMAC_PRECOMP,
3425                         .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3426                 },
3427         },
3428         {
3429                 .aead = {
3430                         .base = {
3431                                 .cra_name = "echainiv(authenc(hmac(sha512),"
3432                                             "cbc(aes)))",
3433                                 .cra_driver_name = "echainiv-authenc-"
3434                                                    "hmac-sha512-cbc-aes-caam",
3435                                 .cra_blocksize = AES_BLOCK_SIZE,
3436                         },
3437                         .setkey = aead_setkey,
3438                         .setauthsize = aead_setauthsize,
3439                         .encrypt = aead_encrypt,
3440                         .decrypt = aead_decrypt,
3441                         .ivsize = AES_BLOCK_SIZE,
3442                         .maxauthsize = SHA512_DIGEST_SIZE,
3443                 },
3444                 .caam = {
3445                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3446                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3447                                            OP_ALG_AAI_HMAC_PRECOMP,
3448                         .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3449                         .geniv = true,
3450                 },
3451         },
3452         {
3453                 .aead = {
3454                         .base = {
3455                                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
3456                                 .cra_driver_name = "authenc-hmac-md5-"
3457                                                    "cbc-des3_ede-caam",
3458                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3459                         },
3460                         .setkey = aead_setkey,
3461                         .setauthsize = aead_setauthsize,
3462                         .encrypt = aead_encrypt,
3463                         .decrypt = aead_decrypt,
3464                         .ivsize = DES3_EDE_BLOCK_SIZE,
3465                         .maxauthsize = MD5_DIGEST_SIZE,
3466                 },
3467                 .caam = {
3468                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3469                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3470                                            OP_ALG_AAI_HMAC_PRECOMP,
3471                         .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3472                 }
3473         },
3474         {
3475                 .aead = {
3476                         .base = {
3477                                 .cra_name = "echainiv(authenc(hmac(md5),"
3478                                             "cbc(des3_ede)))",
3479                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
3480                                                    "cbc-des3_ede-caam",
3481                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3482                         },
3483                         .setkey = aead_setkey,
3484                         .setauthsize = aead_setauthsize,
3485                         .encrypt = aead_encrypt,
3486                         .decrypt = aead_decrypt,
3487                         .ivsize = DES3_EDE_BLOCK_SIZE,
3488                         .maxauthsize = MD5_DIGEST_SIZE,
3489                 },
3490                 .caam = {
3491                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3492                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3493                                            OP_ALG_AAI_HMAC_PRECOMP,
3494                         .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3495                         .geniv = true,
3496                 }
3497         },
3498         {
3499                 .aead = {
3500                         .base = {
3501                                 .cra_name = "authenc(hmac(sha1),"
3502                                             "cbc(des3_ede))",
3503                                 .cra_driver_name = "authenc-hmac-sha1-"
3504                                                    "cbc-des3_ede-caam",
3505                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3506                         },
3507                         .setkey = aead_setkey,
3508                         .setauthsize = aead_setauthsize,
3509                         .encrypt = aead_encrypt,
3510                         .decrypt = aead_decrypt,
3511                         .ivsize = DES3_EDE_BLOCK_SIZE,
3512                         .maxauthsize = SHA1_DIGEST_SIZE,
3513                 },
3514                 .caam = {
3515                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3516                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3517                                            OP_ALG_AAI_HMAC_PRECOMP,
3518                         .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3519                 },
3520         },
3521         {
3522                 .aead = {
3523                         .base = {
3524                                 .cra_name = "echainiv(authenc(hmac(sha1),"
3525                                             "cbc(des3_ede)))",
3526                                 .cra_driver_name = "echainiv-authenc-"
3527                                                    "hmac-sha1-"
3528                                                    "cbc-des3_ede-caam",
3529                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3530                         },
3531                         .setkey = aead_setkey,
3532                         .setauthsize = aead_setauthsize,
3533                         .encrypt = aead_encrypt,
3534                         .decrypt = aead_decrypt,
3535                         .ivsize = DES3_EDE_BLOCK_SIZE,
3536                         .maxauthsize = SHA1_DIGEST_SIZE,
3537                 },
3538                 .caam = {
3539                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3540                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3541                                            OP_ALG_AAI_HMAC_PRECOMP,
3542                         .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3543                         .geniv = true,
3544                 },
3545         },
3546         {
3547                 .aead = {
3548                         .base = {
3549                                 .cra_name = "authenc(hmac(sha224),"
3550                                             "cbc(des3_ede))",
3551                                 .cra_driver_name = "authenc-hmac-sha224-"
3552                                                    "cbc-des3_ede-caam",
3553                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3554                         },
3555                         .setkey = aead_setkey,
3556                         .setauthsize = aead_setauthsize,
3557                         .encrypt = aead_encrypt,
3558                         .decrypt = aead_decrypt,
3559                         .ivsize = DES3_EDE_BLOCK_SIZE,
3560                         .maxauthsize = SHA224_DIGEST_SIZE,
3561                 },
3562                 .caam = {
3563                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3564                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3565                                            OP_ALG_AAI_HMAC_PRECOMP,
3566                         .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3567                 },
3568         },
3569         {
3570                 .aead = {
3571                         .base = {
3572                                 .cra_name = "echainiv(authenc(hmac(sha224),"
3573                                             "cbc(des3_ede)))",
3574                                 .cra_driver_name = "echainiv-authenc-"
3575                                                    "hmac-sha224-"
3576                                                    "cbc-des3_ede-caam",
3577                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3578                         },
3579                         .setkey = aead_setkey,
3580                         .setauthsize = aead_setauthsize,
3581                         .encrypt = aead_encrypt,
3582                         .decrypt = aead_decrypt,
3583                         .ivsize = DES3_EDE_BLOCK_SIZE,
3584                         .maxauthsize = SHA224_DIGEST_SIZE,
3585                 },
3586                 .caam = {
3587                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3588                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3589                                            OP_ALG_AAI_HMAC_PRECOMP,
3590                         .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3591                         .geniv = true,
3592                 },
3593         },
3594         {
3595                 .aead = {
3596                         .base = {
3597                                 .cra_name = "authenc(hmac(sha256),"
3598                                             "cbc(des3_ede))",
3599                                 .cra_driver_name = "authenc-hmac-sha256-"
3600                                                    "cbc-des3_ede-caam",
3601                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3602                         },
3603                         .setkey = aead_setkey,
3604                         .setauthsize = aead_setauthsize,
3605                         .encrypt = aead_encrypt,
3606                         .decrypt = aead_decrypt,
3607                         .ivsize = DES3_EDE_BLOCK_SIZE,
3608                         .maxauthsize = SHA256_DIGEST_SIZE,
3609                 },
3610                 .caam = {
3611                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3612                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3613                                            OP_ALG_AAI_HMAC_PRECOMP,
3614                         .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3615                 },
3616         },
3617         {
3618                 .aead = {
3619                         .base = {
3620                                 .cra_name = "echainiv(authenc(hmac(sha256),"
3621                                             "cbc(des3_ede)))",
3622                                 .cra_driver_name = "echainiv-authenc-"
3623                                                    "hmac-sha256-"
3624                                                    "cbc-des3_ede-caam",
3625                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3626                         },
3627                         .setkey = aead_setkey,
3628                         .setauthsize = aead_setauthsize,
3629                         .encrypt = aead_encrypt,
3630                         .decrypt = aead_decrypt,
3631                         .ivsize = DES3_EDE_BLOCK_SIZE,
3632                         .maxauthsize = SHA256_DIGEST_SIZE,
3633                 },
3634                 .caam = {
3635                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3636                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3637                                            OP_ALG_AAI_HMAC_PRECOMP,
3638                         .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3639                         .geniv = true,
3640                 },
3641         },
3642         {
3643                 .aead = {
3644                         .base = {
3645                                 .cra_name = "authenc(hmac(sha384),"
3646                                             "cbc(des3_ede))",
3647                                 .cra_driver_name = "authenc-hmac-sha384-"
3648                                                    "cbc-des3_ede-caam",
3649                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3650                         },
3651                         .setkey = aead_setkey,
3652                         .setauthsize = aead_setauthsize,
3653                         .encrypt = aead_encrypt,
3654                         .decrypt = aead_decrypt,
3655                         .ivsize = DES3_EDE_BLOCK_SIZE,
3656                         .maxauthsize = SHA384_DIGEST_SIZE,
3657                 },
3658                 .caam = {
3659                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3660                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3661                                            OP_ALG_AAI_HMAC_PRECOMP,
3662                         .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3663                 },
3664         },
3665         {
3666                 .aead = {
3667                         .base = {
3668                                 .cra_name = "echainiv(authenc(hmac(sha384),"
3669                                             "cbc(des3_ede)))",
3670                                 .cra_driver_name = "echainiv-authenc-"
3671                                                    "hmac-sha384-"
3672                                                    "cbc-des3_ede-caam",
3673                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3674                         },
3675                         .setkey = aead_setkey,
3676                         .setauthsize = aead_setauthsize,
3677                         .encrypt = aead_encrypt,
3678                         .decrypt = aead_decrypt,
3679                         .ivsize = DES3_EDE_BLOCK_SIZE,
3680                         .maxauthsize = SHA384_DIGEST_SIZE,
3681                 },
3682                 .caam = {
3683                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3684                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3685                                            OP_ALG_AAI_HMAC_PRECOMP,
3686                         .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3687                         .geniv = true,
3688                 },
3689         },
3690         {
3691                 .aead = {
3692                         .base = {
3693                                 .cra_name = "authenc(hmac(sha512),"
3694                                             "cbc(des3_ede))",
3695                                 .cra_driver_name = "authenc-hmac-sha512-"
3696                                                    "cbc-des3_ede-caam",
3697                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3698                         },
3699                         .setkey = aead_setkey,
3700                         .setauthsize = aead_setauthsize,
3701                         .encrypt = aead_encrypt,
3702                         .decrypt = aead_decrypt,
3703                         .ivsize = DES3_EDE_BLOCK_SIZE,
3704                         .maxauthsize = SHA512_DIGEST_SIZE,
3705                 },
3706                 .caam = {
3707                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3708                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3709                                            OP_ALG_AAI_HMAC_PRECOMP,
3710                         .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3711                 },
3712         },
3713         {
3714                 .aead = {
3715                         .base = {
3716                                 .cra_name = "echainiv(authenc(hmac(sha512),"
3717                                             "cbc(des3_ede)))",
3718                                 .cra_driver_name = "echainiv-authenc-"
3719                                                    "hmac-sha512-"
3720                                                    "cbc-des3_ede-caam",
3721                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3722                         },
3723                         .setkey = aead_setkey,
3724                         .setauthsize = aead_setauthsize,
3725                         .encrypt = aead_encrypt,
3726                         .decrypt = aead_decrypt,
3727                         .ivsize = DES3_EDE_BLOCK_SIZE,
3728                         .maxauthsize = SHA512_DIGEST_SIZE,
3729                 },
3730                 .caam = {
3731                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3732                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3733                                            OP_ALG_AAI_HMAC_PRECOMP,
3734                         .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3735                         .geniv = true,
3736                 },
3737         },
3738         {
3739                 .aead = {
3740                         .base = {
3741                                 .cra_name = "authenc(hmac(md5),cbc(des))",
3742                                 .cra_driver_name = "authenc-hmac-md5-"
3743                                                    "cbc-des-caam",
3744                                 .cra_blocksize = DES_BLOCK_SIZE,
3745                         },
3746                         .setkey = aead_setkey,
3747                         .setauthsize = aead_setauthsize,
3748                         .encrypt = aead_encrypt,
3749                         .decrypt = aead_decrypt,
3750                         .ivsize = DES_BLOCK_SIZE,
3751                         .maxauthsize = MD5_DIGEST_SIZE,
3752                 },
3753                 .caam = {
3754                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3755                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3756                                            OP_ALG_AAI_HMAC_PRECOMP,
3757                         .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3758                 },
3759         },
3760         {
3761                 .aead = {
3762                         .base = {
3763                                 .cra_name = "echainiv(authenc(hmac(md5),"
3764                                             "cbc(des)))",
3765                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
3766                                                    "cbc-des-caam",
3767                                 .cra_blocksize = DES_BLOCK_SIZE,
3768                         },
3769                         .setkey = aead_setkey,
3770                         .setauthsize = aead_setauthsize,
3771                         .encrypt = aead_encrypt,
3772                         .decrypt = aead_decrypt,
3773                         .ivsize = DES_BLOCK_SIZE,
3774                         .maxauthsize = MD5_DIGEST_SIZE,
3775                 },
3776                 .caam = {
3777                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3778                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3779                                            OP_ALG_AAI_HMAC_PRECOMP,
3780                         .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3781                         .geniv = true,
3782                 },
3783         },
3784         {
3785                 .aead = {
3786                         .base = {
3787                                 .cra_name = "authenc(hmac(sha1),cbc(des))",
3788                                 .cra_driver_name = "authenc-hmac-sha1-"
3789                                                    "cbc-des-caam",
3790                                 .cra_blocksize = DES_BLOCK_SIZE,
3791                         },
3792                         .setkey = aead_setkey,
3793                         .setauthsize = aead_setauthsize,
3794                         .encrypt = aead_encrypt,
3795                         .decrypt = aead_decrypt,
3796                         .ivsize = DES_BLOCK_SIZE,
3797                         .maxauthsize = SHA1_DIGEST_SIZE,
3798                 },
3799                 .caam = {
3800                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3801                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3802                                            OP_ALG_AAI_HMAC_PRECOMP,
3803                         .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3804                 },
3805         },
3806         {
3807                 .aead = {
3808                         .base = {
3809                                 .cra_name = "echainiv(authenc(hmac(sha1),"
3810                                             "cbc(des)))",
3811                                 .cra_driver_name = "echainiv-authenc-"
3812                                                    "hmac-sha1-cbc-des-caam",
3813                                 .cra_blocksize = DES_BLOCK_SIZE,
3814                         },
3815                         .setkey = aead_setkey,
3816                         .setauthsize = aead_setauthsize,
3817                         .encrypt = aead_encrypt,
3818                         .decrypt = aead_decrypt,
3819                         .ivsize = DES_BLOCK_SIZE,
3820                         .maxauthsize = SHA1_DIGEST_SIZE,
3821                 },
3822                 .caam = {
3823                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3824                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3825                                            OP_ALG_AAI_HMAC_PRECOMP,
3826                         .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3827                         .geniv = true,
3828                 },
3829         },
3830         {
3831                 .aead = {
3832                         .base = {
3833                                 .cra_name = "authenc(hmac(sha224),cbc(des))",
3834                                 .cra_driver_name = "authenc-hmac-sha224-"
3835                                                    "cbc-des-caam",
3836                                 .cra_blocksize = DES_BLOCK_SIZE,
3837                         },
3838                         .setkey = aead_setkey,
3839                         .setauthsize = aead_setauthsize,
3840                         .encrypt = aead_encrypt,
3841                         .decrypt = aead_decrypt,
3842                         .ivsize = DES_BLOCK_SIZE,
3843                         .maxauthsize = SHA224_DIGEST_SIZE,
3844                 },
3845                 .caam = {
3846                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3847                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3848                                            OP_ALG_AAI_HMAC_PRECOMP,
3849                         .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3850                 },
3851         },
3852         {
3853                 .aead = {
3854                         .base = {
3855                                 .cra_name = "echainiv(authenc(hmac(sha224),"
3856                                             "cbc(des)))",
3857                                 .cra_driver_name = "echainiv-authenc-"
3858                                                    "hmac-sha224-cbc-des-caam",
3859                                 .cra_blocksize = DES_BLOCK_SIZE,
3860                         },
3861                         .setkey = aead_setkey,
3862                         .setauthsize = aead_setauthsize,
3863                         .encrypt = aead_encrypt,
3864                         .decrypt = aead_decrypt,
3865                         .ivsize = DES_BLOCK_SIZE,
3866                         .maxauthsize = SHA224_DIGEST_SIZE,
3867                 },
3868                 .caam = {
3869                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3870                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3871                                            OP_ALG_AAI_HMAC_PRECOMP,
3872                         .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3873                         .geniv = true,
3874                 },
3875         },
3876         {
3877                 .aead = {
3878                         .base = {
3879                                 .cra_name = "authenc(hmac(sha256),cbc(des))",
3880                                 .cra_driver_name = "authenc-hmac-sha256-"
3881                                                    "cbc-des-caam",
3882                                 .cra_blocksize = DES_BLOCK_SIZE,
3883                         },
3884                         .setkey = aead_setkey,
3885                         .setauthsize = aead_setauthsize,
3886                         .encrypt = aead_encrypt,
3887                         .decrypt = aead_decrypt,
3888                         .ivsize = DES_BLOCK_SIZE,
3889                         .maxauthsize = SHA256_DIGEST_SIZE,
3890                 },
3891                 .caam = {
3892                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3893                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3894                                            OP_ALG_AAI_HMAC_PRECOMP,
3895                         .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3896                 },
3897         },
3898         {
3899                 .aead = {
3900                         .base = {
3901                                 .cra_name = "echainiv(authenc(hmac(sha256),"
3902                                             "cbc(des)))",
3903                                 .cra_driver_name = "echainiv-authenc-"
3904                                                    "hmac-sha256-cbc-des-caam",
3905                                 .cra_blocksize = DES_BLOCK_SIZE,
3906                         },
3907                         .setkey = aead_setkey,
3908                         .setauthsize = aead_setauthsize,
3909                         .encrypt = aead_encrypt,
3910                         .decrypt = aead_decrypt,
3911                         .ivsize = DES_BLOCK_SIZE,
3912                         .maxauthsize = SHA256_DIGEST_SIZE,
3913                 },
3914                 .caam = {
3915                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3916                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3917                                            OP_ALG_AAI_HMAC_PRECOMP,
3918                         .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3919                         .geniv = true,
3920                 },
3921         },
3922         {
3923                 .aead = {
3924                         .base = {
3925                                 .cra_name = "authenc(hmac(sha384),cbc(des))",
3926                                 .cra_driver_name = "authenc-hmac-sha384-"
3927                                                    "cbc-des-caam",
3928                                 .cra_blocksize = DES_BLOCK_SIZE,
3929                         },
3930                         .setkey = aead_setkey,
3931                         .setauthsize = aead_setauthsize,
3932                         .encrypt = aead_encrypt,
3933                         .decrypt = aead_decrypt,
3934                         .ivsize = DES_BLOCK_SIZE,
3935                         .maxauthsize = SHA384_DIGEST_SIZE,
3936                 },
3937                 .caam = {
3938                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3939                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3940                                            OP_ALG_AAI_HMAC_PRECOMP,
3941                         .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3942                 },
3943         },
3944         {
3945                 .aead = {
3946                         .base = {
3947                                 .cra_name = "echainiv(authenc(hmac(sha384),"
3948                                             "cbc(des)))",
3949                                 .cra_driver_name = "echainiv-authenc-"
3950                                                    "hmac-sha384-cbc-des-caam",
3951                                 .cra_blocksize = DES_BLOCK_SIZE,
3952                         },
3953                         .setkey = aead_setkey,
3954                         .setauthsize = aead_setauthsize,
3955                         .encrypt = aead_encrypt,
3956                         .decrypt = aead_decrypt,
3957                         .ivsize = DES_BLOCK_SIZE,
3958                         .maxauthsize = SHA384_DIGEST_SIZE,
3959                 },
3960                 .caam = {
3961                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3962                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3963                                            OP_ALG_AAI_HMAC_PRECOMP,
3964                         .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3965                         .geniv = true,
3966                 },
3967         },
3968         {
3969                 .aead = {
3970                         .base = {
3971                                 .cra_name = "authenc(hmac(sha512),cbc(des))",
3972                                 .cra_driver_name = "authenc-hmac-sha512-"
3973                                                    "cbc-des-caam",
3974                                 .cra_blocksize = DES_BLOCK_SIZE,
3975                         },
3976                         .setkey = aead_setkey,
3977                         .setauthsize = aead_setauthsize,
3978                         .encrypt = aead_encrypt,
3979                         .decrypt = aead_decrypt,
3980                         .ivsize = DES_BLOCK_SIZE,
3981                         .maxauthsize = SHA512_DIGEST_SIZE,
3982                 },
3983                 .caam = {
3984                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3985                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3986                                            OP_ALG_AAI_HMAC_PRECOMP,
3987                         .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3988                 },
3989         },
3990         {
3991                 .aead = {
3992                         .base = {
3993                                 .cra_name = "echainiv(authenc(hmac(sha512),"
3994                                             "cbc(des)))",
3995                                 .cra_driver_name = "echainiv-authenc-"
3996                                                    "hmac-sha512-cbc-des-caam",
3997                                 .cra_blocksize = DES_BLOCK_SIZE,
3998                         },
3999                         .setkey = aead_setkey,
4000                         .setauthsize = aead_setauthsize,
4001                         .encrypt = aead_encrypt,
4002                         .decrypt = aead_decrypt,
4003                         .ivsize = DES_BLOCK_SIZE,
4004                         .maxauthsize = SHA512_DIGEST_SIZE,
4005                 },
4006                 .caam = {
4007                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
4008                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
4009                                            OP_ALG_AAI_HMAC_PRECOMP,
4010                         .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
4011                         .geniv = true,
4012                 },
4013         },
4014         {
4015                 .aead = {
4016                         .base = {
4017                                 .cra_name = "authenc(hmac(md5),"
4018                                             "rfc3686(ctr(aes)))",
4019                                 .cra_driver_name = "authenc-hmac-md5-"
4020                                                    "rfc3686-ctr-aes-caam",
4021                                 .cra_blocksize = 1,
4022                         },
4023                         .setkey = aead_setkey,
4024                         .setauthsize = aead_setauthsize,
4025                         .encrypt = aead_encrypt,
4026                         .decrypt = aead_decrypt,
4027                         .ivsize = CTR_RFC3686_IV_SIZE,
4028                         .maxauthsize = MD5_DIGEST_SIZE,
4029                 },
4030                 .caam = {
4031                         .class1_alg_type = OP_ALG_ALGSEL_AES |
4032                                            OP_ALG_AAI_CTR_MOD128,
4033                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
4034                                            OP_ALG_AAI_HMAC_PRECOMP,
4035                         .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
4036                         .rfc3686 = true,
4037                 },
4038         },
4039         {
4040                 .aead = {
4041                         .base = {
4042                                 .cra_name = "seqiv(authenc("
4043                                             "hmac(md5),rfc3686(ctr(aes))))",
4044                                 .cra_driver_name = "seqiv-authenc-hmac-md5-"
4045                                                    "rfc3686-ctr-aes-caam",
4046                                 .cra_blocksize = 1,
4047                         },
4048                         .setkey = aead_setkey,
4049                         .setauthsize = aead_setauthsize,
4050                         .encrypt = aead_encrypt,
4051                         .decrypt = aead_decrypt,
4052                         .ivsize = CTR_RFC3686_IV_SIZE,
4053                         .maxauthsize = MD5_DIGEST_SIZE,
4054                 },
4055                 .caam = {
4056                         .class1_alg_type = OP_ALG_ALGSEL_AES |
4057                                            OP_ALG_AAI_CTR_MOD128,
4058                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
4059                                            OP_ALG_AAI_HMAC_PRECOMP,
4060                         .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
4061                         .rfc3686 = true,
4062                         .geniv = true,
4063                 },
4064         },
4065         {
4066                 .aead = {
4067                         .base = {
4068                                 .cra_name = "authenc(hmac(sha1),"
4069                                             "rfc3686(ctr(aes)))",
4070                                 .cra_driver_name = "authenc-hmac-sha1-"
4071                                                    "rfc3686-ctr-aes-caam",
4072                                 .cra_blocksize = 1,
4073                         },
4074                         .setkey = aead_setkey,
4075                         .setauthsize = aead_setauthsize,
4076                         .encrypt = aead_encrypt,
4077                         .decrypt = aead_decrypt,
4078                         .ivsize = CTR_RFC3686_IV_SIZE,
4079                         .maxauthsize = SHA1_DIGEST_SIZE,
4080                 },
4081                 .caam = {
4082                         .class1_alg_type = OP_ALG_ALGSEL_AES |
4083                                            OP_ALG_AAI_CTR_MOD128,
4084                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
4085                                            OP_ALG_AAI_HMAC_PRECOMP,
4086                         .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
4087                         .rfc3686 = true,
4088                 },
4089         },
4090         {
4091                 .aead = {
4092                         .base = {
4093                                 .cra_name = "seqiv(authenc("
4094                                             "hmac(sha1),rfc3686(ctr(aes))))",
4095                                 .cra_driver_name = "seqiv-authenc-hmac-sha1-"
4096                                                    "rfc3686-ctr-aes-caam",
4097                                 .cra_blocksize = 1,
4098                         },
4099                         .setkey = aead_setkey,
4100                         .setauthsize = aead_setauthsize,
4101                         .encrypt = aead_encrypt,
4102                         .decrypt = aead_decrypt,
4103                         .ivsize = CTR_RFC3686_IV_SIZE,
4104                         .maxauthsize = SHA1_DIGEST_SIZE,
4105                 },
4106                 .caam = {
4107                         .class1_alg_type = OP_ALG_ALGSEL_AES |
4108                                            OP_ALG_AAI_CTR_MOD128,
4109                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
4110                                            OP_ALG_AAI_HMAC_PRECOMP,
4111                         .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
4112                         .rfc3686 = true,
4113                         .geniv = true,
4114                 },
4115         },
4116         {
4117                 .aead = {
4118                         .base = {
4119                                 .cra_name = "authenc(hmac(sha224),"
4120                                             "rfc3686(ctr(aes)))",
4121                                 .cra_driver_name = "authenc-hmac-sha224-"
4122                                                    "rfc3686-ctr-aes-caam",
4123                                 .cra_blocksize = 1,
4124                         },
4125                         .setkey = aead_setkey,
4126                         .setauthsize = aead_setauthsize,
4127                         .encrypt = aead_encrypt,
4128                         .decrypt = aead_decrypt,
4129                         .ivsize = CTR_RFC3686_IV_SIZE,
4130                         .maxauthsize = SHA224_DIGEST_SIZE,
4131                 },
4132                 .caam = {
4133                         .class1_alg_type = OP_ALG_ALGSEL_AES |
4134                                            OP_ALG_AAI_CTR_MOD128,
4135                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
4136                                            OP_ALG_AAI_HMAC_PRECOMP,
4137                         .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
4138                         .rfc3686 = true,
4139                 },
4140         },
4141         {
4142                 .aead = {
4143                         .base = {
4144                                 .cra_name = "seqiv(authenc("
4145                                             "hmac(sha224),rfc3686(ctr(aes))))",
4146                                 .cra_driver_name = "seqiv-authenc-hmac-sha224-"
4147                                                    "rfc3686-ctr-aes-caam",
4148                                 .cra_blocksize = 1,
4149                         },
4150                         .setkey = aead_setkey,
4151                         .setauthsize = aead_setauthsize,
4152                         .encrypt = aead_encrypt,
4153                         .decrypt = aead_decrypt,
4154                         .ivsize = CTR_RFC3686_IV_SIZE,
4155                         .maxauthsize = SHA224_DIGEST_SIZE,
4156                 },
4157                 .caam = {
4158                         .class1_alg_type = OP_ALG_ALGSEL_AES |
4159                                            OP_ALG_AAI_CTR_MOD128,
4160                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
4161                                            OP_ALG_AAI_HMAC_PRECOMP,
4162                         .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
4163                         .rfc3686 = true,
4164                         .geniv = true,
4165                 },
4166         },
4167         {
4168                 .aead = {
4169                         .base = {
4170                                 .cra_name = "authenc(hmac(sha256),"
4171                                             "rfc3686(ctr(aes)))",
4172                                 .cra_driver_name = "authenc-hmac-sha256-"
4173                                                    "rfc3686-ctr-aes-caam",
4174                                 .cra_blocksize = 1,
4175                         },
4176                         .setkey = aead_setkey,
4177                         .setauthsize = aead_setauthsize,
4178                         .encrypt = aead_encrypt,
4179                         .decrypt = aead_decrypt,
4180                         .ivsize = CTR_RFC3686_IV_SIZE,
4181                         .maxauthsize = SHA256_DIGEST_SIZE,
4182                 },
4183                 .caam = {
4184                         .class1_alg_type = OP_ALG_ALGSEL_AES |
4185                                            OP_ALG_AAI_CTR_MOD128,
4186                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
4187                                            OP_ALG_AAI_HMAC_PRECOMP,
4188                         .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
4189                         .rfc3686 = true,
4190                 },
4191         },
4192         {
4193                 .aead = {
4194                         .base = {
4195                                 .cra_name = "seqiv(authenc(hmac(sha256),"
4196                                             "rfc3686(ctr(aes))))",
4197                                 .cra_driver_name = "seqiv-authenc-hmac-sha256-"
4198                                                    "rfc3686-ctr-aes-caam",
4199                                 .cra_blocksize = 1,
4200                         },
4201                         .setkey = aead_setkey,
4202                         .setauthsize = aead_setauthsize,
4203                         .encrypt = aead_encrypt,
4204                         .decrypt = aead_decrypt,
4205                         .ivsize = CTR_RFC3686_IV_SIZE,
4206                         .maxauthsize = SHA256_DIGEST_SIZE,
4207                 },
4208                 .caam = {
4209                         .class1_alg_type = OP_ALG_ALGSEL_AES |
4210                                            OP_ALG_AAI_CTR_MOD128,
4211                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
4212                                            OP_ALG_AAI_HMAC_PRECOMP,
4213                         .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
4214                         .rfc3686 = true,
4215                         .geniv = true,
4216                 },
4217         },
4218         {
4219                 .aead = {
4220                         .base = {
4221                                 .cra_name = "authenc(hmac(sha384),"
4222                                             "rfc3686(ctr(aes)))",
4223                                 .cra_driver_name = "authenc-hmac-sha384-"
4224                                                    "rfc3686-ctr-aes-caam",
4225                                 .cra_blocksize = 1,
4226                         },
4227                         .setkey = aead_setkey,
4228                         .setauthsize = aead_setauthsize,
4229                         .encrypt = aead_encrypt,
4230                         .decrypt = aead_decrypt,
4231                         .ivsize = CTR_RFC3686_IV_SIZE,
4232                         .maxauthsize = SHA384_DIGEST_SIZE,
4233                 },
4234                 .caam = {
4235                         .class1_alg_type = OP_ALG_ALGSEL_AES |
4236                                            OP_ALG_AAI_CTR_MOD128,
4237                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
4238                                            OP_ALG_AAI_HMAC_PRECOMP,
4239                         .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
4240                         .rfc3686 = true,
4241                 },
4242         },
4243         {
4244                 .aead = {
4245                         .base = {
4246                                 .cra_name = "seqiv(authenc(hmac(sha384),"
4247                                             "rfc3686(ctr(aes))))",
4248                                 .cra_driver_name = "seqiv-authenc-hmac-sha384-"
4249                                                    "rfc3686-ctr-aes-caam",
4250                                 .cra_blocksize = 1,
4251                         },
4252                         .setkey = aead_setkey,
4253                         .setauthsize = aead_setauthsize,
4254                         .encrypt = aead_encrypt,
4255                         .decrypt = aead_decrypt,
4256                         .ivsize = CTR_RFC3686_IV_SIZE,
4257                         .maxauthsize = SHA384_DIGEST_SIZE,
4258                 },
4259                 .caam = {
4260                         .class1_alg_type = OP_ALG_ALGSEL_AES |
4261                                            OP_ALG_AAI_CTR_MOD128,
4262                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
4263                                            OP_ALG_AAI_HMAC_PRECOMP,
4264                         .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
4265                         .rfc3686 = true,
4266                         .geniv = true,
4267                 },
4268         },
4269         {
4270                 .aead = {
4271                         .base = {
4272                                 .cra_name = "authenc(hmac(sha512),"
4273                                             "rfc3686(ctr(aes)))",
4274                                 .cra_driver_name = "authenc-hmac-sha512-"
4275                                                    "rfc3686-ctr-aes-caam",
4276                                 .cra_blocksize = 1,
4277                         },
4278                         .setkey = aead_setkey,
4279                         .setauthsize = aead_setauthsize,
4280                         .encrypt = aead_encrypt,
4281                         .decrypt = aead_decrypt,
4282                         .ivsize = CTR_RFC3686_IV_SIZE,
4283                         .maxauthsize = SHA512_DIGEST_SIZE,
4284                 },
4285                 .caam = {
4286                         .class1_alg_type = OP_ALG_ALGSEL_AES |
4287                                            OP_ALG_AAI_CTR_MOD128,
4288                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
4289                                            OP_ALG_AAI_HMAC_PRECOMP,
4290                         .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
4291                         .rfc3686 = true,
4292                 },
4293         },
4294         {
4295                 .aead = {
4296                         .base = {
4297                                 .cra_name = "seqiv(authenc(hmac(sha512),"
4298                                             "rfc3686(ctr(aes))))",
4299                                 .cra_driver_name = "seqiv-authenc-hmac-sha512-"
4300                                                    "rfc3686-ctr-aes-caam",
4301                                 .cra_blocksize = 1,
4302                         },
4303                         .setkey = aead_setkey,
4304                         .setauthsize = aead_setauthsize,
4305                         .encrypt = aead_encrypt,
4306                         .decrypt = aead_decrypt,
4307                         .ivsize = CTR_RFC3686_IV_SIZE,
4308                         .maxauthsize = SHA512_DIGEST_SIZE,
4309                 },
4310                 .caam = {
4311                         .class1_alg_type = OP_ALG_ALGSEL_AES |
4312                                            OP_ALG_AAI_CTR_MOD128,
4313                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
4314                                            OP_ALG_AAI_HMAC_PRECOMP,
4315                         .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
4316                         .rfc3686 = true,
4317                         .geniv = true,
4318                 },
4319         },
4320 };
4321
4322 struct caam_crypto_alg {
4323         struct crypto_alg crypto_alg;
4324         struct list_head entry;
4325         struct caam_alg_entry caam;
4326 };
4327
4328 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam)
4329 {
4330         ctx->jrdev = caam_jr_alloc();
4331         if (IS_ERR(ctx->jrdev)) {
4332                 pr_err("Job Ring Device allocation for transform failed\n");
4333                 return PTR_ERR(ctx->jrdev);
4334         }
4335
4336         /* copy descriptor header template value */
4337         ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
4338         ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
4339         ctx->alg_op = OP_TYPE_CLASS2_ALG | caam->alg_op;
4340
4341         return 0;
4342 }
4343
4344 static int caam_cra_init(struct crypto_tfm *tfm)
4345 {
4346         struct crypto_alg *alg = tfm->__crt_alg;
4347         struct caam_crypto_alg *caam_alg =
4348                  container_of(alg, struct caam_crypto_alg, crypto_alg);
4349         struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
4350
4351         return caam_init_common(ctx, &caam_alg->caam);
4352 }
4353
4354 static int caam_aead_init(struct crypto_aead *tfm)
4355 {
4356         struct aead_alg *alg = crypto_aead_alg(tfm);
4357         struct caam_aead_alg *caam_alg =
4358                  container_of(alg, struct caam_aead_alg, aead);
4359         struct caam_ctx *ctx = crypto_aead_ctx(tfm);
4360
4361         return caam_init_common(ctx, &caam_alg->caam);
4362 }
4363
4364 static void caam_exit_common(struct caam_ctx *ctx)
4365 {
4366         if (ctx->sh_desc_enc_dma &&
4367             !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma))
4368                 dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma,
4369                                  desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE);
4370         if (ctx->sh_desc_dec_dma &&
4371             !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma))
4372                 dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma,
4373                                  desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE);
4374         if (ctx->sh_desc_givenc_dma &&
4375             !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma))
4376                 dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma,
4377                                  desc_bytes(ctx->sh_desc_givenc),
4378                                  DMA_TO_DEVICE);
4379         if (ctx->key_dma &&
4380             !dma_mapping_error(ctx->jrdev, ctx->key_dma))
4381                 dma_unmap_single(ctx->jrdev, ctx->key_dma,
4382                                  ctx->enckeylen + ctx->split_key_pad_len,
4383                                  DMA_TO_DEVICE);
4384
4385         caam_jr_free(ctx->jrdev);
4386 }
4387
4388 static void caam_cra_exit(struct crypto_tfm *tfm)
4389 {
4390         caam_exit_common(crypto_tfm_ctx(tfm));
4391 }
4392
4393 static void caam_aead_exit(struct crypto_aead *tfm)
4394 {
4395         caam_exit_common(crypto_aead_ctx(tfm));
4396 }
4397
4398 static void __exit caam_algapi_exit(void)
4399 {
4400
4401         struct caam_crypto_alg *t_alg, *n;
4402         int i;
4403
4404         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
4405                 struct caam_aead_alg *t_alg = driver_aeads + i;
4406
4407                 if (t_alg->registered)
4408                         crypto_unregister_aead(&t_alg->aead);
4409         }
4410
4411         if (!alg_list.next)
4412                 return;
4413
4414         list_for_each_entry_safe(t_alg, n, &alg_list, entry) {
4415                 crypto_unregister_alg(&t_alg->crypto_alg);
4416                 list_del(&t_alg->entry);
4417                 kfree(t_alg);
4418         }
4419 }
4420
4421 static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template
4422                                               *template)
4423 {
4424         struct caam_crypto_alg *t_alg;
4425         struct crypto_alg *alg;
4426
4427         t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
4428         if (!t_alg) {
4429                 pr_err("failed to allocate t_alg\n");
4430                 return ERR_PTR(-ENOMEM);
4431         }
4432
4433         alg = &t_alg->crypto_alg;
4434
4435         snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name);
4436         snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4437                  template->driver_name);
4438         alg->cra_module = THIS_MODULE;
4439         alg->cra_init = caam_cra_init;
4440         alg->cra_exit = caam_cra_exit;
4441         alg->cra_priority = CAAM_CRA_PRIORITY;
4442         alg->cra_blocksize = template->blocksize;
4443         alg->cra_alignmask = 0;
4444         alg->cra_ctxsize = sizeof(struct caam_ctx);
4445         alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY |
4446                          template->type;
4447         switch (template->type) {
4448         case CRYPTO_ALG_TYPE_GIVCIPHER:
4449                 alg->cra_type = &crypto_givcipher_type;
4450                 alg->cra_ablkcipher = template->template_ablkcipher;
4451                 break;
4452         case CRYPTO_ALG_TYPE_ABLKCIPHER:
4453                 alg->cra_type = &crypto_ablkcipher_type;
4454                 alg->cra_ablkcipher = template->template_ablkcipher;
4455                 break;
4456         }
4457
4458         t_alg->caam.class1_alg_type = template->class1_alg_type;
4459         t_alg->caam.class2_alg_type = template->class2_alg_type;
4460         t_alg->caam.alg_op = template->alg_op;
4461
4462         return t_alg;
4463 }
4464
4465 static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
4466 {
4467         struct aead_alg *alg = &t_alg->aead;
4468
4469         alg->base.cra_module = THIS_MODULE;
4470         alg->base.cra_priority = CAAM_CRA_PRIORITY;
4471         alg->base.cra_ctxsize = sizeof(struct caam_ctx);
4472         alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
4473
4474         alg->init = caam_aead_init;
4475         alg->exit = caam_aead_exit;
4476 }
4477
4478 static int __init caam_algapi_init(void)
4479 {
4480         struct device_node *dev_node;
4481         struct platform_device *pdev;
4482         struct device *ctrldev;
4483         struct caam_drv_private *priv;
4484         int i = 0, err = 0;
4485         u32 cha_vid, cha_inst, des_inst, aes_inst, md_inst;
4486         unsigned int md_limit = SHA512_DIGEST_SIZE;
4487         bool registered = false;
4488
4489         dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
4490         if (!dev_node) {
4491                 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
4492                 if (!dev_node)
4493                         return -ENODEV;
4494         }
4495
4496         pdev = of_find_device_by_node(dev_node);
4497         if (!pdev) {
4498                 of_node_put(dev_node);
4499                 return -ENODEV;
4500         }
4501
4502         ctrldev = &pdev->dev;
4503         priv = dev_get_drvdata(ctrldev);
4504         of_node_put(dev_node);
4505
4506         /*
4507          * If priv is NULL, it's probably because the caam driver wasn't
4508          * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
4509          */
4510         if (!priv)
4511                 return -ENODEV;
4512
4513
4514         INIT_LIST_HEAD(&alg_list);
4515
4516         /*
4517          * Register crypto algorithms the device supports.
4518          * First, detect presence and attributes of DES, AES, and MD blocks.
4519          */
4520         cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
4521         cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
4522         des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> CHA_ID_LS_DES_SHIFT;
4523         aes_inst = (cha_inst & CHA_ID_LS_AES_MASK) >> CHA_ID_LS_AES_SHIFT;
4524         md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
4525
4526         /* If MD is present, limit digest size based on LP256 */
4527         if (md_inst && ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256))
4528                 md_limit = SHA256_DIGEST_SIZE;
4529
4530         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4531                 struct caam_crypto_alg *t_alg;
4532                 struct caam_alg_template *alg = driver_algs + i;
4533                 u32 alg_sel = alg->class1_alg_type & OP_ALG_ALGSEL_MASK;
4534
4535                 /* Skip DES algorithms if not supported by device */
4536                 if (!des_inst &&
4537                     ((alg_sel == OP_ALG_ALGSEL_3DES) ||
4538                      (alg_sel == OP_ALG_ALGSEL_DES)))
4539                                 continue;
4540
4541                 /* Skip AES algorithms if not supported by device */
4542                 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES))
4543                                 continue;
4544
4545                 t_alg = caam_alg_alloc(alg);
4546                 if (IS_ERR(t_alg)) {
4547                         err = PTR_ERR(t_alg);
4548                         pr_warn("%s alg allocation failed\n", alg->driver_name);
4549                         continue;
4550                 }
4551
4552                 err = crypto_register_alg(&t_alg->crypto_alg);
4553                 if (err) {
4554                         pr_warn("%s alg registration failed\n",
4555                                 t_alg->crypto_alg.cra_driver_name);
4556                         kfree(t_alg);
4557                         continue;
4558                 }
4559
4560                 list_add_tail(&t_alg->entry, &alg_list);
4561                 registered = true;
4562         }
4563
4564         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
4565                 struct caam_aead_alg *t_alg = driver_aeads + i;
4566                 u32 c1_alg_sel = t_alg->caam.class1_alg_type &
4567                                  OP_ALG_ALGSEL_MASK;
4568                 u32 c2_alg_sel = t_alg->caam.class2_alg_type &
4569                                  OP_ALG_ALGSEL_MASK;
4570                 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
4571
4572                 /* Skip DES algorithms if not supported by device */
4573                 if (!des_inst &&
4574                     ((c1_alg_sel == OP_ALG_ALGSEL_3DES) ||
4575                      (c1_alg_sel == OP_ALG_ALGSEL_DES)))
4576                                 continue;
4577
4578                 /* Skip AES algorithms if not supported by device */
4579                 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES))
4580                                 continue;
4581
4582                 /*
4583                  * Check support for AES algorithms not available
4584                  * on LP devices.
4585                  */
4586                 if ((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP)
4587                         if (alg_aai == OP_ALG_AAI_GCM)
4588                                 continue;
4589
4590                 /*
4591                  * Skip algorithms requiring message digests
4592                  * if MD or MD size is not supported by device.
4593                  */
4594                 if (c2_alg_sel &&
4595                     (!md_inst || (t_alg->aead.maxauthsize > md_limit)))
4596                                 continue;
4597
4598                 caam_aead_alg_init(t_alg);
4599
4600                 err = crypto_register_aead(&t_alg->aead);
4601                 if (err) {
4602                         pr_warn("%s alg registration failed\n",
4603                                 t_alg->aead.base.cra_driver_name);
4604                         continue;
4605                 }
4606
4607                 t_alg->registered = true;
4608                 registered = true;
4609         }
4610
4611         if (registered)
4612                 pr_info("caam algorithms registered in /proc/crypto\n");
4613
4614         return err;
4615 }
4616
4617 module_init(caam_algapi_init);
4618 module_exit(caam_algapi_exit);
4619
4620 MODULE_LICENSE("GPL");
4621 MODULE_DESCRIPTION("FSL CAAM support for crypto API");
4622 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");