drivers/crypto/caam/caamhash.c

   1 /*
   2  * caam - Freescale FSL CAAM support for ahash functions of crypto API
   3  *
   4  * Copyright 2011 Freescale Semiconductor, Inc.
   5  *
   6  * Based on caamalg.c crypto API driver.
   7  *
   8  * relationship of digest job descriptor or first job descriptor after init to
   9  * shared descriptors:
  10  *
  11  * ---------------                     ---------------
  12  * | JobDesc #1  |-------------------->|  ShareDesc  |
  13  * | *(packet 1) |                     |  (hashKey)  |
  14  * ---------------                     | (operation) |
  15  *                                     ---------------
  16  *
  17  * relationship of subsequent job descriptors to shared descriptors:
  18  *
  19  * ---------------                     ---------------
  20  * | JobDesc #2  |-------------------->|  ShareDesc  |
  21  * | *(packet 2) |      |------------->|  (hashKey)  |
  22  * ---------------      |    |-------->| (operation) |
  23  *       .              |    |         | (load ctx2) |
  24  *       .              |    |         ---------------
  25  * ---------------      |    |
  26  * | JobDesc #3  |------|    |
  27  * | *(packet 3) |           |
  28  * ---------------           |
  29  *       .                   |
  30  *       .                   |
  31  * ---------------           |
  32  * | JobDesc #4  |------------
  33  * | *(packet 4) |
  34  * ---------------
  35  *
  36  * The SharedDesc never changes for a connection unless rekeyed, but
  37  * each packet will likely be in a different place. So all we need
  38  * to know to process the packet is where the input is, where the
  39  * output goes, and what context we want to process with. Context is
  40  * in the SharedDesc, packet references in the JobDesc.
  41  *
  42  * So, a job desc looks like:
  43  *
  44  * ---------------------
  45  * | Header            |
  46  * | ShareDesc Pointer |
  47  * | SEQ_OUT_PTR       |
  48  * | (output buffer)   |
  49  * | (output length)   |
  50  * | SEQ_IN_PTR        |
  51  * | (input buffer)    |
  52  * | (input length)    |
  53  * ---------------------
  54  */
  55
  56 #include "compat.h"
  57
  58 #include "regs.h"
  59 #include "intern.h"
  60 #include "desc_constr.h"
  61 #include "jr.h"
  62 #include "error.h"
  63 #include "sg_sw_sec4.h"
  64 #include "key_gen.h"
  65
  66 #define CAAM_CRA_PRIORITY               3000
  67
  68 /* max hash key is max split key size */
  69 #define CAAM_MAX_HASH_KEY_SIZE          (SHA512_DIGEST_SIZE * 2)
  70
  71 #define CAAM_MAX_HASH_BLOCK_SIZE        SHA512_BLOCK_SIZE
  72 #define CAAM_MAX_HASH_DIGEST_SIZE       SHA512_DIGEST_SIZE
  73
  74 /* length of descriptors text */
  75 #define DESC_AHASH_BASE                 (4 * CAAM_CMD_SZ)
  76 #define DESC_AHASH_UPDATE_LEN           (6 * CAAM_CMD_SZ)
  77 #define DESC_AHASH_UPDATE_FIRST_LEN     (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ)
  78 #define DESC_AHASH_FINAL_LEN            (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ)
  79 #define DESC_AHASH_FINUP_LEN            (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ)
  80 #define DESC_AHASH_DIGEST_LEN           (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ)
  81
  82 #define DESC_HASH_MAX_USED_BYTES        (DESC_AHASH_FINAL_LEN + \
  83                                          CAAM_MAX_HASH_KEY_SIZE)
  84 #define DESC_HASH_MAX_USED_LEN          (DESC_HASH_MAX_USED_BYTES / CAAM_CMD_SZ)
  85
  86 /* caam context sizes for hashes: running digest + 8 */
  87 #define HASH_MSG_LEN                    8
  88 #define MAX_CTX_LEN                     (HASH_MSG_LEN + SHA512_DIGEST_SIZE)
  89
  90 #ifdef DEBUG
  91 /* for print_hex_dumps with line references */
  92 #define debug(format, arg...) printk(format, arg)
  93 #else
  94 #define debug(format, arg...)
  95 #endif
  96
  97
  98 static struct list_head hash_list;
  99
 100 /* ahash per-session context */
 101 struct caam_hash_ctx {
 102         u32 sh_desc_update[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
 103         u32 sh_desc_update_first[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
 104         u32 sh_desc_fin[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
 105         u32 sh_desc_digest[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
 106         u32 sh_desc_finup[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
 107         dma_addr_t sh_desc_update_dma ____cacheline_aligned;
 108         dma_addr_t sh_desc_update_first_dma;
 109         dma_addr_t sh_desc_fin_dma;
 110         dma_addr_t sh_desc_digest_dma;
 111         dma_addr_t sh_desc_finup_dma;
 112         struct device *jrdev;
 113         u32 alg_type;
 114         u32 alg_op;
 115         u8 key[CAAM_MAX_HASH_KEY_SIZE];
 116         dma_addr_t key_dma;
 117         int ctx_len;
 118         unsigned int split_key_len;
 119         unsigned int split_key_pad_len;
 120 };
 121
 122 /* ahash state */
 123 struct caam_hash_state {
 124         dma_addr_t buf_dma;
 125         dma_addr_t ctx_dma;
 126         u8 buf_0[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
 127         int buflen_0;
 128         u8 buf_1[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
 129         int buflen_1;
 130         u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned;
 131         int (*update)(struct ahash_request *req);
 132         int (*final)(struct ahash_request *req);
 133         int (*finup)(struct ahash_request *req);
 134         int current_buf;
 135 };
 136
 137 struct caam_export_state {
 138         u8 buf[CAAM_MAX_HASH_BLOCK_SIZE];
 139         u8 caam_ctx[MAX_CTX_LEN];
 140         int buflen;
 141         int (*update)(struct ahash_request *req);
 142         int (*final)(struct ahash_request *req);
 143         int (*finup)(struct ahash_request *req);
 144 };
 145
 146 /* Common job descriptor seq in/out ptr routines */
 147
 148 /* Map state->caam_ctx, and append seq_out_ptr command that points to it */
 149 static inline int map_seq_out_ptr_ctx(u32 *desc, struct device *jrdev,
 150                                       struct caam_hash_state *state,
 151                                       int ctx_len)
 152 {
 153         state->ctx_dma = dma_map_single(jrdev, state->caam_ctx,
 154                                         ctx_len, DMA_FROM_DEVICE);
 155         if (dma_mapping_error(jrdev, state->ctx_dma)) {
 156                 dev_err(jrdev, "unable to map ctx\n");
 157                 return -ENOMEM;
 158         }
 159
 160         append_seq_out_ptr(desc, state->ctx_dma, ctx_len, 0);
 161
 162         return 0;
 163 }
 164
 165 /* Map req->result, and append seq_out_ptr command that points to it */
 166 static inline dma_addr_t map_seq_out_ptr_result(u32 *desc, struct device *jrdev,
 167                                                 u8 *result, int digestsize)
 168 {
 169         dma_addr_t dst_dma;
 170
 171         dst_dma = dma_map_single(jrdev, result, digestsize, DMA_FROM_DEVICE);
 172         append_seq_out_ptr(desc, dst_dma, digestsize, 0);
 173
 174         return dst_dma;
 175 }
 176
 177 /* Map current buffer in state and put it in link table */
 178 static inline dma_addr_t buf_map_to_sec4_sg(struct device *jrdev,
 179                                             struct sec4_sg_entry *sec4_sg,
 180                                             u8 *buf, int buflen)
 181 {
 182         dma_addr_t buf_dma;
 183
 184         buf_dma = dma_map_single(jrdev, buf, buflen, DMA_TO_DEVICE);
 185         dma_to_sec4_sg_one(sec4_sg, buf_dma, buflen, 0);
 186
 187         return buf_dma;
 188 }
 189
 190 /*
 191  * Only put buffer in link table if it contains data, which is possible,
 192  * since a buffer has previously been used, and needs to be unmapped,
 193  */
 194 static inline dma_addr_t
 195 try_buf_map_to_sec4_sg(struct device *jrdev, struct sec4_sg_entry *sec4_sg,
 196                        u8 *buf, dma_addr_t buf_dma, int buflen,
 197                        int last_buflen)
 198 {
 199         if (buf_dma && !dma_mapping_error(jrdev, buf_dma))
 200                 dma_unmap_single(jrdev, buf_dma, last_buflen, DMA_TO_DEVICE);
 201         if (buflen)
 202                 buf_dma = buf_map_to_sec4_sg(jrdev, sec4_sg, buf, buflen);
 203         else
 204                 buf_dma = 0;
 205
 206         return buf_dma;
 207 }
 208
 209 /* Map state->caam_ctx, and add it to link table */
 210 static inline int ctx_map_to_sec4_sg(u32 *desc, struct device *jrdev,
 211                                      struct caam_hash_state *state, int ctx_len,
 212                                      struct sec4_sg_entry *sec4_sg, u32 flag)
 213 {
 214         state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, ctx_len, flag);
 215         if (dma_mapping_error(jrdev, state->ctx_dma)) {
 216                 dev_err(jrdev, "unable to map ctx\n");
 217                 return -ENOMEM;
 218         }
 219
 220         dma_to_sec4_sg_one(sec4_sg, state->ctx_dma, ctx_len, 0);
 221
 222         return 0;
 223 }
 224
 225 /* Common shared descriptor commands */
 226 static inline void append_key_ahash(u32 *desc, struct caam_hash_ctx *ctx)
 227 {
 228         append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
 229                           ctx->split_key_len, CLASS_2 |
 230                           KEY_DEST_MDHA_SPLIT | KEY_ENC);
 231 }
 232
 233 /* Append key if it has been set */
 234 static inline void init_sh_desc_key_ahash(u32 *desc, struct caam_hash_ctx *ctx)
 235 {
 236         u32 *key_jump_cmd;
 237
 238         init_sh_desc(desc, HDR_SHARE_SERIAL);
 239
 240         if (ctx->split_key_len) {
 241                 /* Skip if already shared */
 242                 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 243                                            JUMP_COND_SHRD);
 244
 245                 append_key_ahash(desc, ctx);
 246
 247                 set_jump_tgt_here(desc, key_jump_cmd);
 248         }
 249
 250         /* Propagate errors from shared to job descriptor */
 251         append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD);
 252 }
 253
 254 /*
 255  * For ahash read data from seqin following state->caam_ctx,
 256  * and write resulting class2 context to seqout, which may be state->caam_ctx
 257  * or req->result
 258  */
 259 static inline void ahash_append_load_str(u32 *desc, int digestsize)
 260 {
 261         /* Calculate remaining bytes to read */
 262         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 263
 264         /* Read remaining bytes */
 265         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 |
 266                              FIFOLD_TYPE_MSG | KEY_VLF);
 267
 268         /* Store class2 context bytes */
 269         append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
 270                          LDST_SRCDST_BYTE_CONTEXT);
 271 }
 272
 273 /*
 274  * For ahash update, final and finup, import context, read and write to seqout
 275  */
 276 static inline void ahash_ctx_data_to_out(u32 *desc, u32 op, u32 state,
 277                                          int digestsize,
 278                                          struct caam_hash_ctx *ctx)
 279 {
 280         init_sh_desc_key_ahash(desc, ctx);
 281
 282         /* Import context from software */
 283         append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
 284                    LDST_CLASS_2_CCB | ctx->ctx_len);
 285
 286         /* Class 2 operation */
 287         append_operation(desc, op | state | OP_ALG_ENCRYPT);
 288
 289         /*
 290          * Load from buf and/or src and write to req->result or state->context
 291          */
 292         ahash_append_load_str(desc, digestsize);
 293 }
 294
 295 /* For ahash firsts and digest, read and write to seqout */
 296 static inline void ahash_data_to_out(u32 *desc, u32 op, u32 state,
 297                                      int digestsize, struct caam_hash_ctx *ctx)
 298 {
 299         init_sh_desc_key_ahash(desc, ctx);
 300
 301         /* Class 2 operation */
 302         append_operation(desc, op | state | OP_ALG_ENCRYPT);
 303
 304         /*
 305          * Load from buf and/or src and write to req->result or state->context
 306          */
 307         ahash_append_load_str(desc, digestsize);
 308 }
 309
 310 static int ahash_set_sh_desc(struct crypto_ahash *ahash)
 311 {
 312         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 313         int digestsize = crypto_ahash_digestsize(ahash);
 314         struct device *jrdev = ctx->jrdev;
 315         u32 have_key = 0;
 316         u32 *desc;
 317
 318         if (ctx->split_key_len)
 319                 have_key = OP_ALG_AAI_HMAC_PRECOMP;
 320
 321         /* ahash_update shared descriptor */
 322         desc = ctx->sh_desc_update;
 323
 324         init_sh_desc(desc, HDR_SHARE_SERIAL);
 325
 326         /* Import context from software */
 327         append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
 328                    LDST_CLASS_2_CCB | ctx->ctx_len);
 329
 330         /* Class 2 operation */
 331         append_operation(desc, ctx->alg_type | OP_ALG_AS_UPDATE |
 332                          OP_ALG_ENCRYPT);
 333
 334         /* Load data and write to result or context */
 335         ahash_append_load_str(desc, ctx->ctx_len);
 336
 337         ctx->sh_desc_update_dma = dma_map_single(jrdev, desc, desc_bytes(desc),
 338                                                  DMA_TO_DEVICE);
 339         if (dma_mapping_error(jrdev, ctx->sh_desc_update_dma)) {
 340                 dev_err(jrdev, "unable to map shared descriptor\n");
 341                 return -ENOMEM;
 342         }
 343 #ifdef DEBUG
 344         print_hex_dump(KERN_ERR,
 345                        "ahash update shdesc@"__stringify(__LINE__)": ",
 346                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 347 #endif
 348
 349         /* ahash_update_first shared descriptor */
 350         desc = ctx->sh_desc_update_first;
 351
 352         ahash_data_to_out(desc, have_key | ctx->alg_type, OP_ALG_AS_INIT,
 353                           ctx->ctx_len, ctx);
 354
 355         ctx->sh_desc_update_first_dma = dma_map_single(jrdev, desc,
 356                                                        desc_bytes(desc),
 357                                                        DMA_TO_DEVICE);
 358         if (dma_mapping_error(jrdev, ctx->sh_desc_update_first_dma)) {
 359                 dev_err(jrdev, "unable to map shared descriptor\n");
 360                 return -ENOMEM;
 361         }
 362 #ifdef DEBUG
 363         print_hex_dump(KERN_ERR,
 364                        "ahash update first shdesc@"__stringify(__LINE__)": ",
 365                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 366 #endif
 367
 368         /* ahash_final shared descriptor */
 369         desc = ctx->sh_desc_fin;
 370
 371         ahash_ctx_data_to_out(desc, have_key | ctx->alg_type,
 372                               OP_ALG_AS_FINALIZE, digestsize, ctx);
 373
 374         ctx->sh_desc_fin_dma = dma_map_single(jrdev, desc, desc_bytes(desc),
 375                                               DMA_TO_DEVICE);
 376         if (dma_mapping_error(jrdev, ctx->sh_desc_fin_dma)) {
 377                 dev_err(jrdev, "unable to map shared descriptor\n");
 378                 return -ENOMEM;
 379         }
 380 #ifdef DEBUG
 381         print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ",
 382                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 383                        desc_bytes(desc), 1);
 384 #endif
 385
 386         /* ahash_finup shared descriptor */
 387         desc = ctx->sh_desc_finup;
 388
 389         ahash_ctx_data_to_out(desc, have_key | ctx->alg_type,
 390                               OP_ALG_AS_FINALIZE, digestsize, ctx);
 391
 392         ctx->sh_desc_finup_dma = dma_map_single(jrdev, desc, desc_bytes(desc),
 393                                                 DMA_TO_DEVICE);
 394         if (dma_mapping_error(jrdev, ctx->sh_desc_finup_dma)) {
 395                 dev_err(jrdev, "unable to map shared descriptor\n");
 396                 return -ENOMEM;
 397         }
 398 #ifdef DEBUG
 399         print_hex_dump(KERN_ERR, "ahash finup shdesc@"__stringify(__LINE__)": ",
 400                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 401                        desc_bytes(desc), 1);
 402 #endif
 403
 404         /* ahash_digest shared descriptor */
 405         desc = ctx->sh_desc_digest;
 406
 407         ahash_data_to_out(desc, have_key | ctx->alg_type, OP_ALG_AS_INITFINAL,
 408                           digestsize, ctx);
 409
 410         ctx->sh_desc_digest_dma = dma_map_single(jrdev, desc,
 411                                                  desc_bytes(desc),
 412                                                  DMA_TO_DEVICE);
 413         if (dma_mapping_error(jrdev, ctx->sh_desc_digest_dma)) {
 414                 dev_err(jrdev, "unable to map shared descriptor\n");
 415                 return -ENOMEM;
 416         }
 417 #ifdef DEBUG
 418         print_hex_dump(KERN_ERR,
 419                        "ahash digest shdesc@"__stringify(__LINE__)": ",
 420                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 421                        desc_bytes(desc), 1);
 422 #endif
 423
 424         return 0;
 425 }
 426
 427 static int gen_split_hash_key(struct caam_hash_ctx *ctx, const u8 *key_in,
 428                               u32 keylen)
 429 {
 430         return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len,
 431                                ctx->split_key_pad_len, key_in, keylen,
 432                                ctx->alg_op);
 433 }
 434
 435 /* Digest hash size if it is too large */
 436 static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in,
 437                            u32 *keylen, u8 *key_out, u32 digestsize)
 438 {
 439         struct device *jrdev = ctx->jrdev;
 440         u32 *desc;
 441         struct split_key_result result;
 442         dma_addr_t src_dma, dst_dma;
 443         int ret;
 444
 445         desc = kmalloc(CAAM_CMD_SZ * 8 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
 446         if (!desc) {
 447                 dev_err(jrdev, "unable to allocate key input memory\n");
 448                 return -ENOMEM;
 449         }
 450
 451         init_job_desc(desc, 0);
 452
 453         src_dma = dma_map_single(jrdev, (void *)key_in, *keylen,
 454                                  DMA_TO_DEVICE);
 455         if (dma_mapping_error(jrdev, src_dma)) {
 456                 dev_err(jrdev, "unable to map key input memory\n");
 457                 kfree(desc);
 458                 return -ENOMEM;
 459         }
 460         dst_dma = dma_map_single(jrdev, (void *)key_out, digestsize,
 461                                  DMA_FROM_DEVICE);
 462         if (dma_mapping_error(jrdev, dst_dma)) {
 463                 dev_err(jrdev, "unable to map key output memory\n");
 464                 dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE);
 465                 kfree(desc);
 466                 return -ENOMEM;
 467         }
 468
 469         /* Job descriptor to perform unkeyed hash on key_in */
 470         append_operation(desc, ctx->alg_type | OP_ALG_ENCRYPT |
 471                          OP_ALG_AS_INITFINAL);
 472         append_seq_in_ptr(desc, src_dma, *keylen, 0);
 473         append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 |
 474                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG);
 475         append_seq_out_ptr(desc, dst_dma, digestsize, 0);
 476         append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
 477                          LDST_SRCDST_BYTE_CONTEXT);
 478
 479 #ifdef DEBUG
 480         print_hex_dump(KERN_ERR, "key_in@"__stringify(__LINE__)": ",
 481                        DUMP_PREFIX_ADDRESS, 16, 4, key_in, *keylen, 1);
 482         print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
 483                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 484 #endif
 485
 486         result.err = 0;
 487         init_completion(&result.completion);
 488
 489         ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result);
 490         if (!ret) {
 491                 /* in progress */
 492                 wait_for_completion_interruptible(&result.completion);
 493                 ret = result.err;
 494 #ifdef DEBUG
 495                 print_hex_dump(KERN_ERR,
 496                                "digested key@"__stringify(__LINE__)": ",
 497                                DUMP_PREFIX_ADDRESS, 16, 4, key_in,
 498                                digestsize, 1);
 499 #endif
 500         }
 501         dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE);
 502         dma_unmap_single(jrdev, dst_dma, digestsize, DMA_FROM_DEVICE);
 503
 504         *keylen = digestsize;
 505
 506         kfree(desc);
 507
 508         return ret;
 509 }
 510
 511 static int ahash_setkey(struct crypto_ahash *ahash,
 512                         const u8 *key, unsigned int keylen)
 513 {
 514         /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
 515         static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
 516         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 517         struct device *jrdev = ctx->jrdev;
 518         int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
 519         int digestsize = crypto_ahash_digestsize(ahash);
 520         int ret;
 521         u8 *hashed_key = NULL;
 522
 523 #ifdef DEBUG
 524         printk(KERN_ERR "keylen %d\n", keylen);
 525 #endif
 526
 527         if (keylen > blocksize) {
 528                 hashed_key = kmalloc_array(digestsize,
 529                                            sizeof(*hashed_key),
 530                                            GFP_KERNEL | GFP_DMA);
 531                 if (!hashed_key)
 532                         return -ENOMEM;
 533                 ret = hash_digest_key(ctx, key, &keylen, hashed_key,
 534                                       digestsize);
 535                 if (ret)
 536                         goto bad_free_key;
 537                 key = hashed_key;
 538         }
 539
 540         /* Pick class 2 key length from algorithm submask */
 541         ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >>
 542                                       OP_ALG_ALGSEL_SHIFT] * 2;
 543         ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16);
 544
 545 #ifdef DEBUG
 546         printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n",
 547                ctx->split_key_len, ctx->split_key_pad_len);
 548         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
 549                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 550 #endif
 551
 552         ret = gen_split_hash_key(ctx, key, keylen);
 553         if (ret)
 554                 goto bad_free_key;
 555
 556         ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len,
 557                                       DMA_TO_DEVICE);
 558         if (dma_mapping_error(jrdev, ctx->key_dma)) {
 559                 dev_err(jrdev, "unable to map key i/o memory\n");
 560                 ret = -ENOMEM;
 561                 goto error_free_key;
 562         }
 563 #ifdef DEBUG
 564         print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
 565                        DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 566                        ctx->split_key_pad_len, 1);
 567 #endif
 568
 569         ret = ahash_set_sh_desc(ahash);
 570         if (ret) {
 571                 dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len,
 572                                  DMA_TO_DEVICE);
 573         }
 574  error_free_key:
 575         kfree(hashed_key);
 576         return ret;
 577  bad_free_key:
 578         kfree(hashed_key);
 579         crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN);
 580         return -EINVAL;
 581 }
 582
 583 /*
 584  * ahash_edesc - s/w-extended ahash descriptor
 585  * @dst_dma: physical mapped address of req->result
 586  * @sec4_sg_dma: physical mapped address of h/w link table
 587  * @src_nents: number of segments in input scatterlist
 588  * @sec4_sg_bytes: length of dma mapped sec4_sg space
 589  * @hw_desc: the h/w job descriptor followed by any referenced link tables
 590  * @sec4_sg: h/w link table
 591  */
 592 struct ahash_edesc {
 593         dma_addr_t dst_dma;
 594         dma_addr_t sec4_sg_dma;
 595         int src_nents;
 596         int sec4_sg_bytes;
 597         u32 hw_desc[DESC_JOB_IO_LEN / sizeof(u32)] ____cacheline_aligned;
 598         struct sec4_sg_entry sec4_sg[0];
 599 };
 600
 601 static inline void ahash_unmap(struct device *dev,
 602                         struct ahash_edesc *edesc,
 603                         struct ahash_request *req, int dst_len)
 604 {
 605         if (edesc->src_nents)
 606                 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
 607         if (edesc->dst_dma)
 608                 dma_unmap_single(dev, edesc->dst_dma, dst_len, DMA_FROM_DEVICE);
 609
 610         if (edesc->sec4_sg_bytes)
 611                 dma_unmap_single(dev, edesc->sec4_sg_dma,
 612                                  edesc->sec4_sg_bytes, DMA_TO_DEVICE);
 613 }
 614
 615 static inline void ahash_unmap_ctx(struct device *dev,
 616                         struct ahash_edesc *edesc,
 617                         struct ahash_request *req, int dst_len, u32 flag)
 618 {
 619         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 620         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 621         struct caam_hash_state *state = ahash_request_ctx(req);
 622
 623         if (state->ctx_dma)
 624                 dma_unmap_single(dev, state->ctx_dma, ctx->ctx_len, flag);
 625         ahash_unmap(dev, edesc, req, dst_len);
 626 }
 627
 628 static void ahash_done(struct device *jrdev, u32 *desc, u32 err,
 629                        void *context)
 630 {
 631         struct ahash_request *req = context;
 632         struct ahash_edesc *edesc;
 633         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 634         int digestsize = crypto_ahash_digestsize(ahash);
 635 #ifdef DEBUG
 636         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 637         struct caam_hash_state *state = ahash_request_ctx(req);
 638
 639         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 640 #endif
 641
 642         edesc = (struct ahash_edesc *)((char *)desc -
 643                  offsetof(struct ahash_edesc, hw_desc));
 644         if (err)
 645                 caam_jr_strstatus(jrdev, err);
 646
 647         ahash_unmap(jrdev, edesc, req, digestsize);
 648         kfree(edesc);
 649
 650 #ifdef DEBUG
 651         print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
 652                        DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
 653                        ctx->ctx_len, 1);
 654         if (req->result)
 655                 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
 656                                DUMP_PREFIX_ADDRESS, 16, 4, req->result,
 657                                digestsize, 1);
 658 #endif
 659
 660         req->base.complete(&req->base, err);
 661 }
 662
 663 static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err,
 664                             void *context)
 665 {
 666         struct ahash_request *req = context;
 667         struct ahash_edesc *edesc;
 668         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 669         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 670 #ifdef DEBUG
 671         struct caam_hash_state *state = ahash_request_ctx(req);
 672         int digestsize = crypto_ahash_digestsize(ahash);
 673
 674         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 675 #endif
 676
 677         edesc = (struct ahash_edesc *)((char *)desc -
 678                  offsetof(struct ahash_edesc, hw_desc));
 679         if (err)
 680                 caam_jr_strstatus(jrdev, err);
 681
 682         ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
 683         kfree(edesc);
 684
 685 #ifdef DEBUG
 686         print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
 687                        DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
 688                        ctx->ctx_len, 1);
 689         if (req->result)
 690                 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
 691                                DUMP_PREFIX_ADDRESS, 16, 4, req->result,
 692                                digestsize, 1);
 693 #endif
 694
 695         req->base.complete(&req->base, err);
 696 }
 697
 698 static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err,
 699                                void *context)
 700 {
 701         struct ahash_request *req = context;
 702         struct ahash_edesc *edesc;
 703         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 704         int digestsize = crypto_ahash_digestsize(ahash);
 705 #ifdef DEBUG
 706         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 707         struct caam_hash_state *state = ahash_request_ctx(req);
 708
 709         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 710 #endif
 711
 712         edesc = (struct ahash_edesc *)((char *)desc -
 713                  offsetof(struct ahash_edesc, hw_desc));
 714         if (err)
 715                 caam_jr_strstatus(jrdev, err);
 716
 717         ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_TO_DEVICE);
 718         kfree(edesc);
 719
 720 #ifdef DEBUG
 721         print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
 722                        DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
 723                        ctx->ctx_len, 1);
 724         if (req->result)
 725                 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
 726                                DUMP_PREFIX_ADDRESS, 16, 4, req->result,
 727                                digestsize, 1);
 728 #endif
 729
 730         req->base.complete(&req->base, err);
 731 }
 732
 733 static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,
 734                                void *context)
 735 {
 736         struct ahash_request *req = context;
 737         struct ahash_edesc *edesc;
 738         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 739         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 740 #ifdef DEBUG
 741         struct caam_hash_state *state = ahash_request_ctx(req);
 742         int digestsize = crypto_ahash_digestsize(ahash);
 743
 744         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 745 #endif
 746
 747         edesc = (struct ahash_edesc *)((char *)desc -
 748                  offsetof(struct ahash_edesc, hw_desc));
 749         if (err)
 750                 caam_jr_strstatus(jrdev, err);
 751
 752         ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE);
 753         kfree(edesc);
 754
 755 #ifdef DEBUG
 756         print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
 757                        DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
 758                        ctx->ctx_len, 1);
 759         if (req->result)
 760                 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
 761                                DUMP_PREFIX_ADDRESS, 16, 4, req->result,
 762                                digestsize, 1);
 763 #endif
 764
 765         req->base.complete(&req->base, err);
 766 }
 767
 768 /*
 769  * Allocate an enhanced descriptor, which contains the hardware descriptor
 770  * and space for hardware scatter table containing sg_num entries.
 771  */
 772 static struct ahash_edesc *ahash_edesc_alloc(struct caam_hash_ctx *ctx,
 773                                              int sg_num, u32 *sh_desc,
 774                                              dma_addr_t sh_desc_dma,
 775                                              gfp_t flags)
 776 {
 777         struct ahash_edesc *edesc;
 778         unsigned int sg_size = sg_num * sizeof(struct sec4_sg_entry);
 779
 780         edesc = kzalloc(sizeof(*edesc) + sg_size, GFP_DMA | flags);
 781         if (!edesc) {
 782                 dev_err(ctx->jrdev, "could not allocate extended descriptor\n");
 783                 return NULL;
 784         }
 785
 786         init_job_desc_shared(edesc->hw_desc, sh_desc_dma, desc_len(sh_desc),
 787                              HDR_SHARE_DEFER | HDR_REVERSE);
 788
 789         return edesc;
 790 }
 791
 792 static int ahash_edesc_add_src(struct caam_hash_ctx *ctx,
 793                                struct ahash_edesc *edesc,
 794                                struct ahash_request *req, int nents,
 795                                unsigned int first_sg,
 796                                unsigned int first_bytes, size_t to_hash)
 797 {
 798         dma_addr_t src_dma;
 799         u32 options;
 800
 801         if (nents > 1 || first_sg) {
 802                 struct sec4_sg_entry *sg = edesc->sec4_sg;
 803                 unsigned int sgsize = sizeof(*sg) * (first_sg + nents);
 804
 805                 sg_to_sec4_sg_last(req->src, nents, sg + first_sg, 0);
 806
 807                 src_dma = dma_map_single(ctx->jrdev, sg, sgsize, DMA_TO_DEVICE);
 808                 if (dma_mapping_error(ctx->jrdev, src_dma)) {
 809                         dev_err(ctx->jrdev, "unable to map S/G table\n");
 810                         return -ENOMEM;
 811                 }
 812
 813                 edesc->sec4_sg_bytes = sgsize;
 814                 edesc->sec4_sg_dma = src_dma;
 815                 options = LDST_SGF;
 816         } else {
 817                 src_dma = sg_dma_address(req->src);
 818                 options = 0;
 819         }
 820
 821         append_seq_in_ptr(edesc->hw_desc, src_dma, first_bytes + to_hash,
 822                           options);
 823
 824         return 0;
 825 }
 826
 827 /* submit update job descriptor */
 828 static int ahash_update_ctx(struct ahash_request *req)
 829 {
 830         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 831         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 832         struct caam_hash_state *state = ahash_request_ctx(req);
 833         struct device *jrdev = ctx->jrdev;
 834         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
 835                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
 836         u8 *buf = state->current_buf ? state->buf_1 : state->buf_0;
 837         int *buflen = state->current_buf ? &state->buflen_1 : &state->buflen_0;
 838         u8 *next_buf = state->current_buf ? state->buf_0 : state->buf_1;
 839         int *next_buflen = state->current_buf ? &state->buflen_0 :
 840                            &state->buflen_1, last_buflen;
 841         int in_len = *buflen + req->nbytes, to_hash;
 842         u32 *desc;
 843         int src_nents, mapped_nents, sec4_sg_bytes, sec4_sg_src_index;
 844         struct ahash_edesc *edesc;
 845         int ret = 0;
 846
 847         last_buflen = *next_buflen;
 848         *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
 849         to_hash = in_len - *next_buflen;
 850
 851         if (to_hash) {
 852                 src_nents = sg_nents_for_len(req->src,
 853                                              req->nbytes - (*next_buflen));
 854                 if (src_nents < 0) {
 855                         dev_err(jrdev, "Invalid number of src SG.\n");
 856                         return src_nents;
 857                 }
 858
 859                 if (src_nents) {
 860                         mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
 861                                                   DMA_TO_DEVICE);
 862                         if (!mapped_nents) {
 863                                 dev_err(jrdev, "unable to DMA map source\n");
 864                                 return -ENOMEM;
 865                         }
 866                 } else {
 867                         mapped_nents = 0;
 868                 }
 869
 870                 sec4_sg_src_index = 1 + (*buflen ? 1 : 0);
 871                 sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) *
 872                                  sizeof(struct sec4_sg_entry);
 873
 874                 /*
 875                  * allocate space for base edesc and hw desc commands,
 876                  * link tables
 877                  */
 878                 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
 879                                           ctx->sh_desc_update,
 880                                           ctx->sh_desc_update_dma, flags);
 881                 if (!edesc) {
 882                         dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
 883                         return -ENOMEM;
 884                 }
 885
 886                 edesc->src_nents = src_nents;
 887                 edesc->sec4_sg_bytes = sec4_sg_bytes;
 888
 889                 ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len,
 890                                          edesc->sec4_sg, DMA_BIDIRECTIONAL);
 891                 if (ret)
 892                         goto unmap_ctx;
 893
 894                 state->buf_dma = try_buf_map_to_sec4_sg(jrdev,
 895                                                         edesc->sec4_sg + 1,
 896                                                         buf, state->buf_dma,
 897                                                         *buflen, last_buflen);
 898
 899                 if (mapped_nents) {
 900                         sg_to_sec4_sg_last(req->src, mapped_nents,
 901                                            edesc->sec4_sg + sec4_sg_src_index,
 902                                            0);
 903                         if (*next_buflen)
 904                                 scatterwalk_map_and_copy(next_buf, req->src,
 905                                                          to_hash - *buflen,
 906                                                          *next_buflen, 0);
 907                 } else {
 908                         (edesc->sec4_sg + sec4_sg_src_index - 1)->len |=
 909                                 cpu_to_caam32(SEC4_SG_LEN_FIN);
 910                 }
 911
 912                 state->current_buf = !state->current_buf;
 913
 914                 desc = edesc->hw_desc;
 915
 916                 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
 917                                                      sec4_sg_bytes,
 918                                                      DMA_TO_DEVICE);
 919                 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
 920                         dev_err(jrdev, "unable to map S/G table\n");
 921                         ret = -ENOMEM;
 922                         goto unmap_ctx;
 923                 }
 924
 925                 append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len +
 926                                        to_hash, LDST_SGF);
 927
 928                 append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0);
 929
 930 #ifdef DEBUG
 931                 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
 932                                DUMP_PREFIX_ADDRESS, 16, 4, desc,
 933                                desc_bytes(desc), 1);
 934 #endif
 935
 936                 ret = caam_jr_enqueue(jrdev, desc, ahash_done_bi, req);
 937                 if (ret)
 938                         goto unmap_ctx;
 939
 940                 ret = -EINPROGRESS;
 941         } else if (*next_buflen) {
 942                 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
 943                                          req->nbytes, 0);
 944                 *buflen = *next_buflen;
 945                 *next_buflen = last_buflen;
 946         }
 947 #ifdef DEBUG
 948         print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ",
 949                        DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
 950         print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
 951                        DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
 952                        *next_buflen, 1);
 953 #endif
 954
 955         return ret;
 956  unmap_ctx:
 957         ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
 958         kfree(edesc);
 959         return ret;
 960 }
 961
 962 static int ahash_final_ctx(struct ahash_request *req)
 963 {
 964         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 965         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 966         struct caam_hash_state *state = ahash_request_ctx(req);
 967         struct device *jrdev = ctx->jrdev;
 968         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
 969                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
 970         u8 *buf = state->current_buf ? state->buf_1 : state->buf_0;
 971         int buflen = state->current_buf ? state->buflen_1 : state->buflen_0;
 972         int last_buflen = state->current_buf ? state->buflen_0 :
 973                           state->buflen_1;
 974         u32 *desc;
 975         int sec4_sg_bytes, sec4_sg_src_index;
 976         int digestsize = crypto_ahash_digestsize(ahash);
 977         struct ahash_edesc *edesc;
 978         int ret;
 979
 980         sec4_sg_src_index = 1 + (buflen ? 1 : 0);
 981         sec4_sg_bytes = sec4_sg_src_index * sizeof(struct sec4_sg_entry);
 982
 983         /* allocate space for base edesc and hw desc commands, link tables */
 984         edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index,
 985                                   ctx->sh_desc_fin, ctx->sh_desc_fin_dma,
 986                                   flags);
 987         if (!edesc)
 988                 return -ENOMEM;
 989
 990         desc = edesc->hw_desc;
 991
 992         edesc->sec4_sg_bytes = sec4_sg_bytes;
 993         edesc->src_nents = 0;
 994
 995         ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len,
 996                                  edesc->sec4_sg, DMA_TO_DEVICE);
 997         if (ret)
 998                 goto unmap_ctx;
 999
1000         state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1,
1001                                                 buf, state->buf_dma, buflen,
1002                                                 last_buflen);
1003         (edesc->sec4_sg + sec4_sg_src_index - 1)->len |=
1004                 cpu_to_caam32(SEC4_SG_LEN_FIN);
1005
1006         edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1007                                             sec4_sg_bytes, DMA_TO_DEVICE);
1008         if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1009                 dev_err(jrdev, "unable to map S/G table\n");
1010                 ret = -ENOMEM;
1011                 goto unmap_ctx;
1012         }
1013
1014         append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + buflen,
1015                           LDST_SGF);
1016
1017         edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
1018                                                 digestsize);
1019         if (dma_mapping_error(jrdev, edesc->dst_dma)) {
1020                 dev_err(jrdev, "unable to map dst\n");
1021                 ret = -ENOMEM;
1022                 goto unmap_ctx;
1023         }
1024
1025 #ifdef DEBUG
1026         print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1027                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1028 #endif
1029
1030         ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req);
1031         if (ret)
1032                 goto unmap_ctx;
1033
1034         return -EINPROGRESS;
1035  unmap_ctx:
1036         ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
1037         kfree(edesc);
1038         return ret;
1039 }
1040
1041 static int ahash_finup_ctx(struct ahash_request *req)
1042 {
1043         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1044         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1045         struct caam_hash_state *state = ahash_request_ctx(req);
1046         struct device *jrdev = ctx->jrdev;
1047         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1048                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1049         u8 *buf = state->current_buf ? state->buf_1 : state->buf_0;
1050         int buflen = state->current_buf ? state->buflen_1 : state->buflen_0;
1051         int last_buflen = state->current_buf ? state->buflen_0 :
1052                           state->buflen_1;
1053         u32 *desc;
1054         int sec4_sg_src_index;
1055         int src_nents, mapped_nents;
1056         int digestsize = crypto_ahash_digestsize(ahash);
1057         struct ahash_edesc *edesc;
1058         int ret;
1059
1060         src_nents = sg_nents_for_len(req->src, req->nbytes);
1061         if (src_nents < 0) {
1062                 dev_err(jrdev, "Invalid number of src SG.\n");
1063                 return src_nents;
1064         }
1065
1066         if (src_nents) {
1067                 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1068                                           DMA_TO_DEVICE);
1069                 if (!mapped_nents) {
1070                         dev_err(jrdev, "unable to DMA map source\n");
1071                         return -ENOMEM;
1072                 }
1073         } else {
1074                 mapped_nents = 0;
1075         }
1076
1077         sec4_sg_src_index = 1 + (buflen ? 1 : 0);
1078
1079         /* allocate space for base edesc and hw desc commands, link tables */
1080         edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
1081                                   ctx->sh_desc_finup, ctx->sh_desc_finup_dma,
1082                                   flags);
1083         if (!edesc) {
1084                 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1085                 return -ENOMEM;
1086         }
1087
1088         desc = edesc->hw_desc;
1089
1090         edesc->src_nents = src_nents;
1091
1092         ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len,
1093                                  edesc->sec4_sg, DMA_TO_DEVICE);
1094         if (ret)
1095                 goto unmap_ctx;
1096
1097         state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1,
1098                                                 buf, state->buf_dma, buflen,
1099                                                 last_buflen);
1100
1101         ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents,
1102                                   sec4_sg_src_index, ctx->ctx_len + buflen,
1103                                   req->nbytes);
1104         if (ret)
1105                 goto unmap_ctx;
1106
1107         edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
1108                                                 digestsize);
1109         if (dma_mapping_error(jrdev, edesc->dst_dma)) {
1110                 dev_err(jrdev, "unable to map dst\n");
1111                 ret = -ENOMEM;
1112                 goto unmap_ctx;
1113         }
1114
1115 #ifdef DEBUG
1116         print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1117                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1118 #endif
1119
1120         ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req);
1121         if (ret)
1122                 goto unmap_ctx;
1123
1124         return -EINPROGRESS;
1125  unmap_ctx:
1126         ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
1127         kfree(edesc);
1128         return ret;
1129 }
1130
1131 static int ahash_digest(struct ahash_request *req)
1132 {
1133         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1134         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1135         struct device *jrdev = ctx->jrdev;
1136         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1137                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1138         u32 *desc;
1139         int digestsize = crypto_ahash_digestsize(ahash);
1140         int src_nents, mapped_nents;
1141         struct ahash_edesc *edesc;
1142         int ret;
1143
1144         src_nents = sg_nents_for_len(req->src, req->nbytes);
1145         if (src_nents < 0) {
1146                 dev_err(jrdev, "Invalid number of src SG.\n");
1147                 return src_nents;
1148         }
1149
1150         if (src_nents) {
1151                 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1152                                           DMA_TO_DEVICE);
1153                 if (!mapped_nents) {
1154                         dev_err(jrdev, "unable to map source for DMA\n");
1155                         return -ENOMEM;
1156                 }
1157         } else {
1158                 mapped_nents = 0;
1159         }
1160
1161         /* allocate space for base edesc and hw desc commands, link tables */
1162         edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ? mapped_nents : 0,
1163                                   ctx->sh_desc_digest, ctx->sh_desc_digest_dma,
1164                                   flags);
1165         if (!edesc) {
1166                 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1167                 return -ENOMEM;
1168         }
1169
1170         edesc->src_nents = src_nents;
1171
1172         ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0,
1173                                   req->nbytes);
1174         if (ret) {
1175                 ahash_unmap(jrdev, edesc, req, digestsize);
1176                 kfree(edesc);
1177                 return ret;
1178         }
1179
1180         desc = edesc->hw_desc;
1181
1182         edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
1183                                                 digestsize);
1184         if (dma_mapping_error(jrdev, edesc->dst_dma)) {
1185                 dev_err(jrdev, "unable to map dst\n");
1186                 ahash_unmap(jrdev, edesc, req, digestsize);
1187                 kfree(edesc);
1188                 return -ENOMEM;
1189         }
1190
1191 #ifdef DEBUG
1192         print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1193                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1194 #endif
1195
1196         ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
1197         if (!ret) {
1198                 ret = -EINPROGRESS;
1199         } else {
1200                 ahash_unmap(jrdev, edesc, req, digestsize);
1201                 kfree(edesc);
1202         }
1203
1204         return ret;
1205 }
1206
1207 /* submit ahash final if it the first job descriptor */
1208 static int ahash_final_no_ctx(struct ahash_request *req)
1209 {
1210         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1211         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1212         struct caam_hash_state *state = ahash_request_ctx(req);
1213         struct device *jrdev = ctx->jrdev;
1214         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1215                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1216         u8 *buf = state->current_buf ? state->buf_1 : state->buf_0;
1217         int buflen = state->current_buf ? state->buflen_1 : state->buflen_0;
1218         u32 *desc;
1219         int digestsize = crypto_ahash_digestsize(ahash);
1220         struct ahash_edesc *edesc;
1221         int ret;
1222
1223         /* allocate space for base edesc and hw desc commands, link tables */
1224         edesc = ahash_edesc_alloc(ctx, 0, ctx->sh_desc_digest,
1225                                   ctx->sh_desc_digest_dma, flags);
1226         if (!edesc)
1227                 return -ENOMEM;
1228
1229         desc = edesc->hw_desc;
1230
1231         state->buf_dma = dma_map_single(jrdev, buf, buflen, DMA_TO_DEVICE);
1232         if (dma_mapping_error(jrdev, state->buf_dma)) {
1233                 dev_err(jrdev, "unable to map src\n");
1234                 ahash_unmap(jrdev, edesc, req, digestsize);
1235                 kfree(edesc);
1236                 return -ENOMEM;
1237         }
1238
1239         append_seq_in_ptr(desc, state->buf_dma, buflen, 0);
1240
1241         edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
1242                                                 digestsize);
1243         if (dma_mapping_error(jrdev, edesc->dst_dma)) {
1244                 dev_err(jrdev, "unable to map dst\n");
1245                 ahash_unmap(jrdev, edesc, req, digestsize);
1246                 kfree(edesc);
1247                 return -ENOMEM;
1248         }
1249         edesc->src_nents = 0;
1250
1251 #ifdef DEBUG
1252         print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1253                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1254 #endif
1255
1256         ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
1257         if (!ret) {
1258                 ret = -EINPROGRESS;
1259         } else {
1260                 ahash_unmap(jrdev, edesc, req, digestsize);
1261                 kfree(edesc);
1262         }
1263
1264         return ret;
1265 }
1266
1267 /* submit ahash update if it the first job descriptor after update */
1268 static int ahash_update_no_ctx(struct ahash_request *req)
1269 {
1270         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1271         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1272         struct caam_hash_state *state = ahash_request_ctx(req);
1273         struct device *jrdev = ctx->jrdev;
1274         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1275                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1276         u8 *buf = state->current_buf ? state->buf_1 : state->buf_0;
1277         int *buflen = state->current_buf ? &state->buflen_1 : &state->buflen_0;
1278         u8 *next_buf = state->current_buf ? state->buf_0 : state->buf_1;
1279         int *next_buflen = state->current_buf ? &state->buflen_0 :
1280                            &state->buflen_1;
1281         int in_len = *buflen + req->nbytes, to_hash;
1282         int sec4_sg_bytes, src_nents, mapped_nents;
1283         struct ahash_edesc *edesc;
1284         u32 *desc;
1285         int ret = 0;
1286
1287         *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
1288         to_hash = in_len - *next_buflen;
1289
1290         if (to_hash) {
1291                 src_nents = sg_nents_for_len(req->src,
1292                                              req->nbytes - *next_buflen);
1293                 if (src_nents < 0) {
1294                         dev_err(jrdev, "Invalid number of src SG.\n");
1295                         return src_nents;
1296                 }
1297
1298                 if (src_nents) {
1299                         mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1300                                                   DMA_TO_DEVICE);
1301                         if (!mapped_nents) {
1302                                 dev_err(jrdev, "unable to DMA map source\n");
1303                                 return -ENOMEM;
1304                         }
1305                 } else {
1306                         mapped_nents = 0;
1307                 }
1308
1309                 sec4_sg_bytes = (1 + mapped_nents) *
1310                                 sizeof(struct sec4_sg_entry);
1311
1312                 /*
1313                  * allocate space for base edesc and hw desc commands,
1314                  * link tables
1315                  */
1316                 edesc = ahash_edesc_alloc(ctx, 1 + mapped_nents,
1317                                           ctx->sh_desc_update_first,
1318                                           ctx->sh_desc_update_first_dma,
1319                                           flags);
1320                 if (!edesc) {
1321                         dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1322                         return -ENOMEM;
1323                 }
1324
1325                 edesc->src_nents = src_nents;
1326                 edesc->sec4_sg_bytes = sec4_sg_bytes;
1327                 edesc->dst_dma = 0;
1328
1329                 state->buf_dma = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg,
1330                                                     buf, *buflen);
1331                 sg_to_sec4_sg_last(req->src, mapped_nents,
1332                                    edesc->sec4_sg + 1, 0);
1333
1334                 if (*next_buflen) {
1335                         scatterwalk_map_and_copy(next_buf, req->src,
1336                                                  to_hash - *buflen,
1337                                                  *next_buflen, 0);
1338                 }
1339
1340                 state->current_buf = !state->current_buf;
1341
1342                 desc = edesc->hw_desc;
1343
1344                 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1345                                                     sec4_sg_bytes,
1346                                                     DMA_TO_DEVICE);
1347                 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1348                         dev_err(jrdev, "unable to map S/G table\n");
1349                         ret = -ENOMEM;
1350                         goto unmap_ctx;
1351                 }
1352
1353                 append_seq_in_ptr(desc, edesc->sec4_sg_dma, to_hash, LDST_SGF);
1354
1355                 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
1356                 if (ret)
1357                         goto unmap_ctx;
1358
1359 #ifdef DEBUG
1360                 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1361                                DUMP_PREFIX_ADDRESS, 16, 4, desc,
1362                                desc_bytes(desc), 1);
1363 #endif
1364
1365                 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req);
1366                 if (ret)
1367                         goto unmap_ctx;
1368
1369                 ret = -EINPROGRESS;
1370                 state->update = ahash_update_ctx;
1371                 state->finup = ahash_finup_ctx;
1372                 state->final = ahash_final_ctx;
1373         } else if (*next_buflen) {
1374                 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
1375                                          req->nbytes, 0);
1376                 *buflen = *next_buflen;
1377                 *next_buflen = 0;
1378         }
1379 #ifdef DEBUG
1380         print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ",
1381                        DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
1382         print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
1383                        DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
1384                        *next_buflen, 1);
1385 #endif
1386
1387         return ret;
1388  unmap_ctx:
1389         ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);
1390         kfree(edesc);
1391         return ret;
1392 }
1393
1394 /* submit ahash finup if it the first job descriptor after update */
1395 static int ahash_finup_no_ctx(struct ahash_request *req)
1396 {
1397         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1398         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1399         struct caam_hash_state *state = ahash_request_ctx(req);
1400         struct device *jrdev = ctx->jrdev;
1401         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1402                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1403         u8 *buf = state->current_buf ? state->buf_1 : state->buf_0;
1404         int buflen = state->current_buf ? state->buflen_1 : state->buflen_0;
1405         int last_buflen = state->current_buf ? state->buflen_0 :
1406                           state->buflen_1;
1407         u32 *desc;
1408         int sec4_sg_bytes, sec4_sg_src_index, src_nents, mapped_nents;
1409         int digestsize = crypto_ahash_digestsize(ahash);
1410         struct ahash_edesc *edesc;
1411         int ret;
1412
1413         src_nents = sg_nents_for_len(req->src, req->nbytes);
1414         if (src_nents < 0) {
1415                 dev_err(jrdev, "Invalid number of src SG.\n");
1416                 return src_nents;
1417         }
1418
1419         if (src_nents) {
1420                 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1421                                           DMA_TO_DEVICE);
1422                 if (!mapped_nents) {
1423                         dev_err(jrdev, "unable to DMA map source\n");
1424                         return -ENOMEM;
1425                 }
1426         } else {
1427                 mapped_nents = 0;
1428         }
1429
1430         sec4_sg_src_index = 2;
1431         sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) *
1432                          sizeof(struct sec4_sg_entry);
1433
1434         /* allocate space for base edesc and hw desc commands, link tables */
1435         edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
1436                                   ctx->sh_desc_digest, ctx->sh_desc_digest_dma,
1437                                   flags);
1438         if (!edesc) {
1439                 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1440                 return -ENOMEM;
1441         }
1442
1443         desc = edesc->hw_desc;
1444
1445         edesc->src_nents = src_nents;
1446         edesc->sec4_sg_bytes = sec4_sg_bytes;
1447
1448         state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, buf,
1449                                                 state->buf_dma, buflen,
1450                                                 last_buflen);
1451
1452         ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 1, buflen,
1453                                   req->nbytes);
1454         if (ret) {
1455                 dev_err(jrdev, "unable to map S/G table\n");
1456                 ahash_unmap(jrdev, edesc, req, digestsize);
1457                 kfree(edesc);
1458                 return -ENOMEM;
1459         }
1460
1461         edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
1462                                                 digestsize);
1463         if (dma_mapping_error(jrdev, edesc->dst_dma)) {
1464                 dev_err(jrdev, "unable to map dst\n");
1465                 ahash_unmap(jrdev, edesc, req, digestsize);
1466                 kfree(edesc);
1467                 return -ENOMEM;
1468         }
1469
1470 #ifdef DEBUG
1471         print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1472                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1473 #endif
1474
1475         ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
1476         if (!ret) {
1477                 ret = -EINPROGRESS;
1478         } else {
1479                 ahash_unmap(jrdev, edesc, req, digestsize);
1480                 kfree(edesc);
1481         }
1482
1483         return ret;
1484 }
1485
1486 /* submit first update job descriptor after init */
1487 static int ahash_update_first(struct ahash_request *req)
1488 {
1489         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1490         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1491         struct caam_hash_state *state = ahash_request_ctx(req);
1492         struct device *jrdev = ctx->jrdev;
1493         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1494                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1495         u8 *next_buf = state->current_buf ? state->buf_1 : state->buf_0;
1496         int *next_buflen = state->current_buf ?
1497                 &state->buflen_1 : &state->buflen_0;
1498         int to_hash;
1499         u32 *desc;
1500         int src_nents, mapped_nents;
1501         struct ahash_edesc *edesc;
1502         int ret = 0;
1503
1504         *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) -
1505                                       1);
1506         to_hash = req->nbytes - *next_buflen;
1507
1508         if (to_hash) {
1509                 src_nents = sg_nents_for_len(req->src,
1510                                              req->nbytes - *next_buflen);
1511                 if (src_nents < 0) {
1512                         dev_err(jrdev, "Invalid number of src SG.\n");
1513                         return src_nents;
1514                 }
1515
1516                 if (src_nents) {
1517                         mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1518                                                   DMA_TO_DEVICE);
1519                         if (!mapped_nents) {
1520                                 dev_err(jrdev, "unable to map source for DMA\n");
1521                                 return -ENOMEM;
1522                         }
1523                 } else {
1524                         mapped_nents = 0;
1525                 }
1526
1527                 /*
1528                  * allocate space for base edesc and hw desc commands,
1529                  * link tables
1530                  */
1531                 edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ?
1532                                           mapped_nents : 0,
1533                                           ctx->sh_desc_update_first,
1534                                           ctx->sh_desc_update_first_dma,
1535                                           flags);
1536                 if (!edesc) {
1537                         dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1538                         return -ENOMEM;
1539                 }
1540
1541                 edesc->src_nents = src_nents;
1542                 edesc->dst_dma = 0;
1543
1544                 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0,
1545                                           to_hash);
1546                 if (ret)
1547                         goto unmap_ctx;
1548
1549                 if (*next_buflen)
1550                         scatterwalk_map_and_copy(next_buf, req->src, to_hash,
1551                                                  *next_buflen, 0);
1552
1553                 desc = edesc->hw_desc;
1554
1555                 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
1556                 if (ret)
1557                         goto unmap_ctx;
1558
1559 #ifdef DEBUG
1560                 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1561                                DUMP_PREFIX_ADDRESS, 16, 4, desc,
1562                                desc_bytes(desc), 1);
1563 #endif
1564
1565                 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req);
1566                 if (ret)
1567                         goto unmap_ctx;
1568
1569                 ret = -EINPROGRESS;
1570                 state->update = ahash_update_ctx;
1571                 state->finup = ahash_finup_ctx;
1572                 state->final = ahash_final_ctx;
1573         } else if (*next_buflen) {
1574                 state->update = ahash_update_no_ctx;
1575                 state->finup = ahash_finup_no_ctx;
1576                 state->final = ahash_final_no_ctx;
1577                 scatterwalk_map_and_copy(next_buf, req->src, 0,
1578                                          req->nbytes, 0);
1579         }
1580 #ifdef DEBUG
1581         print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
1582                        DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
1583                        *next_buflen, 1);
1584 #endif
1585
1586         return ret;
1587  unmap_ctx:
1588         ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);
1589         kfree(edesc);
1590         return ret;
1591 }
1592
1593 static int ahash_finup_first(struct ahash_request *req)
1594 {
1595         return ahash_digest(req);
1596 }
1597
1598 static int ahash_init(struct ahash_request *req)
1599 {
1600         struct caam_hash_state *state = ahash_request_ctx(req);
1601
1602         state->update = ahash_update_first;
1603         state->finup = ahash_finup_first;
1604         state->final = ahash_final_no_ctx;
1605
1606         state->current_buf = 0;
1607         state->buf_dma = 0;
1608         state->buflen_0 = 0;
1609         state->buflen_1 = 0;
1610
1611         return 0;
1612 }
1613
1614 static int ahash_update(struct ahash_request *req)
1615 {
1616         struct caam_hash_state *state = ahash_request_ctx(req);
1617
1618         return state->update(req);
1619 }
1620
1621 static int ahash_finup(struct ahash_request *req)
1622 {
1623         struct caam_hash_state *state = ahash_request_ctx(req);
1624
1625         return state->finup(req);
1626 }
1627
1628 static int ahash_final(struct ahash_request *req)
1629 {
1630         struct caam_hash_state *state = ahash_request_ctx(req);
1631
1632         return state->final(req);
1633 }
1634
1635 static int ahash_export(struct ahash_request *req, void *out)
1636 {
1637         struct caam_hash_state *state = ahash_request_ctx(req);
1638         struct caam_export_state *export = out;
1639         int len;
1640         u8 *buf;
1641
1642         if (state->current_buf) {
1643                 buf = state->buf_1;
1644                 len = state->buflen_1;
1645         } else {
1646                 buf = state->buf_0;
1647                 len = state->buflen_0;
1648         }
1649
1650         memcpy(export->buf, buf, len);
1651         memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx));
1652         export->buflen = len;
1653         export->update = state->update;
1654         export->final = state->final;
1655         export->finup = state->finup;
1656
1657         return 0;
1658 }
1659
1660 static int ahash_import(struct ahash_request *req, const void *in)
1661 {
1662         struct caam_hash_state *state = ahash_request_ctx(req);
1663         const struct caam_export_state *export = in;
1664
1665         memset(state, 0, sizeof(*state));
1666         memcpy(state->buf_0, export->buf, export->buflen);
1667         memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx));
1668         state->buflen_0 = export->buflen;
1669         state->update = export->update;
1670         state->final = export->final;
1671         state->finup = export->finup;
1672
1673         return 0;
1674 }
1675
1676 struct caam_hash_template {
1677         char name[CRYPTO_MAX_ALG_NAME];
1678         char driver_name[CRYPTO_MAX_ALG_NAME];
1679         char hmac_name[CRYPTO_MAX_ALG_NAME];
1680         char hmac_driver_name[CRYPTO_MAX_ALG_NAME];
1681         unsigned int blocksize;
1682         struct ahash_alg template_ahash;
1683         u32 alg_type;
1684         u32 alg_op;
1685 };
1686
1687 /* ahash descriptors */
1688 static struct caam_hash_template driver_hash[] = {
1689         {
1690                 .name = "sha1",
1691                 .driver_name = "sha1-caam",
1692                 .hmac_name = "hmac(sha1)",
1693                 .hmac_driver_name = "hmac-sha1-caam",
1694                 .blocksize = SHA1_BLOCK_SIZE,
1695                 .template_ahash = {
1696                         .init = ahash_init,
1697                         .update = ahash_update,
1698                         .final = ahash_final,
1699                         .finup = ahash_finup,
1700                         .digest = ahash_digest,
1701                         .export = ahash_export,
1702                         .import = ahash_import,
1703                         .setkey = ahash_setkey,
1704                         .halg = {
1705                                 .digestsize = SHA1_DIGEST_SIZE,
1706                                 .statesize = sizeof(struct caam_export_state),
1707                         },
1708                 },
1709                 .alg_type = OP_ALG_ALGSEL_SHA1,
1710                 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
1711         }, {
1712                 .name = "sha224",
1713                 .driver_name = "sha224-caam",
1714                 .hmac_name = "hmac(sha224)",
1715                 .hmac_driver_name = "hmac-sha224-caam",
1716                 .blocksize = SHA224_BLOCK_SIZE,
1717                 .template_ahash = {
1718                         .init = ahash_init,
1719                         .update = ahash_update,
1720                         .final = ahash_final,
1721                         .finup = ahash_finup,
1722                         .digest = ahash_digest,
1723                         .export = ahash_export,
1724                         .import = ahash_import,
1725                         .setkey = ahash_setkey,
1726                         .halg = {
1727                                 .digestsize = SHA224_DIGEST_SIZE,
1728                                 .statesize = sizeof(struct caam_export_state),
1729                         },
1730                 },
1731                 .alg_type = OP_ALG_ALGSEL_SHA224,
1732                 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
1733         }, {
1734                 .name = "sha256",
1735                 .driver_name = "sha256-caam",
1736                 .hmac_name = "hmac(sha256)",
1737                 .hmac_driver_name = "hmac-sha256-caam",
1738                 .blocksize = SHA256_BLOCK_SIZE,
1739                 .template_ahash = {
1740                         .init = ahash_init,
1741                         .update = ahash_update,
1742                         .final = ahash_final,
1743                         .finup = ahash_finup,
1744                         .digest = ahash_digest,
1745                         .export = ahash_export,
1746                         .import = ahash_import,
1747                         .setkey = ahash_setkey,
1748                         .halg = {
1749                                 .digestsize = SHA256_DIGEST_SIZE,
1750                                 .statesize = sizeof(struct caam_export_state),
1751                         },
1752                 },
1753                 .alg_type = OP_ALG_ALGSEL_SHA256,
1754                 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
1755         }, {
1756                 .name = "sha384",
1757                 .driver_name = "sha384-caam",
1758                 .hmac_name = "hmac(sha384)",
1759                 .hmac_driver_name = "hmac-sha384-caam",
1760                 .blocksize = SHA384_BLOCK_SIZE,
1761                 .template_ahash = {
1762                         .init = ahash_init,
1763                         .update = ahash_update,
1764                         .final = ahash_final,
1765                         .finup = ahash_finup,
1766                         .digest = ahash_digest,
1767                         .export = ahash_export,
1768                         .import = ahash_import,
1769                         .setkey = ahash_setkey,
1770                         .halg = {
1771                                 .digestsize = SHA384_DIGEST_SIZE,
1772                                 .statesize = sizeof(struct caam_export_state),
1773                         },
1774                 },
1775                 .alg_type = OP_ALG_ALGSEL_SHA384,
1776                 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
1777         }, {
1778                 .name = "sha512",
1779                 .driver_name = "sha512-caam",
1780                 .hmac_name = "hmac(sha512)",
1781                 .hmac_driver_name = "hmac-sha512-caam",
1782                 .blocksize = SHA512_BLOCK_SIZE,
1783                 .template_ahash = {
1784                         .init = ahash_init,
1785                         .update = ahash_update,
1786                         .final = ahash_final,
1787                         .finup = ahash_finup,
1788                         .digest = ahash_digest,
1789                         .export = ahash_export,
1790                         .import = ahash_import,
1791                         .setkey = ahash_setkey,
1792                         .halg = {
1793                                 .digestsize = SHA512_DIGEST_SIZE,
1794                                 .statesize = sizeof(struct caam_export_state),
1795                         },
1796                 },
1797                 .alg_type = OP_ALG_ALGSEL_SHA512,
1798                 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
1799         }, {
1800                 .name = "md5",
1801                 .driver_name = "md5-caam",
1802                 .hmac_name = "hmac(md5)",
1803                 .hmac_driver_name = "hmac-md5-caam",
1804                 .blocksize = MD5_BLOCK_WORDS * 4,
1805                 .template_ahash = {
1806                         .init = ahash_init,
1807                         .update = ahash_update,
1808                         .final = ahash_final,
1809                         .finup = ahash_finup,
1810                         .digest = ahash_digest,
1811                         .export = ahash_export,
1812                         .import = ahash_import,
1813                         .setkey = ahash_setkey,
1814                         .halg = {
1815                                 .digestsize = MD5_DIGEST_SIZE,
1816                                 .statesize = sizeof(struct caam_export_state),
1817                         },
1818                 },
1819                 .alg_type = OP_ALG_ALGSEL_MD5,
1820                 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
1821         },
1822 };
1823
1824 struct caam_hash_alg {
1825         struct list_head entry;
1826         int alg_type;
1827         int alg_op;
1828         struct ahash_alg ahash_alg;
1829 };
1830
1831 static int caam_hash_cra_init(struct crypto_tfm *tfm)
1832 {
1833         struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
1834         struct crypto_alg *base = tfm->__crt_alg;
1835         struct hash_alg_common *halg =
1836                  container_of(base, struct hash_alg_common, base);
1837         struct ahash_alg *alg =
1838                  container_of(halg, struct ahash_alg, halg);
1839         struct caam_hash_alg *caam_hash =
1840                  container_of(alg, struct caam_hash_alg, ahash_alg);
1841         struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1842         /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */
1843         static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE,
1844                                          HASH_MSG_LEN + SHA1_DIGEST_SIZE,
1845                                          HASH_MSG_LEN + 32,
1846                                          HASH_MSG_LEN + SHA256_DIGEST_SIZE,
1847                                          HASH_MSG_LEN + 64,
1848                                          HASH_MSG_LEN + SHA512_DIGEST_SIZE };
1849
1850         /*
1851          * Get a Job ring from Job Ring driver to ensure in-order
1852          * crypto request processing per tfm
1853          */
1854         ctx->jrdev = caam_jr_alloc();
1855         if (IS_ERR(ctx->jrdev)) {
1856                 pr_err("Job Ring Device allocation for transform failed\n");
1857                 return PTR_ERR(ctx->jrdev);
1858         }
1859         /* copy descriptor header template value */
1860         ctx->alg_type = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
1861         ctx->alg_op = OP_TYPE_CLASS2_ALG | caam_hash->alg_op;
1862
1863         ctx->ctx_len = runninglen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >>
1864                                   OP_ALG_ALGSEL_SHIFT];
1865
1866         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1867                                  sizeof(struct caam_hash_state));
1868         return ahash_set_sh_desc(ahash);
1869 }
1870
1871 static void caam_hash_cra_exit(struct crypto_tfm *tfm)
1872 {
1873         struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1874
1875         if (ctx->sh_desc_update_dma &&
1876             !dma_mapping_error(ctx->jrdev, ctx->sh_desc_update_dma))
1877                 dma_unmap_single(ctx->jrdev, ctx->sh_desc_update_dma,
1878                                  desc_bytes(ctx->sh_desc_update),
1879                                  DMA_TO_DEVICE);
1880         if (ctx->sh_desc_update_first_dma &&
1881             !dma_mapping_error(ctx->jrdev, ctx->sh_desc_update_first_dma))
1882                 dma_unmap_single(ctx->jrdev, ctx->sh_desc_update_first_dma,
1883                                  desc_bytes(ctx->sh_desc_update_first),
1884                                  DMA_TO_DEVICE);
1885         if (ctx->sh_desc_fin_dma &&
1886             !dma_mapping_error(ctx->jrdev, ctx->sh_desc_fin_dma))
1887                 dma_unmap_single(ctx->jrdev, ctx->sh_desc_fin_dma,
1888                                  desc_bytes(ctx->sh_desc_fin), DMA_TO_DEVICE);
1889         if (ctx->sh_desc_digest_dma &&
1890             !dma_mapping_error(ctx->jrdev, ctx->sh_desc_digest_dma))
1891                 dma_unmap_single(ctx->jrdev, ctx->sh_desc_digest_dma,
1892                                  desc_bytes(ctx->sh_desc_digest),
1893                                  DMA_TO_DEVICE);
1894         if (ctx->sh_desc_finup_dma &&
1895             !dma_mapping_error(ctx->jrdev, ctx->sh_desc_finup_dma))
1896                 dma_unmap_single(ctx->jrdev, ctx->sh_desc_finup_dma,
1897                                  desc_bytes(ctx->sh_desc_finup), DMA_TO_DEVICE);
1898
1899         caam_jr_free(ctx->jrdev);
1900 }
1901
1902 static void __exit caam_algapi_hash_exit(void)
1903 {
1904         struct caam_hash_alg *t_alg, *n;
1905
1906         if (!hash_list.next)
1907                 return;
1908
1909         list_for_each_entry_safe(t_alg, n, &hash_list, entry) {
1910                 crypto_unregister_ahash(&t_alg->ahash_alg);
1911                 list_del(&t_alg->entry);
1912                 kfree(t_alg);
1913         }
1914 }
1915
1916 static struct caam_hash_alg *
1917 caam_hash_alloc(struct caam_hash_template *template,
1918                 bool keyed)
1919 {
1920         struct caam_hash_alg *t_alg;
1921         struct ahash_alg *halg;
1922         struct crypto_alg *alg;
1923
1924         t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
1925         if (!t_alg) {
1926                 pr_err("failed to allocate t_alg\n");
1927                 return ERR_PTR(-ENOMEM);
1928         }
1929
1930         t_alg->ahash_alg = template->template_ahash;
1931         halg = &t_alg->ahash_alg;
1932         alg = &halg->halg.base;
1933
1934         if (keyed) {
1935                 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
1936                          template->hmac_name);
1937                 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1938                          template->hmac_driver_name);
1939         } else {
1940                 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
1941                          template->name);
1942                 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1943                          template->driver_name);
1944                 t_alg->ahash_alg.setkey = NULL;
1945         }
1946         alg->cra_module = THIS_MODULE;
1947         alg->cra_init = caam_hash_cra_init;
1948         alg->cra_exit = caam_hash_cra_exit;
1949         alg->cra_ctxsize = sizeof(struct caam_hash_ctx);
1950         alg->cra_priority = CAAM_CRA_PRIORITY;
1951         alg->cra_blocksize = template->blocksize;
1952         alg->cra_alignmask = 0;
1953         alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_TYPE_AHASH;
1954         alg->cra_type = &crypto_ahash_type;
1955
1956         t_alg->alg_type = template->alg_type;
1957         t_alg->alg_op = template->alg_op;
1958
1959         return t_alg;
1960 }
1961
1962 static int __init caam_algapi_hash_init(void)
1963 {
1964         struct device_node *dev_node;
1965         struct platform_device *pdev;
1966         struct device *ctrldev;
1967         int i = 0, err = 0;
1968         struct caam_drv_private *priv;
1969         unsigned int md_limit = SHA512_DIGEST_SIZE;
1970         u32 cha_inst, cha_vid;
1971
1972         dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
1973         if (!dev_node) {
1974                 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
1975                 if (!dev_node)
1976                         return -ENODEV;
1977         }
1978
1979         pdev = of_find_device_by_node(dev_node);
1980         if (!pdev) {
1981                 of_node_put(dev_node);
1982                 return -ENODEV;
1983         }
1984
1985         ctrldev = &pdev->dev;
1986         priv = dev_get_drvdata(ctrldev);
1987         of_node_put(dev_node);
1988
1989         /*
1990          * If priv is NULL, it's probably because the caam driver wasn't
1991          * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
1992          */
1993         if (!priv)
1994                 return -ENODEV;
1995
1996         /*
1997          * Register crypto algorithms the device supports.  First, identify
1998          * presence and attributes of MD block.
1999          */
2000         cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
2001         cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
2002
2003         /*
2004          * Skip registration of any hashing algorithms if MD block
2005          * is not present.
2006          */
2007         if (!((cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT))
2008                 return -ENODEV;
2009
2010         /* Limit digest size based on LP256 */
2011         if ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256)
2012                 md_limit = SHA256_DIGEST_SIZE;
2013
2014         INIT_LIST_HEAD(&hash_list);
2015
2016         /* register crypto algorithms the device supports */
2017         for (i = 0; i < ARRAY_SIZE(driver_hash); i++) {
2018                 struct caam_hash_alg *t_alg;
2019                 struct caam_hash_template *alg = driver_hash + i;
2020
2021                 /* If MD size is not supported by device, skip registration */
2022                 if (alg->template_ahash.halg.digestsize > md_limit)
2023                         continue;
2024
2025                 /* register hmac version */
2026                 t_alg = caam_hash_alloc(alg, true);
2027                 if (IS_ERR(t_alg)) {
2028                         err = PTR_ERR(t_alg);
2029                         pr_warn("%s alg allocation failed\n", alg->driver_name);
2030                         continue;
2031                 }
2032
2033                 err = crypto_register_ahash(&t_alg->ahash_alg);
2034                 if (err) {
2035                         pr_warn("%s alg registration failed: %d\n",
2036                                 t_alg->ahash_alg.halg.base.cra_driver_name,
2037                                 err);
2038                         kfree(t_alg);
2039                 } else
2040                         list_add_tail(&t_alg->entry, &hash_list);
2041
2042                 /* register unkeyed version */
2043                 t_alg = caam_hash_alloc(alg, false);
2044                 if (IS_ERR(t_alg)) {
2045                         err = PTR_ERR(t_alg);
2046                         pr_warn("%s alg allocation failed\n", alg->driver_name);
2047                         continue;
2048                 }
2049
2050                 err = crypto_register_ahash(&t_alg->ahash_alg);
2051                 if (err) {
2052                         pr_warn("%s alg registration failed: %d\n",
2053                                 t_alg->ahash_alg.halg.base.cra_driver_name,
2054                                 err);
2055                         kfree(t_alg);
2056                 } else
2057                         list_add_tail(&t_alg->entry, &hash_list);
2058         }
2059
2060         return err;
2061 }
2062
2063 module_init(caam_algapi_hash_init);
2064 module_exit(caam_algapi_hash_exit);
2065
2066 MODULE_LICENSE("GPL");
2067 MODULE_DESCRIPTION("FSL CAAM support for ahash functions of crypto API");
2068 MODULE_AUTHOR("Freescale Semiconductor - NMG");