drivers/crypto/atmel-sha.c

   1 /*
   2  * Cryptographic API.
   3  *
   4  * Support for ATMEL SHA1/SHA256 HW acceleration.
   5  *
   6  * Copyright (c) 2012 Eukréa Electromatique - ATMEL
   7  * Author: Nicolas Royer <nicolas@eukrea.com>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2 as published
  11  * by the Free Software Foundation.
  12  *
  13  * Some ideas are from omap-sham.c drivers.
  14  */
  15
  16
  17 #include <linux/kernel.h>
  18 #include <linux/module.h>
  19 #include <linux/slab.h>
  20 #include <linux/err.h>
  21 #include <linux/clk.h>
  22 #include <linux/io.h>
  23 #include <linux/hw_random.h>
  24 #include <linux/platform_device.h>
  25
  26 #include <linux/device.h>
  27 #include <linux/init.h>
  28 #include <linux/errno.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/irq.h>
  31 #include <linux/scatterlist.h>
  32 #include <linux/dma-mapping.h>
  33 #include <linux/of_device.h>
  34 #include <linux/delay.h>
  35 #include <linux/crypto.h>
  36 #include <linux/cryptohash.h>
  37 #include <crypto/scatterwalk.h>
  38 #include <crypto/algapi.h>
  39 #include <crypto/sha.h>
  40 #include <crypto/hash.h>
  41 #include <crypto/internal/hash.h>
  42 #include <linux/platform_data/crypto-atmel.h>
  43 #include "atmel-sha-regs.h"
  44
  45 /* SHA flags */
  46 #define SHA_FLAGS_BUSY                  BIT(0)
  47 #define SHA_FLAGS_FINAL                 BIT(1)
  48 #define SHA_FLAGS_DMA_ACTIVE    BIT(2)
  49 #define SHA_FLAGS_OUTPUT_READY  BIT(3)
  50 #define SHA_FLAGS_INIT                  BIT(4)
  51 #define SHA_FLAGS_CPU                   BIT(5)
  52 #define SHA_FLAGS_DMA_READY             BIT(6)
  53
  54 #define SHA_FLAGS_FINUP         BIT(16)
  55 #define SHA_FLAGS_SG            BIT(17)
  56 #define SHA_FLAGS_ALGO_MASK     GENMASK(22, 18)
  57 #define SHA_FLAGS_SHA1          BIT(18)
  58 #define SHA_FLAGS_SHA224        BIT(19)
  59 #define SHA_FLAGS_SHA256        BIT(20)
  60 #define SHA_FLAGS_SHA384        BIT(21)
  61 #define SHA_FLAGS_SHA512        BIT(22)
  62 #define SHA_FLAGS_ERROR         BIT(23)
  63 #define SHA_FLAGS_PAD           BIT(24)
  64 #define SHA_FLAGS_RESTORE       BIT(25)
  65
  66 #define SHA_OP_UPDATE   1
  67 #define SHA_OP_FINAL    2
  68
  69 #define SHA_BUFFER_LEN          (PAGE_SIZE / 16)
  70
  71 #define ATMEL_SHA_DMA_THRESHOLD         56
  72
  73 struct atmel_sha_caps {
  74         bool    has_dma;
  75         bool    has_dualbuff;
  76         bool    has_sha224;
  77         bool    has_sha_384_512;
  78         bool    has_uihv;
  79 };
  80
  81 struct atmel_sha_dev;
  82
  83 /*
  84  * .statesize = sizeof(struct atmel_sha_reqctx) must be <= PAGE_SIZE / 8 as
  85  * tested by the ahash_prepare_alg() function.
  86  */
  87 struct atmel_sha_reqctx {
  88         struct atmel_sha_dev    *dd;
  89         unsigned long   flags;
  90         unsigned long   op;
  91
  92         u8      digest[SHA512_DIGEST_SIZE] __aligned(sizeof(u32));
  93         u64     digcnt[2];
  94         size_t  bufcnt;
  95         size_t  buflen;
  96         dma_addr_t      dma_addr;
  97
  98         /* walk state */
  99         struct scatterlist      *sg;
 100         unsigned int    offset; /* offset in current sg */
 101         unsigned int    total;  /* total request */
 102
 103         size_t block_size;
 104
 105         u8 buffer[SHA_BUFFER_LEN + SHA512_BLOCK_SIZE] __aligned(sizeof(u32));
 106 };
 107
 108 struct atmel_sha_ctx {
 109         struct atmel_sha_dev    *dd;
 110
 111         unsigned long           flags;
 112 };
 113
 114 #define ATMEL_SHA_QUEUE_LENGTH  50
 115
 116 struct atmel_sha_dma {
 117         struct dma_chan                 *chan;
 118         struct dma_slave_config dma_conf;
 119 };
 120
 121 struct atmel_sha_dev {
 122         struct list_head        list;
 123         unsigned long           phys_base;
 124         struct device           *dev;
 125         struct clk                      *iclk;
 126         int                                     irq;
 127         void __iomem            *io_base;
 128
 129         spinlock_t              lock;
 130         int                     err;
 131         struct tasklet_struct   done_task;
 132         struct tasklet_struct   queue_task;
 133
 134         unsigned long           flags;
 135         struct crypto_queue     queue;
 136         struct ahash_request    *req;
 137
 138         struct atmel_sha_dma    dma_lch_in;
 139
 140         struct atmel_sha_caps   caps;
 141
 142         u32     hw_version;
 143 };
 144
 145 struct atmel_sha_drv {
 146         struct list_head        dev_list;
 147         spinlock_t              lock;
 148 };
 149
 150 static struct atmel_sha_drv atmel_sha = {
 151         .dev_list = LIST_HEAD_INIT(atmel_sha.dev_list),
 152         .lock = __SPIN_LOCK_UNLOCKED(atmel_sha.lock),
 153 };
 154
 155 static inline u32 atmel_sha_read(struct atmel_sha_dev *dd, u32 offset)
 156 {
 157         return readl_relaxed(dd->io_base + offset);
 158 }
 159
 160 static inline void atmel_sha_write(struct atmel_sha_dev *dd,
 161                                         u32 offset, u32 value)
 162 {
 163         writel_relaxed(value, dd->io_base + offset);
 164 }
 165
 166 static size_t atmel_sha_append_sg(struct atmel_sha_reqctx *ctx)
 167 {
 168         size_t count;
 169
 170         while ((ctx->bufcnt < ctx->buflen) && ctx->total) {
 171                 count = min(ctx->sg->length - ctx->offset, ctx->total);
 172                 count = min(count, ctx->buflen - ctx->bufcnt);
 173
 174                 if (count <= 0) {
 175                         /*
 176                         * Check if count <= 0 because the buffer is full or
 177                         * because the sg length is 0. In the latest case,
 178                         * check if there is another sg in the list, a 0 length
 179                         * sg doesn't necessarily mean the end of the sg list.
 180                         */
 181                         if ((ctx->sg->length == 0) && !sg_is_last(ctx->sg)) {
 182                                 ctx->sg = sg_next(ctx->sg);
 183                                 continue;
 184                         } else {
 185                                 break;
 186                         }
 187                 }
 188
 189                 scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, ctx->sg,
 190                         ctx->offset, count, 0);
 191
 192                 ctx->bufcnt += count;
 193                 ctx->offset += count;
 194                 ctx->total -= count;
 195
 196                 if (ctx->offset == ctx->sg->length) {
 197                         ctx->sg = sg_next(ctx->sg);
 198                         if (ctx->sg)
 199                                 ctx->offset = 0;
 200                         else
 201                                 ctx->total = 0;
 202                 }
 203         }
 204
 205         return 0;
 206 }
 207
 208 /*
 209  * The purpose of this padding is to ensure that the padded message is a
 210  * multiple of 512 bits (SHA1/SHA224/SHA256) or 1024 bits (SHA384/SHA512).
 211  * The bit "1" is appended at the end of the message followed by
 212  * "padlen-1" zero bits. Then a 64 bits block (SHA1/SHA224/SHA256) or
 213  * 128 bits block (SHA384/SHA512) equals to the message length in bits
 214  * is appended.
 215  *
 216  * For SHA1/SHA224/SHA256, padlen is calculated as followed:
 217  *  - if message length < 56 bytes then padlen = 56 - message length
 218  *  - else padlen = 64 + 56 - message length
 219  *
 220  * For SHA384/SHA512, padlen is calculated as followed:
 221  *  - if message length < 112 bytes then padlen = 112 - message length
 222  *  - else padlen = 128 + 112 - message length
 223  */
 224 static void atmel_sha_fill_padding(struct atmel_sha_reqctx *ctx, int length)
 225 {
 226         unsigned int index, padlen;
 227         u64 bits[2];
 228         u64 size[2];
 229
 230         size[0] = ctx->digcnt[0];
 231         size[1] = ctx->digcnt[1];
 232
 233         size[0] += ctx->bufcnt;
 234         if (size[0] < ctx->bufcnt)
 235                 size[1]++;
 236
 237         size[0] += length;
 238         if (size[0]  < length)
 239                 size[1]++;
 240
 241         bits[1] = cpu_to_be64(size[0] << 3);
 242         bits[0] = cpu_to_be64(size[1] << 3 | size[0] >> 61);
 243
 244         if (ctx->flags & (SHA_FLAGS_SHA384 | SHA_FLAGS_SHA512)) {
 245                 index = ctx->bufcnt & 0x7f;
 246                 padlen = (index < 112) ? (112 - index) : ((128+112) - index);
 247                 *(ctx->buffer + ctx->bufcnt) = 0x80;
 248                 memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen-1);
 249                 memcpy(ctx->buffer + ctx->bufcnt + padlen, bits, 16);
 250                 ctx->bufcnt += padlen + 16;
 251                 ctx->flags |= SHA_FLAGS_PAD;
 252         } else {
 253                 index = ctx->bufcnt & 0x3f;
 254                 padlen = (index < 56) ? (56 - index) : ((64+56) - index);
 255                 *(ctx->buffer + ctx->bufcnt) = 0x80;
 256                 memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen-1);
 257                 memcpy(ctx->buffer + ctx->bufcnt + padlen, &bits[1], 8);
 258                 ctx->bufcnt += padlen + 8;
 259                 ctx->flags |= SHA_FLAGS_PAD;
 260         }
 261 }
 262
 263 static int atmel_sha_init(struct ahash_request *req)
 264 {
 265         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 266         struct atmel_sha_ctx *tctx = crypto_ahash_ctx(tfm);
 267         struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
 268         struct atmel_sha_dev *dd = NULL;
 269         struct atmel_sha_dev *tmp;
 270
 271         spin_lock_bh(&atmel_sha.lock);
 272         if (!tctx->dd) {
 273                 list_for_each_entry(tmp, &atmel_sha.dev_list, list) {
 274                         dd = tmp;
 275                         break;
 276                 }
 277                 tctx->dd = dd;
 278         } else {
 279                 dd = tctx->dd;
 280         }
 281
 282         spin_unlock_bh(&atmel_sha.lock);
 283
 284         ctx->dd = dd;
 285
 286         ctx->flags = 0;
 287
 288         dev_dbg(dd->dev, "init: digest size: %d\n",
 289                 crypto_ahash_digestsize(tfm));
 290
 291         switch (crypto_ahash_digestsize(tfm)) {
 292         case SHA1_DIGEST_SIZE:
 293                 ctx->flags |= SHA_FLAGS_SHA1;
 294                 ctx->block_size = SHA1_BLOCK_SIZE;
 295                 break;
 296         case SHA224_DIGEST_SIZE:
 297                 ctx->flags |= SHA_FLAGS_SHA224;
 298                 ctx->block_size = SHA224_BLOCK_SIZE;
 299                 break;
 300         case SHA256_DIGEST_SIZE:
 301                 ctx->flags |= SHA_FLAGS_SHA256;
 302                 ctx->block_size = SHA256_BLOCK_SIZE;
 303                 break;
 304         case SHA384_DIGEST_SIZE:
 305                 ctx->flags |= SHA_FLAGS_SHA384;
 306                 ctx->block_size = SHA384_BLOCK_SIZE;
 307                 break;
 308         case SHA512_DIGEST_SIZE:
 309                 ctx->flags |= SHA_FLAGS_SHA512;
 310                 ctx->block_size = SHA512_BLOCK_SIZE;
 311                 break;
 312         default:
 313                 return -EINVAL;
 314                 break;
 315         }
 316
 317         ctx->bufcnt = 0;
 318         ctx->digcnt[0] = 0;
 319         ctx->digcnt[1] = 0;
 320         ctx->buflen = SHA_BUFFER_LEN;
 321
 322         return 0;
 323 }
 324
 325 static void atmel_sha_write_ctrl(struct atmel_sha_dev *dd, int dma)
 326 {
 327         struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
 328         u32 valmr = SHA_MR_MODE_AUTO;
 329         unsigned int i, hashsize = 0;
 330
 331         if (likely(dma)) {
 332                 if (!dd->caps.has_dma)
 333                         atmel_sha_write(dd, SHA_IER, SHA_INT_TXBUFE);
 334                 valmr = SHA_MR_MODE_PDC;
 335                 if (dd->caps.has_dualbuff)
 336                         valmr |= SHA_MR_DUALBUFF;
 337         } else {
 338                 atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY);
 339         }
 340
 341         switch (ctx->flags & SHA_FLAGS_ALGO_MASK) {
 342         case SHA_FLAGS_SHA1:
 343                 valmr |= SHA_MR_ALGO_SHA1;
 344                 hashsize = SHA1_DIGEST_SIZE;
 345                 break;
 346
 347         case SHA_FLAGS_SHA224:
 348                 valmr |= SHA_MR_ALGO_SHA224;
 349                 hashsize = SHA256_DIGEST_SIZE;
 350                 break;
 351
 352         case SHA_FLAGS_SHA256:
 353                 valmr |= SHA_MR_ALGO_SHA256;
 354                 hashsize = SHA256_DIGEST_SIZE;
 355                 break;
 356
 357         case SHA_FLAGS_SHA384:
 358                 valmr |= SHA_MR_ALGO_SHA384;
 359                 hashsize = SHA512_DIGEST_SIZE;
 360                 break;
 361
 362         case SHA_FLAGS_SHA512:
 363                 valmr |= SHA_MR_ALGO_SHA512;
 364                 hashsize = SHA512_DIGEST_SIZE;
 365                 break;
 366
 367         default:
 368                 break;
 369         }
 370
 371         /* Setting CR_FIRST only for the first iteration */
 372         if (!(ctx->digcnt[0] || ctx->digcnt[1])) {
 373                 atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST);
 374         } else if (dd->caps.has_uihv && (ctx->flags & SHA_FLAGS_RESTORE)) {
 375                 const u32 *hash = (const u32 *)ctx->digest;
 376
 377                 /*
 378                  * Restore the hardware context: update the User Initialize
 379                  * Hash Value (UIHV) with the value saved when the latest
 380                  * 'update' operation completed on this very same crypto
 381                  * request.
 382                  */
 383                 ctx->flags &= ~SHA_FLAGS_RESTORE;
 384                 atmel_sha_write(dd, SHA_CR, SHA_CR_WUIHV);
 385                 for (i = 0; i < hashsize / sizeof(u32); ++i)
 386                         atmel_sha_write(dd, SHA_REG_DIN(i), hash[i]);
 387                 atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST);
 388                 valmr |= SHA_MR_UIHV;
 389         }
 390         /*
 391          * WARNING: If the UIHV feature is not available, the hardware CANNOT
 392          * process concurrent requests: the internal registers used to store
 393          * the hash/digest are still set to the partial digest output values
 394          * computed during the latest round.
 395          */
 396
 397         atmel_sha_write(dd, SHA_MR, valmr);
 398 }
 399
 400 static int atmel_sha_xmit_cpu(struct atmel_sha_dev *dd, const u8 *buf,
 401                               size_t length, int final)
 402 {
 403         struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
 404         int count, len32;
 405         const u32 *buffer = (const u32 *)buf;
 406
 407         dev_dbg(dd->dev, "xmit_cpu: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n",
 408                 ctx->digcnt[1], ctx->digcnt[0], length, final);
 409
 410         atmel_sha_write_ctrl(dd, 0);
 411
 412         /* should be non-zero before next lines to disable clocks later */
 413         ctx->digcnt[0] += length;
 414         if (ctx->digcnt[0] < length)
 415                 ctx->digcnt[1]++;
 416
 417         if (final)
 418                 dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */
 419
 420         len32 = DIV_ROUND_UP(length, sizeof(u32));
 421
 422         dd->flags |= SHA_FLAGS_CPU;
 423
 424         for (count = 0; count < len32; count++)
 425                 atmel_sha_write(dd, SHA_REG_DIN(count), buffer[count]);
 426
 427         return -EINPROGRESS;
 428 }
 429
 430 static int atmel_sha_xmit_pdc(struct atmel_sha_dev *dd, dma_addr_t dma_addr1,
 431                 size_t length1, dma_addr_t dma_addr2, size_t length2, int final)
 432 {
 433         struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
 434         int len32;
 435
 436         dev_dbg(dd->dev, "xmit_pdc: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n",
 437                 ctx->digcnt[1], ctx->digcnt[0], length1, final);
 438
 439         len32 = DIV_ROUND_UP(length1, sizeof(u32));
 440         atmel_sha_write(dd, SHA_PTCR, SHA_PTCR_TXTDIS);
 441         atmel_sha_write(dd, SHA_TPR, dma_addr1);
 442         atmel_sha_write(dd, SHA_TCR, len32);
 443
 444         len32 = DIV_ROUND_UP(length2, sizeof(u32));
 445         atmel_sha_write(dd, SHA_TNPR, dma_addr2);
 446         atmel_sha_write(dd, SHA_TNCR, len32);
 447
 448         atmel_sha_write_ctrl(dd, 1);
 449
 450         /* should be non-zero before next lines to disable clocks later */
 451         ctx->digcnt[0] += length1;
 452         if (ctx->digcnt[0] < length1)
 453                 ctx->digcnt[1]++;
 454
 455         if (final)
 456                 dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */
 457
 458         dd->flags |=  SHA_FLAGS_DMA_ACTIVE;
 459
 460         /* Start DMA transfer */
 461         atmel_sha_write(dd, SHA_PTCR, SHA_PTCR_TXTEN);
 462
 463         return -EINPROGRESS;
 464 }
 465
 466 static void atmel_sha_dma_callback(void *data)
 467 {
 468         struct atmel_sha_dev *dd = data;
 469
 470         /* dma_lch_in - completed - wait DATRDY */
 471         atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY);
 472 }
 473
 474 static int atmel_sha_xmit_dma(struct atmel_sha_dev *dd, dma_addr_t dma_addr1,
 475                 size_t length1, dma_addr_t dma_addr2, size_t length2, int final)
 476 {
 477         struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
 478         struct dma_async_tx_descriptor  *in_desc;
 479         struct scatterlist sg[2];
 480
 481         dev_dbg(dd->dev, "xmit_dma: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n",
 482                 ctx->digcnt[1], ctx->digcnt[0], length1, final);
 483
 484         dd->dma_lch_in.dma_conf.src_maxburst = 16;
 485         dd->dma_lch_in.dma_conf.dst_maxburst = 16;
 486
 487         dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf);
 488
 489         if (length2) {
 490                 sg_init_table(sg, 2);
 491                 sg_dma_address(&sg[0]) = dma_addr1;
 492                 sg_dma_len(&sg[0]) = length1;
 493                 sg_dma_address(&sg[1]) = dma_addr2;
 494                 sg_dma_len(&sg[1]) = length2;
 495                 in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, sg, 2,
 496                         DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 497         } else {
 498                 sg_init_table(sg, 1);
 499                 sg_dma_address(&sg[0]) = dma_addr1;
 500                 sg_dma_len(&sg[0]) = length1;
 501                 in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, sg, 1,
 502                         DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 503         }
 504         if (!in_desc)
 505                 return -EINVAL;
 506
 507         in_desc->callback = atmel_sha_dma_callback;
 508         in_desc->callback_param = dd;
 509
 510         atmel_sha_write_ctrl(dd, 1);
 511
 512         /* should be non-zero before next lines to disable clocks later */
 513         ctx->digcnt[0] += length1;
 514         if (ctx->digcnt[0] < length1)
 515                 ctx->digcnt[1]++;
 516
 517         if (final)
 518                 dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */
 519
 520         dd->flags |=  SHA_FLAGS_DMA_ACTIVE;
 521
 522         /* Start DMA transfer */
 523         dmaengine_submit(in_desc);
 524         dma_async_issue_pending(dd->dma_lch_in.chan);
 525
 526         return -EINPROGRESS;
 527 }
 528
 529 static int atmel_sha_xmit_start(struct atmel_sha_dev *dd, dma_addr_t dma_addr1,
 530                 size_t length1, dma_addr_t dma_addr2, size_t length2, int final)
 531 {
 532         if (dd->caps.has_dma)
 533                 return atmel_sha_xmit_dma(dd, dma_addr1, length1,
 534                                 dma_addr2, length2, final);
 535         else
 536                 return atmel_sha_xmit_pdc(dd, dma_addr1, length1,
 537                                 dma_addr2, length2, final);
 538 }
 539
 540 static int atmel_sha_update_cpu(struct atmel_sha_dev *dd)
 541 {
 542         struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
 543         int bufcnt;
 544
 545         atmel_sha_append_sg(ctx);
 546         atmel_sha_fill_padding(ctx, 0);
 547         bufcnt = ctx->bufcnt;
 548         ctx->bufcnt = 0;
 549
 550         return atmel_sha_xmit_cpu(dd, ctx->buffer, bufcnt, 1);
 551 }
 552
 553 static int atmel_sha_xmit_dma_map(struct atmel_sha_dev *dd,
 554                                         struct atmel_sha_reqctx *ctx,
 555                                         size_t length, int final)
 556 {
 557         ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer,
 558                                 ctx->buflen + ctx->block_size, DMA_TO_DEVICE);
 559         if (dma_mapping_error(dd->dev, ctx->dma_addr)) {
 560                 dev_err(dd->dev, "dma %u bytes error\n", ctx->buflen +
 561                                 ctx->block_size);
 562                 return -EINVAL;
 563         }
 564
 565         ctx->flags &= ~SHA_FLAGS_SG;
 566
 567         /* next call does not fail... so no unmap in the case of error */
 568         return atmel_sha_xmit_start(dd, ctx->dma_addr, length, 0, 0, final);
 569 }
 570
 571 static int atmel_sha_update_dma_slow(struct atmel_sha_dev *dd)
 572 {
 573         struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
 574         unsigned int final;
 575         size_t count;
 576
 577         atmel_sha_append_sg(ctx);
 578
 579         final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total;
 580
 581         dev_dbg(dd->dev, "slow: bufcnt: %u, digcnt: 0x%llx 0x%llx, final: %d\n",
 582                  ctx->bufcnt, ctx->digcnt[1], ctx->digcnt[0], final);
 583
 584         if (final)
 585                 atmel_sha_fill_padding(ctx, 0);
 586
 587         if (final || (ctx->bufcnt == ctx->buflen)) {
 588                 count = ctx->bufcnt;
 589                 ctx->bufcnt = 0;
 590                 return atmel_sha_xmit_dma_map(dd, ctx, count, final);
 591         }
 592
 593         return 0;
 594 }
 595
 596 static int atmel_sha_update_dma_start(struct atmel_sha_dev *dd)
 597 {
 598         struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
 599         unsigned int length, final, tail;
 600         struct scatterlist *sg;
 601         unsigned int count;
 602
 603         if (!ctx->total)
 604                 return 0;
 605
 606         if (ctx->bufcnt || ctx->offset)
 607                 return atmel_sha_update_dma_slow(dd);
 608
 609         dev_dbg(dd->dev, "fast: digcnt: 0x%llx 0x%llx, bufcnt: %u, total: %u\n",
 610                 ctx->digcnt[1], ctx->digcnt[0], ctx->bufcnt, ctx->total);
 611
 612         sg = ctx->sg;
 613
 614         if (!IS_ALIGNED(sg->offset, sizeof(u32)))
 615                 return atmel_sha_update_dma_slow(dd);
 616
 617         if (!sg_is_last(sg) && !IS_ALIGNED(sg->length, ctx->block_size))
 618                 /* size is not ctx->block_size aligned */
 619                 return atmel_sha_update_dma_slow(dd);
 620
 621         length = min(ctx->total, sg->length);
 622
 623         if (sg_is_last(sg)) {
 624                 if (!(ctx->flags & SHA_FLAGS_FINUP)) {
 625                         /* not last sg must be ctx->block_size aligned */
 626                         tail = length & (ctx->block_size - 1);
 627                         length -= tail;
 628                 }
 629         }
 630
 631         ctx->total -= length;
 632         ctx->offset = length; /* offset where to start slow */
 633
 634         final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total;
 635
 636         /* Add padding */
 637         if (final) {
 638                 tail = length & (ctx->block_size - 1);
 639                 length -= tail;
 640                 ctx->total += tail;
 641                 ctx->offset = length; /* offset where to start slow */
 642
 643                 sg = ctx->sg;
 644                 atmel_sha_append_sg(ctx);
 645
 646                 atmel_sha_fill_padding(ctx, length);
 647
 648                 ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer,
 649                         ctx->buflen + ctx->block_size, DMA_TO_DEVICE);
 650                 if (dma_mapping_error(dd->dev, ctx->dma_addr)) {
 651                         dev_err(dd->dev, "dma %u bytes error\n",
 652                                 ctx->buflen + ctx->block_size);
 653                         return -EINVAL;
 654                 }
 655
 656                 if (length == 0) {
 657                         ctx->flags &= ~SHA_FLAGS_SG;
 658                         count = ctx->bufcnt;
 659                         ctx->bufcnt = 0;
 660                         return atmel_sha_xmit_start(dd, ctx->dma_addr, count, 0,
 661                                         0, final);
 662                 } else {
 663                         ctx->sg = sg;
 664                         if (!dma_map_sg(dd->dev, ctx->sg, 1,
 665                                 DMA_TO_DEVICE)) {
 666                                         dev_err(dd->dev, "dma_map_sg  error\n");
 667                                         return -EINVAL;
 668                         }
 669
 670                         ctx->flags |= SHA_FLAGS_SG;
 671
 672                         count = ctx->bufcnt;
 673                         ctx->bufcnt = 0;
 674                         return atmel_sha_xmit_start(dd, sg_dma_address(ctx->sg),
 675                                         length, ctx->dma_addr, count, final);
 676                 }
 677         }
 678
 679         if (!dma_map_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE)) {
 680                 dev_err(dd->dev, "dma_map_sg  error\n");
 681                 return -EINVAL;
 682         }
 683
 684         ctx->flags |= SHA_FLAGS_SG;
 685
 686         /* next call does not fail... so no unmap in the case of error */
 687         return atmel_sha_xmit_start(dd, sg_dma_address(ctx->sg), length, 0,
 688                                                                 0, final);
 689 }
 690
 691 static int atmel_sha_update_dma_stop(struct atmel_sha_dev *dd)
 692 {
 693         struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
 694
 695         if (ctx->flags & SHA_FLAGS_SG) {
 696                 dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE);
 697                 if (ctx->sg->length == ctx->offset) {
 698                         ctx->sg = sg_next(ctx->sg);
 699                         if (ctx->sg)
 700                                 ctx->offset = 0;
 701                 }
 702                 if (ctx->flags & SHA_FLAGS_PAD) {
 703                         dma_unmap_single(dd->dev, ctx->dma_addr,
 704                                 ctx->buflen + ctx->block_size, DMA_TO_DEVICE);
 705                 }
 706         } else {
 707                 dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen +
 708                                                 ctx->block_size, DMA_TO_DEVICE);
 709         }
 710
 711         return 0;
 712 }
 713
 714 static int atmel_sha_update_req(struct atmel_sha_dev *dd)
 715 {
 716         struct ahash_request *req = dd->req;
 717         struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
 718         int err;
 719
 720         dev_dbg(dd->dev, "update_req: total: %u, digcnt: 0x%llx 0x%llx\n",
 721                 ctx->total, ctx->digcnt[1], ctx->digcnt[0]);
 722
 723         if (ctx->flags & SHA_FLAGS_CPU)
 724                 err = atmel_sha_update_cpu(dd);
 725         else
 726                 err = atmel_sha_update_dma_start(dd);
 727
 728         /* wait for dma completion before can take more data */
 729         dev_dbg(dd->dev, "update: err: %d, digcnt: 0x%llx 0%llx\n",
 730                         err, ctx->digcnt[1], ctx->digcnt[0]);
 731
 732         return err;
 733 }
 734
 735 static int atmel_sha_final_req(struct atmel_sha_dev *dd)
 736 {
 737         struct ahash_request *req = dd->req;
 738         struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
 739         int err = 0;
 740         int count;
 741
 742         if (ctx->bufcnt >= ATMEL_SHA_DMA_THRESHOLD) {
 743                 atmel_sha_fill_padding(ctx, 0);
 744                 count = ctx->bufcnt;
 745                 ctx->bufcnt = 0;
 746                 err = atmel_sha_xmit_dma_map(dd, ctx, count, 1);
 747         }
 748         /* faster to handle last block with cpu */
 749         else {
 750                 atmel_sha_fill_padding(ctx, 0);
 751                 count = ctx->bufcnt;
 752                 ctx->bufcnt = 0;
 753                 err = atmel_sha_xmit_cpu(dd, ctx->buffer, count, 1);
 754         }
 755
 756         dev_dbg(dd->dev, "final_req: err: %d\n", err);
 757
 758         return err;
 759 }
 760
 761 static void atmel_sha_copy_hash(struct ahash_request *req)
 762 {
 763         struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
 764         u32 *hash = (u32 *)ctx->digest;
 765         unsigned int i, hashsize;
 766
 767         switch (ctx->flags & SHA_FLAGS_ALGO_MASK) {
 768         case SHA_FLAGS_SHA1:
 769                 hashsize = SHA1_DIGEST_SIZE;
 770                 break;
 771
 772         case SHA_FLAGS_SHA224:
 773         case SHA_FLAGS_SHA256:
 774                 hashsize = SHA256_DIGEST_SIZE;
 775                 break;
 776
 777         case SHA_FLAGS_SHA384:
 778         case SHA_FLAGS_SHA512:
 779                 hashsize = SHA512_DIGEST_SIZE;
 780                 break;
 781
 782         default:
 783                 /* Should not happen... */
 784                 return;
 785         }
 786
 787         for (i = 0; i < hashsize / sizeof(u32); ++i)
 788                 hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i));
 789         ctx->flags |= SHA_FLAGS_RESTORE;
 790 }
 791
 792 static void atmel_sha_copy_ready_hash(struct ahash_request *req)
 793 {
 794         struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
 795
 796         if (!req->result)
 797                 return;
 798
 799         if (ctx->flags & SHA_FLAGS_SHA1)
 800                 memcpy(req->result, ctx->digest, SHA1_DIGEST_SIZE);
 801         else if (ctx->flags & SHA_FLAGS_SHA224)
 802                 memcpy(req->result, ctx->digest, SHA224_DIGEST_SIZE);
 803         else if (ctx->flags & SHA_FLAGS_SHA256)
 804                 memcpy(req->result, ctx->digest, SHA256_DIGEST_SIZE);
 805         else if (ctx->flags & SHA_FLAGS_SHA384)
 806                 memcpy(req->result, ctx->digest, SHA384_DIGEST_SIZE);
 807         else
 808                 memcpy(req->result, ctx->digest, SHA512_DIGEST_SIZE);
 809 }
 810
 811 static int atmel_sha_finish(struct ahash_request *req)
 812 {
 813         struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
 814         struct atmel_sha_dev *dd = ctx->dd;
 815
 816         if (ctx->digcnt[0] || ctx->digcnt[1])
 817                 atmel_sha_copy_ready_hash(req);
 818
 819         dev_dbg(dd->dev, "digcnt: 0x%llx 0x%llx, bufcnt: %d\n", ctx->digcnt[1],
 820                 ctx->digcnt[0], ctx->bufcnt);
 821
 822         return 0;
 823 }
 824
 825 static void atmel_sha_finish_req(struct ahash_request *req, int err)
 826 {
 827         struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
 828         struct atmel_sha_dev *dd = ctx->dd;
 829
 830         if (!err) {
 831                 atmel_sha_copy_hash(req);
 832                 if (SHA_FLAGS_FINAL & dd->flags)
 833                         err = atmel_sha_finish(req);
 834         } else {
 835                 ctx->flags |= SHA_FLAGS_ERROR;
 836         }
 837
 838         /* atomic operation is not needed here */
 839         dd->flags &= ~(SHA_FLAGS_BUSY | SHA_FLAGS_FINAL | SHA_FLAGS_CPU |
 840                         SHA_FLAGS_DMA_READY | SHA_FLAGS_OUTPUT_READY);
 841
 842         clk_disable_unprepare(dd->iclk);
 843
 844         if (req->base.complete)
 845                 req->base.complete(&req->base, err);
 846
 847         /* handle new request */
 848         tasklet_schedule(&dd->queue_task);
 849 }
 850
 851 static int atmel_sha_hw_init(struct atmel_sha_dev *dd)
 852 {
 853         int err;
 854
 855         err = clk_prepare_enable(dd->iclk);
 856         if (err)
 857                 return err;
 858
 859         if (!(SHA_FLAGS_INIT & dd->flags)) {
 860                 atmel_sha_write(dd, SHA_CR, SHA_CR_SWRST);
 861                 dd->flags |= SHA_FLAGS_INIT;
 862                 dd->err = 0;
 863         }
 864
 865         return 0;
 866 }
 867
 868 static inline unsigned int atmel_sha_get_version(struct atmel_sha_dev *dd)
 869 {
 870         return atmel_sha_read(dd, SHA_HW_VERSION) & 0x00000fff;
 871 }
 872
 873 static void atmel_sha_hw_version_init(struct atmel_sha_dev *dd)
 874 {
 875         atmel_sha_hw_init(dd);
 876
 877         dd->hw_version = atmel_sha_get_version(dd);
 878
 879         dev_info(dd->dev,
 880                         "version: 0x%x\n", dd->hw_version);
 881
 882         clk_disable_unprepare(dd->iclk);
 883 }
 884
 885 static int atmel_sha_handle_queue(struct atmel_sha_dev *dd,
 886                                   struct ahash_request *req)
 887 {
 888         struct crypto_async_request *async_req, *backlog;
 889         struct atmel_sha_reqctx *ctx;
 890         unsigned long flags;
 891         int err = 0, ret = 0;
 892
 893         spin_lock_irqsave(&dd->lock, flags);
 894         if (req)
 895                 ret = ahash_enqueue_request(&dd->queue, req);
 896
 897         if (SHA_FLAGS_BUSY & dd->flags) {
 898                 spin_unlock_irqrestore(&dd->lock, flags);
 899                 return ret;
 900         }
 901
 902         backlog = crypto_get_backlog(&dd->queue);
 903         async_req = crypto_dequeue_request(&dd->queue);
 904         if (async_req)
 905                 dd->flags |= SHA_FLAGS_BUSY;
 906
 907         spin_unlock_irqrestore(&dd->lock, flags);
 908
 909         if (!async_req)
 910                 return ret;
 911
 912         if (backlog)
 913                 backlog->complete(backlog, -EINPROGRESS);
 914
 915         req = ahash_request_cast(async_req);
 916         dd->req = req;
 917         ctx = ahash_request_ctx(req);
 918
 919         dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n",
 920                                                 ctx->op, req->nbytes);
 921
 922         err = atmel_sha_hw_init(dd);
 923
 924         if (err)
 925                 goto err1;
 926
 927         if (ctx->op == SHA_OP_UPDATE) {
 928                 err = atmel_sha_update_req(dd);
 929                 if (err != -EINPROGRESS && (ctx->flags & SHA_FLAGS_FINUP))
 930                         /* no final() after finup() */
 931                         err = atmel_sha_final_req(dd);
 932         } else if (ctx->op == SHA_OP_FINAL) {
 933                 err = atmel_sha_final_req(dd);
 934         }
 935
 936 err1:
 937         if (err != -EINPROGRESS)
 938                 /* done_task will not finish it, so do it here */
 939                 atmel_sha_finish_req(req, err);
 940
 941         dev_dbg(dd->dev, "exit, err: %d\n", err);
 942
 943         return ret;
 944 }
 945
 946 static int atmel_sha_enqueue(struct ahash_request *req, unsigned int op)
 947 {
 948         struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
 949         struct atmel_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
 950         struct atmel_sha_dev *dd = tctx->dd;
 951
 952         ctx->op = op;
 953
 954         return atmel_sha_handle_queue(dd, req);
 955 }
 956
 957 static int atmel_sha_update(struct ahash_request *req)
 958 {
 959         struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
 960
 961         if (!req->nbytes)
 962                 return 0;
 963
 964         ctx->total = req->nbytes;
 965         ctx->sg = req->src;
 966         ctx->offset = 0;
 967
 968         if (ctx->flags & SHA_FLAGS_FINUP) {
 969                 if (ctx->bufcnt + ctx->total < ATMEL_SHA_DMA_THRESHOLD)
 970                         /* faster to use CPU for short transfers */
 971                         ctx->flags |= SHA_FLAGS_CPU;
 972         } else if (ctx->bufcnt + ctx->total < ctx->buflen) {
 973                 atmel_sha_append_sg(ctx);
 974                 return 0;
 975         }
 976         return atmel_sha_enqueue(req, SHA_OP_UPDATE);
 977 }
 978
 979 static int atmel_sha_final(struct ahash_request *req)
 980 {
 981         struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
 982         struct atmel_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
 983         struct atmel_sha_dev *dd = tctx->dd;
 984
 985         int err = 0;
 986
 987         ctx->flags |= SHA_FLAGS_FINUP;
 988
 989         if (ctx->flags & SHA_FLAGS_ERROR)
 990                 return 0; /* uncompleted hash is not needed */
 991
 992         if (ctx->bufcnt) {
 993                 return atmel_sha_enqueue(req, SHA_OP_FINAL);
 994         } else if (!(ctx->flags & SHA_FLAGS_PAD)) { /* add padding */
 995                 err = atmel_sha_hw_init(dd);
 996                 if (err)
 997                         goto err1;
 998
 999                 dd->req = req;
1000                 dd->flags |= SHA_FLAGS_BUSY;
1001                 err = atmel_sha_final_req(dd);
1002         } else {
1003                 /* copy ready hash (+ finalize hmac) */
1004                 return atmel_sha_finish(req);
1005         }
1006
1007 err1:
1008         if (err != -EINPROGRESS)
1009                 /* done_task will not finish it, so do it here */
1010                 atmel_sha_finish_req(req, err);
1011
1012         return err;
1013 }
1014
1015 static int atmel_sha_finup(struct ahash_request *req)
1016 {
1017         struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
1018         int err1, err2;
1019
1020         ctx->flags |= SHA_FLAGS_FINUP;
1021
1022         err1 = atmel_sha_update(req);
1023         if (err1 == -EINPROGRESS || err1 == -EBUSY)
1024                 return err1;
1025
1026         /*
1027          * final() has to be always called to cleanup resources
1028          * even if udpate() failed, except EINPROGRESS
1029          */
1030         err2 = atmel_sha_final(req);
1031
1032         return err1 ?: err2;
1033 }
1034
1035 static int atmel_sha_digest(struct ahash_request *req)
1036 {
1037         return atmel_sha_init(req) ?: atmel_sha_finup(req);
1038 }
1039
1040
1041 static int atmel_sha_export(struct ahash_request *req, void *out)
1042 {
1043         const struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
1044
1045         memcpy(out, ctx, sizeof(*ctx));
1046         return 0;
1047 }
1048
1049 static int atmel_sha_import(struct ahash_request *req, const void *in)
1050 {
1051         struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
1052
1053         memcpy(ctx, in, sizeof(*ctx));
1054         return 0;
1055 }
1056
1057 static int atmel_sha_cra_init(struct crypto_tfm *tfm)
1058 {
1059         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1060                                  sizeof(struct atmel_sha_reqctx));
1061
1062         return 0;
1063 }
1064
1065 static struct ahash_alg sha_1_256_algs[] = {
1066 {
1067         .init           = atmel_sha_init,
1068         .update         = atmel_sha_update,
1069         .final          = atmel_sha_final,
1070         .finup          = atmel_sha_finup,
1071         .digest         = atmel_sha_digest,
1072         .export         = atmel_sha_export,
1073         .import         = atmel_sha_import,
1074         .halg = {
1075                 .digestsize     = SHA1_DIGEST_SIZE,
1076                 .statesize      = sizeof(struct atmel_sha_reqctx),
1077                 .base   = {
1078                         .cra_name               = "sha1",
1079                         .cra_driver_name        = "atmel-sha1",
1080                         .cra_priority           = 100,
1081                         .cra_flags              = CRYPTO_ALG_ASYNC,
1082                         .cra_blocksize          = SHA1_BLOCK_SIZE,
1083                         .cra_ctxsize            = sizeof(struct atmel_sha_ctx),
1084                         .cra_alignmask          = 0,
1085                         .cra_module             = THIS_MODULE,
1086                         .cra_init               = atmel_sha_cra_init,
1087                 }
1088         }
1089 },
1090 {
1091         .init           = atmel_sha_init,
1092         .update         = atmel_sha_update,
1093         .final          = atmel_sha_final,
1094         .finup          = atmel_sha_finup,
1095         .digest         = atmel_sha_digest,
1096         .export         = atmel_sha_export,
1097         .import         = atmel_sha_import,
1098         .halg = {
1099                 .digestsize     = SHA256_DIGEST_SIZE,
1100                 .statesize      = sizeof(struct atmel_sha_reqctx),
1101                 .base   = {
1102                         .cra_name               = "sha256",
1103                         .cra_driver_name        = "atmel-sha256",
1104                         .cra_priority           = 100,
1105                         .cra_flags              = CRYPTO_ALG_ASYNC,
1106                         .cra_blocksize          = SHA256_BLOCK_SIZE,
1107                         .cra_ctxsize            = sizeof(struct atmel_sha_ctx),
1108                         .cra_alignmask          = 0,
1109                         .cra_module             = THIS_MODULE,
1110                         .cra_init               = atmel_sha_cra_init,
1111                 }
1112         }
1113 },
1114 };
1115
1116 static struct ahash_alg sha_224_alg = {
1117         .init           = atmel_sha_init,
1118         .update         = atmel_sha_update,
1119         .final          = atmel_sha_final,
1120         .finup          = atmel_sha_finup,
1121         .digest         = atmel_sha_digest,
1122         .export         = atmel_sha_export,
1123         .import         = atmel_sha_import,
1124         .halg = {
1125                 .digestsize     = SHA224_DIGEST_SIZE,
1126                 .statesize      = sizeof(struct atmel_sha_reqctx),
1127                 .base   = {
1128                         .cra_name               = "sha224",
1129                         .cra_driver_name        = "atmel-sha224",
1130                         .cra_priority           = 100,
1131                         .cra_flags              = CRYPTO_ALG_ASYNC,
1132                         .cra_blocksize          = SHA224_BLOCK_SIZE,
1133                         .cra_ctxsize            = sizeof(struct atmel_sha_ctx),
1134                         .cra_alignmask          = 0,
1135                         .cra_module             = THIS_MODULE,
1136                         .cra_init               = atmel_sha_cra_init,
1137                 }
1138         }
1139 };
1140
1141 static struct ahash_alg sha_384_512_algs[] = {
1142 {
1143         .init           = atmel_sha_init,
1144         .update         = atmel_sha_update,
1145         .final          = atmel_sha_final,
1146         .finup          = atmel_sha_finup,
1147         .digest         = atmel_sha_digest,
1148         .export         = atmel_sha_export,
1149         .import         = atmel_sha_import,
1150         .halg = {
1151                 .digestsize     = SHA384_DIGEST_SIZE,
1152                 .statesize      = sizeof(struct atmel_sha_reqctx),
1153                 .base   = {
1154                         .cra_name               = "sha384",
1155                         .cra_driver_name        = "atmel-sha384",
1156                         .cra_priority           = 100,
1157                         .cra_flags              = CRYPTO_ALG_ASYNC,
1158                         .cra_blocksize          = SHA384_BLOCK_SIZE,
1159                         .cra_ctxsize            = sizeof(struct atmel_sha_ctx),
1160                         .cra_alignmask          = 0x3,
1161                         .cra_module             = THIS_MODULE,
1162                         .cra_init               = atmel_sha_cra_init,
1163                 }
1164         }
1165 },
1166 {
1167         .init           = atmel_sha_init,
1168         .update         = atmel_sha_update,
1169         .final          = atmel_sha_final,
1170         .finup          = atmel_sha_finup,
1171         .digest         = atmel_sha_digest,
1172         .export         = atmel_sha_export,
1173         .import         = atmel_sha_import,
1174         .halg = {
1175                 .digestsize     = SHA512_DIGEST_SIZE,
1176                 .statesize      = sizeof(struct atmel_sha_reqctx),
1177                 .base   = {
1178                         .cra_name               = "sha512",
1179                         .cra_driver_name        = "atmel-sha512",
1180                         .cra_priority           = 100,
1181                         .cra_flags              = CRYPTO_ALG_ASYNC,
1182                         .cra_blocksize          = SHA512_BLOCK_SIZE,
1183                         .cra_ctxsize            = sizeof(struct atmel_sha_ctx),
1184                         .cra_alignmask          = 0x3,
1185                         .cra_module             = THIS_MODULE,
1186                         .cra_init               = atmel_sha_cra_init,
1187                 }
1188         }
1189 },
1190 };
1191
1192 static void atmel_sha_queue_task(unsigned long data)
1193 {
1194         struct atmel_sha_dev *dd = (struct atmel_sha_dev *)data;
1195
1196         atmel_sha_handle_queue(dd, NULL);
1197 }
1198
1199 static void atmel_sha_done_task(unsigned long data)
1200 {
1201         struct atmel_sha_dev *dd = (struct atmel_sha_dev *)data;
1202         int err = 0;
1203
1204         if (SHA_FLAGS_CPU & dd->flags) {
1205                 if (SHA_FLAGS_OUTPUT_READY & dd->flags) {
1206                         dd->flags &= ~SHA_FLAGS_OUTPUT_READY;
1207                         goto finish;
1208                 }
1209         } else if (SHA_FLAGS_DMA_READY & dd->flags) {
1210                 if (SHA_FLAGS_DMA_ACTIVE & dd->flags) {
1211                         dd->flags &= ~SHA_FLAGS_DMA_ACTIVE;
1212                         atmel_sha_update_dma_stop(dd);
1213                         if (dd->err) {
1214                                 err = dd->err;
1215                                 goto finish;
1216                         }
1217                 }
1218                 if (SHA_FLAGS_OUTPUT_READY & dd->flags) {
1219                         /* hash or semi-hash ready */
1220                         dd->flags &= ~(SHA_FLAGS_DMA_READY |
1221                                                 SHA_FLAGS_OUTPUT_READY);
1222                         err = atmel_sha_update_dma_start(dd);
1223                         if (err != -EINPROGRESS)
1224                                 goto finish;
1225                 }
1226         }
1227         return;
1228
1229 finish:
1230         /* finish curent request */
1231         atmel_sha_finish_req(dd->req, err);
1232 }
1233
1234 static irqreturn_t atmel_sha_irq(int irq, void *dev_id)
1235 {
1236         struct atmel_sha_dev *sha_dd = dev_id;
1237         u32 reg;
1238
1239         reg = atmel_sha_read(sha_dd, SHA_ISR);
1240         if (reg & atmel_sha_read(sha_dd, SHA_IMR)) {
1241                 atmel_sha_write(sha_dd, SHA_IDR, reg);
1242                 if (SHA_FLAGS_BUSY & sha_dd->flags) {
1243                         sha_dd->flags |= SHA_FLAGS_OUTPUT_READY;
1244                         if (!(SHA_FLAGS_CPU & sha_dd->flags))
1245                                 sha_dd->flags |= SHA_FLAGS_DMA_READY;
1246                         tasklet_schedule(&sha_dd->done_task);
1247                 } else {
1248                         dev_warn(sha_dd->dev, "SHA interrupt when no active requests.\n");
1249                 }
1250                 return IRQ_HANDLED;
1251         }
1252
1253         return IRQ_NONE;
1254 }
1255
1256 static void atmel_sha_unregister_algs(struct atmel_sha_dev *dd)
1257 {
1258         int i;
1259
1260         for (i = 0; i < ARRAY_SIZE(sha_1_256_algs); i++)
1261                 crypto_unregister_ahash(&sha_1_256_algs[i]);
1262
1263         if (dd->caps.has_sha224)
1264                 crypto_unregister_ahash(&sha_224_alg);
1265
1266         if (dd->caps.has_sha_384_512) {
1267                 for (i = 0; i < ARRAY_SIZE(sha_384_512_algs); i++)
1268                         crypto_unregister_ahash(&sha_384_512_algs[i]);
1269         }
1270 }
1271
1272 static int atmel_sha_register_algs(struct atmel_sha_dev *dd)
1273 {
1274         int err, i, j;
1275
1276         for (i = 0; i < ARRAY_SIZE(sha_1_256_algs); i++) {
1277                 err = crypto_register_ahash(&sha_1_256_algs[i]);
1278                 if (err)
1279                         goto err_sha_1_256_algs;
1280         }
1281
1282         if (dd->caps.has_sha224) {
1283                 err = crypto_register_ahash(&sha_224_alg);
1284                 if (err)
1285                         goto err_sha_224_algs;
1286         }
1287
1288         if (dd->caps.has_sha_384_512) {
1289                 for (i = 0; i < ARRAY_SIZE(sha_384_512_algs); i++) {
1290                         err = crypto_register_ahash(&sha_384_512_algs[i]);
1291                         if (err)
1292                                 goto err_sha_384_512_algs;
1293                 }
1294         }
1295
1296         return 0;
1297
1298 err_sha_384_512_algs:
1299         for (j = 0; j < i; j++)
1300                 crypto_unregister_ahash(&sha_384_512_algs[j]);
1301         crypto_unregister_ahash(&sha_224_alg);
1302 err_sha_224_algs:
1303         i = ARRAY_SIZE(sha_1_256_algs);
1304 err_sha_1_256_algs:
1305         for (j = 0; j < i; j++)
1306                 crypto_unregister_ahash(&sha_1_256_algs[j]);
1307
1308         return err;
1309 }
1310
1311 static bool atmel_sha_filter(struct dma_chan *chan, void *slave)
1312 {
1313         struct at_dma_slave     *sl = slave;
1314
1315         if (sl && sl->dma_dev == chan->device->dev) {
1316                 chan->private = sl;
1317                 return true;
1318         } else {
1319                 return false;
1320         }
1321 }
1322
1323 static int atmel_sha_dma_init(struct atmel_sha_dev *dd,
1324                                 struct crypto_platform_data *pdata)
1325 {
1326         int err = -ENOMEM;
1327         dma_cap_mask_t mask_in;
1328
1329         /* Try to grab DMA channel */
1330         dma_cap_zero(mask_in);
1331         dma_cap_set(DMA_SLAVE, mask_in);
1332
1333         dd->dma_lch_in.chan = dma_request_slave_channel_compat(mask_in,
1334                         atmel_sha_filter, &pdata->dma_slave->rxdata, dd->dev, "tx");
1335         if (!dd->dma_lch_in.chan) {
1336                 dev_warn(dd->dev, "no DMA channel available\n");
1337                 return err;
1338         }
1339
1340         dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV;
1341         dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base +
1342                 SHA_REG_DIN(0);
1343         dd->dma_lch_in.dma_conf.src_maxburst = 1;
1344         dd->dma_lch_in.dma_conf.src_addr_width =
1345                 DMA_SLAVE_BUSWIDTH_4_BYTES;
1346         dd->dma_lch_in.dma_conf.dst_maxburst = 1;
1347         dd->dma_lch_in.dma_conf.dst_addr_width =
1348                 DMA_SLAVE_BUSWIDTH_4_BYTES;
1349         dd->dma_lch_in.dma_conf.device_fc = false;
1350
1351         return 0;
1352 }
1353
1354 static void atmel_sha_dma_cleanup(struct atmel_sha_dev *dd)
1355 {
1356         dma_release_channel(dd->dma_lch_in.chan);
1357 }
1358
1359 static void atmel_sha_get_cap(struct atmel_sha_dev *dd)
1360 {
1361
1362         dd->caps.has_dma = 0;
1363         dd->caps.has_dualbuff = 0;
1364         dd->caps.has_sha224 = 0;
1365         dd->caps.has_sha_384_512 = 0;
1366         dd->caps.has_uihv = 0;
1367
1368         /* keep only major version number */
1369         switch (dd->hw_version & 0xff0) {
1370         case 0x510:
1371                 dd->caps.has_dma = 1;
1372                 dd->caps.has_dualbuff = 1;
1373                 dd->caps.has_sha224 = 1;
1374                 dd->caps.has_sha_384_512 = 1;
1375                 dd->caps.has_uihv = 1;
1376                 break;
1377         case 0x420:
1378                 dd->caps.has_dma = 1;
1379                 dd->caps.has_dualbuff = 1;
1380                 dd->caps.has_sha224 = 1;
1381                 dd->caps.has_sha_384_512 = 1;
1382                 dd->caps.has_uihv = 1;
1383                 break;
1384         case 0x410:
1385                 dd->caps.has_dma = 1;
1386                 dd->caps.has_dualbuff = 1;
1387                 dd->caps.has_sha224 = 1;
1388                 dd->caps.has_sha_384_512 = 1;
1389                 break;
1390         case 0x400:
1391                 dd->caps.has_dma = 1;
1392                 dd->caps.has_dualbuff = 1;
1393                 dd->caps.has_sha224 = 1;
1394                 break;
1395         case 0x320:
1396                 break;
1397         default:
1398                 dev_warn(dd->dev,
1399                                 "Unmanaged sha version, set minimum capabilities\n");
1400                 break;
1401         }
1402 }
1403
1404 #if defined(CONFIG_OF)
1405 static const struct of_device_id atmel_sha_dt_ids[] = {
1406         { .compatible = "atmel,at91sam9g46-sha" },
1407         { /* sentinel */ }
1408 };
1409
1410 MODULE_DEVICE_TABLE(of, atmel_sha_dt_ids);
1411
1412 static struct crypto_platform_data *atmel_sha_of_init(struct platform_device *pdev)
1413 {
1414         struct device_node *np = pdev->dev.of_node;
1415         struct crypto_platform_data *pdata;
1416
1417         if (!np) {
1418                 dev_err(&pdev->dev, "device node not found\n");
1419                 return ERR_PTR(-EINVAL);
1420         }
1421
1422         pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
1423         if (!pdata) {
1424                 dev_err(&pdev->dev, "could not allocate memory for pdata\n");
1425                 return ERR_PTR(-ENOMEM);
1426         }
1427
1428         pdata->dma_slave = devm_kzalloc(&pdev->dev,
1429                                         sizeof(*(pdata->dma_slave)),
1430                                         GFP_KERNEL);
1431         if (!pdata->dma_slave) {
1432                 dev_err(&pdev->dev, "could not allocate memory for dma_slave\n");
1433                 return ERR_PTR(-ENOMEM);
1434         }
1435
1436         return pdata;
1437 }
1438 #else /* CONFIG_OF */
1439 static inline struct crypto_platform_data *atmel_sha_of_init(struct platform_device *dev)
1440 {
1441         return ERR_PTR(-EINVAL);
1442 }
1443 #endif
1444
1445 static int atmel_sha_probe(struct platform_device *pdev)
1446 {
1447         struct atmel_sha_dev *sha_dd;
1448         struct crypto_platform_data     *pdata;
1449         struct device *dev = &pdev->dev;
1450         struct resource *sha_res;
1451         int err;
1452
1453         sha_dd = devm_kzalloc(&pdev->dev, sizeof(*sha_dd), GFP_KERNEL);
1454         if (sha_dd == NULL) {
1455                 dev_err(dev, "unable to alloc data struct.\n");
1456                 err = -ENOMEM;
1457                 goto sha_dd_err;
1458         }
1459
1460         sha_dd->dev = dev;
1461
1462         platform_set_drvdata(pdev, sha_dd);
1463
1464         INIT_LIST_HEAD(&sha_dd->list);
1465         spin_lock_init(&sha_dd->lock);
1466
1467         tasklet_init(&sha_dd->done_task, atmel_sha_done_task,
1468                                         (unsigned long)sha_dd);
1469         tasklet_init(&sha_dd->queue_task, atmel_sha_queue_task,
1470                                         (unsigned long)sha_dd);
1471
1472         crypto_init_queue(&sha_dd->queue, ATMEL_SHA_QUEUE_LENGTH);
1473
1474         sha_dd->irq = -1;
1475
1476         /* Get the base address */
1477         sha_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1478         if (!sha_res) {
1479                 dev_err(dev, "no MEM resource info\n");
1480                 err = -ENODEV;
1481                 goto res_err;
1482         }
1483         sha_dd->phys_base = sha_res->start;
1484
1485         /* Get the IRQ */
1486         sha_dd->irq = platform_get_irq(pdev,  0);
1487         if (sha_dd->irq < 0) {
1488                 dev_err(dev, "no IRQ resource info\n");
1489                 err = sha_dd->irq;
1490                 goto res_err;
1491         }
1492
1493         err = devm_request_irq(&pdev->dev, sha_dd->irq, atmel_sha_irq,
1494                                IRQF_SHARED, "atmel-sha", sha_dd);
1495         if (err) {
1496                 dev_err(dev, "unable to request sha irq.\n");
1497                 goto res_err;
1498         }
1499
1500         /* Initializing the clock */
1501         sha_dd->iclk = devm_clk_get(&pdev->dev, "sha_clk");
1502         if (IS_ERR(sha_dd->iclk)) {
1503                 dev_err(dev, "clock initialization failed.\n");
1504                 err = PTR_ERR(sha_dd->iclk);
1505                 goto res_err;
1506         }
1507
1508         sha_dd->io_base = devm_ioremap_resource(&pdev->dev, sha_res);
1509         if (!sha_dd->io_base) {
1510                 dev_err(dev, "can't ioremap\n");
1511                 err = -ENOMEM;
1512                 goto res_err;
1513         }
1514
1515         atmel_sha_hw_version_init(sha_dd);
1516
1517         atmel_sha_get_cap(sha_dd);
1518
1519         if (sha_dd->caps.has_dma) {
1520                 pdata = pdev->dev.platform_data;
1521                 if (!pdata) {
1522                         pdata = atmel_sha_of_init(pdev);
1523                         if (IS_ERR(pdata)) {
1524                                 dev_err(&pdev->dev, "platform data not available\n");
1525                                 err = PTR_ERR(pdata);
1526                                 goto res_err;
1527                         }
1528                 }
1529                 if (!pdata->dma_slave) {
1530                         err = -ENXIO;
1531                         goto res_err;
1532                 }
1533                 err = atmel_sha_dma_init(sha_dd, pdata);
1534                 if (err)
1535                         goto err_sha_dma;
1536
1537                 dev_info(dev, "using %s for DMA transfers\n",
1538                                 dma_chan_name(sha_dd->dma_lch_in.chan));
1539         }
1540
1541         spin_lock(&atmel_sha.lock);
1542         list_add_tail(&sha_dd->list, &atmel_sha.dev_list);
1543         spin_unlock(&atmel_sha.lock);
1544
1545         err = atmel_sha_register_algs(sha_dd);
1546         if (err)
1547                 goto err_algs;
1548
1549         dev_info(dev, "Atmel SHA1/SHA256%s%s\n",
1550                         sha_dd->caps.has_sha224 ? "/SHA224" : "",
1551                         sha_dd->caps.has_sha_384_512 ? "/SHA384/SHA512" : "");
1552
1553         return 0;
1554
1555 err_algs:
1556         spin_lock(&atmel_sha.lock);
1557         list_del(&sha_dd->list);
1558         spin_unlock(&atmel_sha.lock);
1559         if (sha_dd->caps.has_dma)
1560                 atmel_sha_dma_cleanup(sha_dd);
1561 err_sha_dma:
1562 res_err:
1563         tasklet_kill(&sha_dd->queue_task);
1564         tasklet_kill(&sha_dd->done_task);
1565 sha_dd_err:
1566         dev_err(dev, "initialization failed.\n");
1567
1568         return err;
1569 }
1570
1571 static int atmel_sha_remove(struct platform_device *pdev)
1572 {
1573         static struct atmel_sha_dev *sha_dd;
1574
1575         sha_dd = platform_get_drvdata(pdev);
1576         if (!sha_dd)
1577                 return -ENODEV;
1578         spin_lock(&atmel_sha.lock);
1579         list_del(&sha_dd->list);
1580         spin_unlock(&atmel_sha.lock);
1581
1582         atmel_sha_unregister_algs(sha_dd);
1583
1584         tasklet_kill(&sha_dd->queue_task);
1585         tasklet_kill(&sha_dd->done_task);
1586
1587         if (sha_dd->caps.has_dma)
1588                 atmel_sha_dma_cleanup(sha_dd);
1589
1590         iounmap(sha_dd->io_base);
1591
1592         clk_put(sha_dd->iclk);
1593
1594         if (sha_dd->irq >= 0)
1595                 free_irq(sha_dd->irq, sha_dd);
1596
1597         return 0;
1598 }
1599
1600 static struct platform_driver atmel_sha_driver = {
1601         .probe          = atmel_sha_probe,
1602         .remove         = atmel_sha_remove,
1603         .driver         = {
1604                 .name   = "atmel_sha",
1605                 .of_match_table = of_match_ptr(atmel_sha_dt_ids),
1606         },
1607 };
1608
1609 module_platform_driver(atmel_sha_driver);
1610
1611 MODULE_DESCRIPTION("Atmel SHA (1/256/224/384/512) hw acceleration support.");
1612 MODULE_LICENSE("GPL v2");
1613 MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique");