drivers/crypto/omap-des.c

   1 /*
   2  * Support for OMAP DES and Triple DES HW acceleration.
   3  *
   4  * Copyright (c) 2013 Texas Instruments Incorporated
   5  * Author: Joel Fernandes <joelf@ti.com>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as published
   9  * by the Free Software Foundation.
  10  *
  11  */
  12
  13 #define pr_fmt(fmt) "%s: " fmt, __func__
  14
  15 #ifdef DEBUG
  16 #define prn(num) printk(#num "=%d\n", num)
  17 #define prx(num) printk(#num "=%x\n", num)
  18 #else
  19 #define prn(num) do { } while (0)
  20 #define prx(num)  do { } while (0)
  21 #endif
  22
  23 #include <linux/err.h>
  24 #include <linux/module.h>
  25 #include <linux/init.h>
  26 #include <linux/errno.h>
  27 #include <linux/kernel.h>
  28 #include <linux/platform_device.h>
  29 #include <linux/scatterlist.h>
  30 #include <linux/dma-mapping.h>
  31 #include <linux/dmaengine.h>
  32 #include <linux/pm_runtime.h>
  33 #include <linux/of.h>
  34 #include <linux/of_device.h>
  35 #include <linux/of_address.h>
  36 #include <linux/io.h>
  37 #include <linux/crypto.h>
  38 #include <linux/interrupt.h>
  39 #include <crypto/scatterwalk.h>
  40 #include <crypto/des.h>
  41 #include <crypto/algapi.h>
  42 #include <crypto/engine.h>
  43
  44 #define DST_MAXBURST                    2
  45
  46 #define DES_BLOCK_WORDS         (DES_BLOCK_SIZE >> 2)
  47
  48 #define _calc_walked(inout) (dd->inout##_walk.offset - dd->inout##_sg->offset)
  49
  50 #define DES_REG_KEY(dd, x)              ((dd)->pdata->key_ofs - \
  51                                                 ((x ^ 0x01) * 0x04))
  52
  53 #define DES_REG_IV(dd, x)               ((dd)->pdata->iv_ofs + ((x) * 0x04))
  54
  55 #define DES_REG_CTRL(dd)                ((dd)->pdata->ctrl_ofs)
  56 #define DES_REG_CTRL_CBC                BIT(4)
  57 #define DES_REG_CTRL_TDES               BIT(3)
  58 #define DES_REG_CTRL_DIRECTION          BIT(2)
  59 #define DES_REG_CTRL_INPUT_READY        BIT(1)
  60 #define DES_REG_CTRL_OUTPUT_READY       BIT(0)
  61
  62 #define DES_REG_DATA_N(dd, x)           ((dd)->pdata->data_ofs + ((x) * 0x04))
  63
  64 #define DES_REG_REV(dd)                 ((dd)->pdata->rev_ofs)
  65
  66 #define DES_REG_MASK(dd)                ((dd)->pdata->mask_ofs)
  67
  68 #define DES_REG_LENGTH_N(x)             (0x24 + ((x) * 0x04))
  69
  70 #define DES_REG_IRQ_STATUS(dd)         ((dd)->pdata->irq_status_ofs)
  71 #define DES_REG_IRQ_ENABLE(dd)         ((dd)->pdata->irq_enable_ofs)
  72 #define DES_REG_IRQ_DATA_IN            BIT(1)
  73 #define DES_REG_IRQ_DATA_OUT           BIT(2)
  74
  75 #define FLAGS_MODE_MASK         0x000f
  76 #define FLAGS_ENCRYPT           BIT(0)
  77 #define FLAGS_CBC               BIT(1)
  78 #define FLAGS_INIT              BIT(4)
  79 #define FLAGS_BUSY              BIT(6)
  80
  81 struct omap_des_ctx {
  82         struct omap_des_dev *dd;
  83
  84         int             keylen;
  85         u32             key[(3 * DES_KEY_SIZE) / sizeof(u32)];
  86         unsigned long   flags;
  87 };
  88
  89 struct omap_des_reqctx {
  90         unsigned long mode;
  91 };
  92
  93 #define OMAP_DES_QUEUE_LENGTH   1
  94 #define OMAP_DES_CACHE_SIZE     0
  95
  96 struct omap_des_algs_info {
  97         struct crypto_alg       *algs_list;
  98         unsigned int            size;
  99         unsigned int            registered;
 100 };
 101
 102 struct omap_des_pdata {
 103         struct omap_des_algs_info       *algs_info;
 104         unsigned int    algs_info_size;
 105
 106         void            (*trigger)(struct omap_des_dev *dd, int length);
 107
 108         u32             key_ofs;
 109         u32             iv_ofs;
 110         u32             ctrl_ofs;
 111         u32             data_ofs;
 112         u32             rev_ofs;
 113         u32             mask_ofs;
 114         u32             irq_enable_ofs;
 115         u32             irq_status_ofs;
 116
 117         u32             dma_enable_in;
 118         u32             dma_enable_out;
 119         u32             dma_start;
 120
 121         u32             major_mask;
 122         u32             major_shift;
 123         u32             minor_mask;
 124         u32             minor_shift;
 125 };
 126
 127 struct omap_des_dev {
 128         struct list_head        list;
 129         unsigned long           phys_base;
 130         void __iomem            *io_base;
 131         struct omap_des_ctx     *ctx;
 132         struct device           *dev;
 133         unsigned long           flags;
 134         int                     err;
 135
 136         struct tasklet_struct   done_task;
 137
 138         struct ablkcipher_request       *req;
 139         struct crypto_engine            *engine;
 140         /*
 141          * total is used by PIO mode for book keeping so introduce
 142          * variable total_save as need it to calc page_order
 143          */
 144         size_t                          total;
 145         size_t                          total_save;
 146
 147         struct scatterlist              *in_sg;
 148         struct scatterlist              *out_sg;
 149
 150         /* Buffers for copying for unaligned cases */
 151         struct scatterlist              in_sgl;
 152         struct scatterlist              out_sgl;
 153         struct scatterlist              *orig_out;
 154         int                             sgs_copied;
 155
 156         struct scatter_walk             in_walk;
 157         struct scatter_walk             out_walk;
 158         struct dma_chan         *dma_lch_in;
 159         struct dma_chan         *dma_lch_out;
 160         int                     in_sg_len;
 161         int                     out_sg_len;
 162         int                     pio_only;
 163         const struct omap_des_pdata     *pdata;
 164 };
 165
 166 /* keep registered devices data here */
 167 static LIST_HEAD(dev_list);
 168 static DEFINE_SPINLOCK(list_lock);
 169
 170 #ifdef DEBUG
 171 #define omap_des_read(dd, offset)                               \
 172         ({                                                              \
 173          int _read_ret;                                          \
 174          _read_ret = __raw_readl(dd->io_base + offset);          \
 175          pr_err("omap_des_read(" #offset "=%#x)= %#x\n",       \
 176                  offset, _read_ret);                            \
 177          _read_ret;                                              \
 178          })
 179 #else
 180 static inline u32 omap_des_read(struct omap_des_dev *dd, u32 offset)
 181 {
 182         return __raw_readl(dd->io_base + offset);
 183 }
 184 #endif
 185
 186 #ifdef DEBUG
 187 #define omap_des_write(dd, offset, value)                               \
 188         do {                                                            \
 189                 pr_err("omap_des_write(" #offset "=%#x) value=%#x\n", \
 190                                 offset, value);                                \
 191                 __raw_writel(value, dd->io_base + offset);              \
 192         } while (0)
 193 #else
 194 static inline void omap_des_write(struct omap_des_dev *dd, u32 offset,
 195                 u32 value)
 196 {
 197         __raw_writel(value, dd->io_base + offset);
 198 }
 199 #endif
 200
 201 static inline void omap_des_write_mask(struct omap_des_dev *dd, u32 offset,
 202                                         u32 value, u32 mask)
 203 {
 204         u32 val;
 205
 206         val = omap_des_read(dd, offset);
 207         val &= ~mask;
 208         val |= value;
 209         omap_des_write(dd, offset, val);
 210 }
 211
 212 static void omap_des_write_n(struct omap_des_dev *dd, u32 offset,
 213                                         u32 *value, int count)
 214 {
 215         for (; count--; value++, offset += 4)
 216                 omap_des_write(dd, offset, *value);
 217 }
 218
 219 static int omap_des_hw_init(struct omap_des_dev *dd)
 220 {
 221         int err;
 222
 223         /*
 224          * clocks are enabled when request starts and disabled when finished.
 225          * It may be long delays between requests.
 226          * Device might go to off mode to save power.
 227          */
 228         err = pm_runtime_get_sync(dd->dev);
 229         if (err < 0) {
 230                 pm_runtime_put_noidle(dd->dev);
 231                 dev_err(dd->dev, "%s: failed to get_sync(%d)\n", __func__, err);
 232                 return err;
 233         }
 234
 235         if (!(dd->flags & FLAGS_INIT)) {
 236                 dd->flags |= FLAGS_INIT;
 237                 dd->err = 0;
 238         }
 239
 240         return 0;
 241 }
 242
 243 static int omap_des_write_ctrl(struct omap_des_dev *dd)
 244 {
 245         unsigned int key32;
 246         int i, err;
 247         u32 val = 0, mask = 0;
 248
 249         err = omap_des_hw_init(dd);
 250         if (err)
 251                 return err;
 252
 253         key32 = dd->ctx->keylen / sizeof(u32);
 254
 255         /* it seems a key should always be set even if it has not changed */
 256         for (i = 0; i < key32; i++) {
 257                 omap_des_write(dd, DES_REG_KEY(dd, i),
 258                                __le32_to_cpu(dd->ctx->key[i]));
 259         }
 260
 261         if ((dd->flags & FLAGS_CBC) && dd->req->info)
 262                 omap_des_write_n(dd, DES_REG_IV(dd, 0), dd->req->info, 2);
 263
 264         if (dd->flags & FLAGS_CBC)
 265                 val |= DES_REG_CTRL_CBC;
 266         if (dd->flags & FLAGS_ENCRYPT)
 267                 val |= DES_REG_CTRL_DIRECTION;
 268         if (key32 == 6)
 269                 val |= DES_REG_CTRL_TDES;
 270
 271         mask |= DES_REG_CTRL_CBC | DES_REG_CTRL_DIRECTION | DES_REG_CTRL_TDES;
 272
 273         omap_des_write_mask(dd, DES_REG_CTRL(dd), val, mask);
 274
 275         return 0;
 276 }
 277
 278 static void omap_des_dma_trigger_omap4(struct omap_des_dev *dd, int length)
 279 {
 280         u32 mask, val;
 281
 282         omap_des_write(dd, DES_REG_LENGTH_N(0), length);
 283
 284         val = dd->pdata->dma_start;
 285
 286         if (dd->dma_lch_out != NULL)
 287                 val |= dd->pdata->dma_enable_out;
 288         if (dd->dma_lch_in != NULL)
 289                 val |= dd->pdata->dma_enable_in;
 290
 291         mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in |
 292                dd->pdata->dma_start;
 293
 294         omap_des_write_mask(dd, DES_REG_MASK(dd), val, mask);
 295 }
 296
 297 static void omap_des_dma_stop(struct omap_des_dev *dd)
 298 {
 299         u32 mask;
 300
 301         mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in |
 302                dd->pdata->dma_start;
 303
 304         omap_des_write_mask(dd, DES_REG_MASK(dd), 0, mask);
 305 }
 306
 307 static struct omap_des_dev *omap_des_find_dev(struct omap_des_ctx *ctx)
 308 {
 309         struct omap_des_dev *dd = NULL, *tmp;
 310
 311         spin_lock_bh(&list_lock);
 312         if (!ctx->dd) {
 313                 list_for_each_entry(tmp, &dev_list, list) {
 314                         /* FIXME: take fist available des core */
 315                         dd = tmp;
 316                         break;
 317                 }
 318                 ctx->dd = dd;
 319         } else {
 320                 /* already found before */
 321                 dd = ctx->dd;
 322         }
 323         spin_unlock_bh(&list_lock);
 324
 325         return dd;
 326 }
 327
 328 static void omap_des_dma_out_callback(void *data)
 329 {
 330         struct omap_des_dev *dd = data;
 331
 332         /* dma_lch_out - completed */
 333         tasklet_schedule(&dd->done_task);
 334 }
 335
 336 static int omap_des_dma_init(struct omap_des_dev *dd)
 337 {
 338         int err;
 339
 340         dd->dma_lch_out = NULL;
 341         dd->dma_lch_in = NULL;
 342
 343         dd->dma_lch_in = dma_request_chan(dd->dev, "rx");
 344         if (IS_ERR(dd->dma_lch_in)) {
 345                 dev_err(dd->dev, "Unable to request in DMA channel\n");
 346                 return PTR_ERR(dd->dma_lch_in);
 347         }
 348
 349         dd->dma_lch_out = dma_request_chan(dd->dev, "tx");
 350         if (IS_ERR(dd->dma_lch_out)) {
 351                 dev_err(dd->dev, "Unable to request out DMA channel\n");
 352                 err = PTR_ERR(dd->dma_lch_out);
 353                 goto err_dma_out;
 354         }
 355
 356         return 0;
 357
 358 err_dma_out:
 359         dma_release_channel(dd->dma_lch_in);
 360
 361         return err;
 362 }
 363
 364 static void omap_des_dma_cleanup(struct omap_des_dev *dd)
 365 {
 366         if (dd->pio_only)
 367                 return;
 368
 369         dma_release_channel(dd->dma_lch_out);
 370         dma_release_channel(dd->dma_lch_in);
 371 }
 372
 373 static void sg_copy_buf(void *buf, struct scatterlist *sg,
 374                               unsigned int start, unsigned int nbytes, int out)
 375 {
 376         struct scatter_walk walk;
 377
 378         if (!nbytes)
 379                 return;
 380
 381         scatterwalk_start(&walk, sg);
 382         scatterwalk_advance(&walk, start);
 383         scatterwalk_copychunks(buf, &walk, nbytes, out);
 384         scatterwalk_done(&walk, out, 0);
 385 }
 386
 387 static int omap_des_crypt_dma(struct crypto_tfm *tfm,
 388                 struct scatterlist *in_sg, struct scatterlist *out_sg,
 389                 int in_sg_len, int out_sg_len)
 390 {
 391         struct omap_des_ctx *ctx = crypto_tfm_ctx(tfm);
 392         struct omap_des_dev *dd = ctx->dd;
 393         struct dma_async_tx_descriptor *tx_in, *tx_out;
 394         struct dma_slave_config cfg;
 395         int ret;
 396
 397         if (dd->pio_only) {
 398                 scatterwalk_start(&dd->in_walk, dd->in_sg);
 399                 scatterwalk_start(&dd->out_walk, dd->out_sg);
 400
 401                 /* Enable DATAIN interrupt and let it take
 402                    care of the rest */
 403                 omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x2);
 404                 return 0;
 405         }
 406
 407         dma_sync_sg_for_device(dd->dev, dd->in_sg, in_sg_len, DMA_TO_DEVICE);
 408
 409         memset(&cfg, 0, sizeof(cfg));
 410
 411         cfg.src_addr = dd->phys_base + DES_REG_DATA_N(dd, 0);
 412         cfg.dst_addr = dd->phys_base + DES_REG_DATA_N(dd, 0);
 413         cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 414         cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 415         cfg.src_maxburst = DST_MAXBURST;
 416         cfg.dst_maxburst = DST_MAXBURST;
 417
 418         /* IN */
 419         ret = dmaengine_slave_config(dd->dma_lch_in, &cfg);
 420         if (ret) {
 421                 dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
 422                         ret);
 423                 return ret;
 424         }
 425
 426         tx_in = dmaengine_prep_slave_sg(dd->dma_lch_in, in_sg, in_sg_len,
 427                                         DMA_MEM_TO_DEV,
 428                                         DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 429         if (!tx_in) {
 430                 dev_err(dd->dev, "IN prep_slave_sg() failed\n");
 431                 return -EINVAL;
 432         }
 433
 434         /* No callback necessary */
 435         tx_in->callback_param = dd;
 436
 437         /* OUT */
 438         ret = dmaengine_slave_config(dd->dma_lch_out, &cfg);
 439         if (ret) {
 440                 dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n",
 441                         ret);
 442                 return ret;
 443         }
 444
 445         tx_out = dmaengine_prep_slave_sg(dd->dma_lch_out, out_sg, out_sg_len,
 446                                         DMA_DEV_TO_MEM,
 447                                         DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 448         if (!tx_out) {
 449                 dev_err(dd->dev, "OUT prep_slave_sg() failed\n");
 450                 return -EINVAL;
 451         }
 452
 453         tx_out->callback = omap_des_dma_out_callback;
 454         tx_out->callback_param = dd;
 455
 456         dmaengine_submit(tx_in);
 457         dmaengine_submit(tx_out);
 458
 459         dma_async_issue_pending(dd->dma_lch_in);
 460         dma_async_issue_pending(dd->dma_lch_out);
 461
 462         /* start DMA */
 463         dd->pdata->trigger(dd, dd->total);
 464
 465         return 0;
 466 }
 467
 468 static int omap_des_crypt_dma_start(struct omap_des_dev *dd)
 469 {
 470         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(
 471                                         crypto_ablkcipher_reqtfm(dd->req));
 472         int err;
 473
 474         pr_debug("total: %d\n", dd->total);
 475
 476         if (!dd->pio_only) {
 477                 err = dma_map_sg(dd->dev, dd->in_sg, dd->in_sg_len,
 478                                  DMA_TO_DEVICE);
 479                 if (!err) {
 480                         dev_err(dd->dev, "dma_map_sg() error\n");
 481                         return -EINVAL;
 482                 }
 483
 484                 err = dma_map_sg(dd->dev, dd->out_sg, dd->out_sg_len,
 485                                  DMA_FROM_DEVICE);
 486                 if (!err) {
 487                         dev_err(dd->dev, "dma_map_sg() error\n");
 488                         return -EINVAL;
 489                 }
 490         }
 491
 492         err = omap_des_crypt_dma(tfm, dd->in_sg, dd->out_sg, dd->in_sg_len,
 493                                  dd->out_sg_len);
 494         if (err && !dd->pio_only) {
 495                 dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
 496                 dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len,
 497                              DMA_FROM_DEVICE);
 498         }
 499
 500         return err;
 501 }
 502
 503 static void omap_des_finish_req(struct omap_des_dev *dd, int err)
 504 {
 505         struct ablkcipher_request *req = dd->req;
 506
 507         pr_debug("err: %d\n", err);
 508
 509         pm_runtime_put(dd->dev);
 510         crypto_finalize_cipher_request(dd->engine, req, err);
 511 }
 512
 513 static int omap_des_crypt_dma_stop(struct omap_des_dev *dd)
 514 {
 515         pr_debug("total: %d\n", dd->total);
 516
 517         omap_des_dma_stop(dd);
 518
 519         dmaengine_terminate_all(dd->dma_lch_in);
 520         dmaengine_terminate_all(dd->dma_lch_out);
 521
 522         return 0;
 523 }
 524
 525 static int omap_des_copy_needed(struct scatterlist *sg)
 526 {
 527         while (sg) {
 528                 if (!IS_ALIGNED(sg->offset, 4))
 529                         return -1;
 530                 if (!IS_ALIGNED(sg->length, DES_BLOCK_SIZE))
 531                         return -1;
 532                 sg = sg_next(sg);
 533         }
 534         return 0;
 535 }
 536
 537 static int omap_des_copy_sgs(struct omap_des_dev *dd)
 538 {
 539         void *buf_in, *buf_out;
 540         int pages;
 541
 542         pages = dd->total >> PAGE_SHIFT;
 543
 544         if (dd->total & (PAGE_SIZE-1))
 545                 pages++;
 546
 547         BUG_ON(!pages);
 548
 549         buf_in = (void *)__get_free_pages(GFP_ATOMIC, pages);
 550         buf_out = (void *)__get_free_pages(GFP_ATOMIC, pages);
 551
 552         if (!buf_in || !buf_out) {
 553                 pr_err("Couldn't allocated pages for unaligned cases.\n");
 554                 return -1;
 555         }
 556
 557         dd->orig_out = dd->out_sg;
 558
 559         sg_copy_buf(buf_in, dd->in_sg, 0, dd->total, 0);
 560
 561         sg_init_table(&dd->in_sgl, 1);
 562         sg_set_buf(&dd->in_sgl, buf_in, dd->total);
 563         dd->in_sg = &dd->in_sgl;
 564         dd->in_sg_len = 1;
 565
 566         sg_init_table(&dd->out_sgl, 1);
 567         sg_set_buf(&dd->out_sgl, buf_out, dd->total);
 568         dd->out_sg = &dd->out_sgl;
 569         dd->out_sg_len = 1;
 570
 571         return 0;
 572 }
 573
 574 static int omap_des_handle_queue(struct omap_des_dev *dd,
 575                                  struct ablkcipher_request *req)
 576 {
 577         if (req)
 578                 return crypto_transfer_cipher_request_to_engine(dd->engine, req);
 579
 580         return 0;
 581 }
 582
 583 static int omap_des_prepare_req(struct crypto_engine *engine,
 584                                 struct ablkcipher_request *req)
 585 {
 586         struct omap_des_ctx *ctx = crypto_ablkcipher_ctx(
 587                         crypto_ablkcipher_reqtfm(req));
 588         struct omap_des_dev *dd = omap_des_find_dev(ctx);
 589         struct omap_des_reqctx *rctx;
 590
 591         if (!dd)
 592                 return -ENODEV;
 593
 594         /* assign new request to device */
 595         dd->req = req;
 596         dd->total = req->nbytes;
 597         dd->total_save = req->nbytes;
 598         dd->in_sg = req->src;
 599         dd->out_sg = req->dst;
 600
 601         dd->in_sg_len = sg_nents_for_len(dd->in_sg, dd->total);
 602         if (dd->in_sg_len < 0)
 603                 return dd->in_sg_len;
 604
 605         dd->out_sg_len = sg_nents_for_len(dd->out_sg, dd->total);
 606         if (dd->out_sg_len < 0)
 607                 return dd->out_sg_len;
 608
 609         if (omap_des_copy_needed(dd->in_sg) ||
 610             omap_des_copy_needed(dd->out_sg)) {
 611                 if (omap_des_copy_sgs(dd))
 612                         pr_err("Failed to copy SGs for unaligned cases\n");
 613                 dd->sgs_copied = 1;
 614         } else {
 615                 dd->sgs_copied = 0;
 616         }
 617
 618         rctx = ablkcipher_request_ctx(req);
 619         ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
 620         rctx->mode &= FLAGS_MODE_MASK;
 621         dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
 622
 623         dd->ctx = ctx;
 624         ctx->dd = dd;
 625
 626         return omap_des_write_ctrl(dd);
 627 }
 628
 629 static int omap_des_crypt_req(struct crypto_engine *engine,
 630                               struct ablkcipher_request *req)
 631 {
 632         struct omap_des_ctx *ctx = crypto_ablkcipher_ctx(
 633                         crypto_ablkcipher_reqtfm(req));
 634         struct omap_des_dev *dd = omap_des_find_dev(ctx);
 635
 636         if (!dd)
 637                 return -ENODEV;
 638
 639         return omap_des_crypt_dma_start(dd);
 640 }
 641
 642 static void omap_des_done_task(unsigned long data)
 643 {
 644         struct omap_des_dev *dd = (struct omap_des_dev *)data;
 645         void *buf_in, *buf_out;
 646         int pages;
 647
 648         pr_debug("enter done_task\n");
 649
 650         if (!dd->pio_only) {
 651                 dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,
 652                                        DMA_FROM_DEVICE);
 653                 dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
 654                 dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len,
 655                              DMA_FROM_DEVICE);
 656                 omap_des_crypt_dma_stop(dd);
 657         }
 658
 659         if (dd->sgs_copied) {
 660                 buf_in = sg_virt(&dd->in_sgl);
 661                 buf_out = sg_virt(&dd->out_sgl);
 662
 663                 sg_copy_buf(buf_out, dd->orig_out, 0, dd->total_save, 1);
 664
 665                 pages = get_order(dd->total_save);
 666                 free_pages((unsigned long)buf_in, pages);
 667                 free_pages((unsigned long)buf_out, pages);
 668         }
 669
 670         omap_des_finish_req(dd, 0);
 671
 672         pr_debug("exit\n");
 673 }
 674
 675 static int omap_des_crypt(struct ablkcipher_request *req, unsigned long mode)
 676 {
 677         struct omap_des_ctx *ctx = crypto_ablkcipher_ctx(
 678                         crypto_ablkcipher_reqtfm(req));
 679         struct omap_des_reqctx *rctx = ablkcipher_request_ctx(req);
 680         struct omap_des_dev *dd;
 681
 682         pr_debug("nbytes: %d, enc: %d, cbc: %d\n", req->nbytes,
 683                  !!(mode & FLAGS_ENCRYPT),
 684                  !!(mode & FLAGS_CBC));
 685
 686         if (!IS_ALIGNED(req->nbytes, DES_BLOCK_SIZE)) {
 687                 pr_err("request size is not exact amount of DES blocks\n");
 688                 return -EINVAL;
 689         }
 690
 691         dd = omap_des_find_dev(ctx);
 692         if (!dd)
 693                 return -ENODEV;
 694
 695         rctx->mode = mode;
 696
 697         return omap_des_handle_queue(dd, req);
 698 }
 699
 700 /* ********************** ALG API ************************************ */
 701
 702 static int omap_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 703                            unsigned int keylen)
 704 {
 705         struct omap_des_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 706
 707         if (keylen != DES_KEY_SIZE && keylen != (3*DES_KEY_SIZE))
 708                 return -EINVAL;
 709
 710         pr_debug("enter, keylen: %d\n", keylen);
 711
 712         memcpy(ctx->key, key, keylen);
 713         ctx->keylen = keylen;
 714
 715         return 0;
 716 }
 717
 718 static int omap_des_ecb_encrypt(struct ablkcipher_request *req)
 719 {
 720         return omap_des_crypt(req, FLAGS_ENCRYPT);
 721 }
 722
 723 static int omap_des_ecb_decrypt(struct ablkcipher_request *req)
 724 {
 725         return omap_des_crypt(req, 0);
 726 }
 727
 728 static int omap_des_cbc_encrypt(struct ablkcipher_request *req)
 729 {
 730         return omap_des_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC);
 731 }
 732
 733 static int omap_des_cbc_decrypt(struct ablkcipher_request *req)
 734 {
 735         return omap_des_crypt(req, FLAGS_CBC);
 736 }
 737
 738 static int omap_des_cra_init(struct crypto_tfm *tfm)
 739 {
 740         pr_debug("enter\n");
 741
 742         tfm->crt_ablkcipher.reqsize = sizeof(struct omap_des_reqctx);
 743
 744         return 0;
 745 }
 746
 747 static void omap_des_cra_exit(struct crypto_tfm *tfm)
 748 {
 749         pr_debug("enter\n");
 750 }
 751
 752 /* ********************** ALGS ************************************ */
 753
 754 static struct crypto_alg algs_ecb_cbc[] = {
 755 {
 756         .cra_name               = "ecb(des)",
 757         .cra_driver_name        = "ecb-des-omap",
 758         .cra_priority           = 100,
 759         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
 760                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
 761                                   CRYPTO_ALG_ASYNC,
 762         .cra_blocksize          = DES_BLOCK_SIZE,
 763         .cra_ctxsize            = sizeof(struct omap_des_ctx),
 764         .cra_alignmask          = 0,
 765         .cra_type               = &crypto_ablkcipher_type,
 766         .cra_module             = THIS_MODULE,
 767         .cra_init               = omap_des_cra_init,
 768         .cra_exit               = omap_des_cra_exit,
 769         .cra_u.ablkcipher = {
 770                 .min_keysize    = DES_KEY_SIZE,
 771                 .max_keysize    = DES_KEY_SIZE,
 772                 .setkey         = omap_des_setkey,
 773                 .encrypt        = omap_des_ecb_encrypt,
 774                 .decrypt        = omap_des_ecb_decrypt,
 775         }
 776 },
 777 {
 778         .cra_name               = "cbc(des)",
 779         .cra_driver_name        = "cbc-des-omap",
 780         .cra_priority           = 100,
 781         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
 782                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
 783                                   CRYPTO_ALG_ASYNC,
 784         .cra_blocksize          = DES_BLOCK_SIZE,
 785         .cra_ctxsize            = sizeof(struct omap_des_ctx),
 786         .cra_alignmask          = 0,
 787         .cra_type               = &crypto_ablkcipher_type,
 788         .cra_module             = THIS_MODULE,
 789         .cra_init               = omap_des_cra_init,
 790         .cra_exit               = omap_des_cra_exit,
 791         .cra_u.ablkcipher = {
 792                 .min_keysize    = DES_KEY_SIZE,
 793                 .max_keysize    = DES_KEY_SIZE,
 794                 .ivsize         = DES_BLOCK_SIZE,
 795                 .setkey         = omap_des_setkey,
 796                 .encrypt        = omap_des_cbc_encrypt,
 797                 .decrypt        = omap_des_cbc_decrypt,
 798         }
 799 },
 800 {
 801         .cra_name               = "ecb(des3_ede)",
 802         .cra_driver_name        = "ecb-des3-omap",
 803         .cra_priority           = 100,
 804         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
 805                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
 806                                   CRYPTO_ALG_ASYNC,
 807         .cra_blocksize          = DES_BLOCK_SIZE,
 808         .cra_ctxsize            = sizeof(struct omap_des_ctx),
 809         .cra_alignmask          = 0,
 810         .cra_type               = &crypto_ablkcipher_type,
 811         .cra_module             = THIS_MODULE,
 812         .cra_init               = omap_des_cra_init,
 813         .cra_exit               = omap_des_cra_exit,
 814         .cra_u.ablkcipher = {
 815                 .min_keysize    = 3*DES_KEY_SIZE,
 816                 .max_keysize    = 3*DES_KEY_SIZE,
 817                 .setkey         = omap_des_setkey,
 818                 .encrypt        = omap_des_ecb_encrypt,
 819                 .decrypt        = omap_des_ecb_decrypt,
 820         }
 821 },
 822 {
 823         .cra_name               = "cbc(des3_ede)",
 824         .cra_driver_name        = "cbc-des3-omap",
 825         .cra_priority           = 100,
 826         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
 827                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
 828                                   CRYPTO_ALG_ASYNC,
 829         .cra_blocksize          = DES_BLOCK_SIZE,
 830         .cra_ctxsize            = sizeof(struct omap_des_ctx),
 831         .cra_alignmask          = 0,
 832         .cra_type               = &crypto_ablkcipher_type,
 833         .cra_module             = THIS_MODULE,
 834         .cra_init               = omap_des_cra_init,
 835         .cra_exit               = omap_des_cra_exit,
 836         .cra_u.ablkcipher = {
 837                 .min_keysize    = 3*DES_KEY_SIZE,
 838                 .max_keysize    = 3*DES_KEY_SIZE,
 839                 .ivsize         = DES_BLOCK_SIZE,
 840                 .setkey         = omap_des_setkey,
 841                 .encrypt        = omap_des_cbc_encrypt,
 842                 .decrypt        = omap_des_cbc_decrypt,
 843         }
 844 }
 845 };
 846
 847 static struct omap_des_algs_info omap_des_algs_info_ecb_cbc[] = {
 848         {
 849                 .algs_list      = algs_ecb_cbc,
 850                 .size           = ARRAY_SIZE(algs_ecb_cbc),
 851         },
 852 };
 853
 854 #ifdef CONFIG_OF
 855 static const struct omap_des_pdata omap_des_pdata_omap4 = {
 856         .algs_info      = omap_des_algs_info_ecb_cbc,
 857         .algs_info_size = ARRAY_SIZE(omap_des_algs_info_ecb_cbc),
 858         .trigger        = omap_des_dma_trigger_omap4,
 859         .key_ofs        = 0x14,
 860         .iv_ofs         = 0x18,
 861         .ctrl_ofs       = 0x20,
 862         .data_ofs       = 0x28,
 863         .rev_ofs        = 0x30,
 864         .mask_ofs       = 0x34,
 865         .irq_status_ofs = 0x3c,
 866         .irq_enable_ofs = 0x40,
 867         .dma_enable_in  = BIT(5),
 868         .dma_enable_out = BIT(6),
 869         .major_mask     = 0x0700,
 870         .major_shift    = 8,
 871         .minor_mask     = 0x003f,
 872         .minor_shift    = 0,
 873 };
 874
 875 static irqreturn_t omap_des_irq(int irq, void *dev_id)
 876 {
 877         struct omap_des_dev *dd = dev_id;
 878         u32 status, i;
 879         u32 *src, *dst;
 880
 881         status = omap_des_read(dd, DES_REG_IRQ_STATUS(dd));
 882         if (status & DES_REG_IRQ_DATA_IN) {
 883                 omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x0);
 884
 885                 BUG_ON(!dd->in_sg);
 886
 887                 BUG_ON(_calc_walked(in) > dd->in_sg->length);
 888
 889                 src = sg_virt(dd->in_sg) + _calc_walked(in);
 890
 891                 for (i = 0; i < DES_BLOCK_WORDS; i++) {
 892                         omap_des_write(dd, DES_REG_DATA_N(dd, i), *src);
 893
 894                         scatterwalk_advance(&dd->in_walk, 4);
 895                         if (dd->in_sg->length == _calc_walked(in)) {
 896                                 dd->in_sg = sg_next(dd->in_sg);
 897                                 if (dd->in_sg) {
 898                                         scatterwalk_start(&dd->in_walk,
 899                                                           dd->in_sg);
 900                                         src = sg_virt(dd->in_sg) +
 901                                               _calc_walked(in);
 902                                 }
 903                         } else {
 904                                 src++;
 905                         }
 906                 }
 907
 908                 /* Clear IRQ status */
 909                 status &= ~DES_REG_IRQ_DATA_IN;
 910                 omap_des_write(dd, DES_REG_IRQ_STATUS(dd), status);
 911
 912                 /* Enable DATA_OUT interrupt */
 913                 omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x4);
 914
 915         } else if (status & DES_REG_IRQ_DATA_OUT) {
 916                 omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x0);
 917
 918                 BUG_ON(!dd->out_sg);
 919
 920                 BUG_ON(_calc_walked(out) > dd->out_sg->length);
 921
 922                 dst = sg_virt(dd->out_sg) + _calc_walked(out);
 923
 924                 for (i = 0; i < DES_BLOCK_WORDS; i++) {
 925                         *dst = omap_des_read(dd, DES_REG_DATA_N(dd, i));
 926                         scatterwalk_advance(&dd->out_walk, 4);
 927                         if (dd->out_sg->length == _calc_walked(out)) {
 928                                 dd->out_sg = sg_next(dd->out_sg);
 929                                 if (dd->out_sg) {
 930                                         scatterwalk_start(&dd->out_walk,
 931                                                           dd->out_sg);
 932                                         dst = sg_virt(dd->out_sg) +
 933                                               _calc_walked(out);
 934                                 }
 935                         } else {
 936                                 dst++;
 937                         }
 938                 }
 939
 940                 BUG_ON(dd->total < DES_BLOCK_SIZE);
 941
 942                 dd->total -= DES_BLOCK_SIZE;
 943
 944                 /* Clear IRQ status */
 945                 status &= ~DES_REG_IRQ_DATA_OUT;
 946                 omap_des_write(dd, DES_REG_IRQ_STATUS(dd), status);
 947
 948                 if (!dd->total)
 949                         /* All bytes read! */
 950                         tasklet_schedule(&dd->done_task);
 951                 else
 952                         /* Enable DATA_IN interrupt for next block */
 953                         omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x2);
 954         }
 955
 956         return IRQ_HANDLED;
 957 }
 958
 959 static const struct of_device_id omap_des_of_match[] = {
 960         {
 961                 .compatible     = "ti,omap4-des",
 962                 .data           = &omap_des_pdata_omap4,
 963         },
 964         {},
 965 };
 966 MODULE_DEVICE_TABLE(of, omap_des_of_match);
 967
 968 static int omap_des_get_of(struct omap_des_dev *dd,
 969                 struct platform_device *pdev)
 970 {
 971         const struct of_device_id *match;
 972
 973         match = of_match_device(of_match_ptr(omap_des_of_match), &pdev->dev);
 974         if (!match) {
 975                 dev_err(&pdev->dev, "no compatible OF match\n");
 976                 return -EINVAL;
 977         }
 978
 979         dd->pdata = match->data;
 980
 981         return 0;
 982 }
 983 #else
 984 static int omap_des_get_of(struct omap_des_dev *dd,
 985                 struct device *dev)
 986 {
 987         return -EINVAL;
 988 }
 989 #endif
 990
 991 static int omap_des_get_pdev(struct omap_des_dev *dd,
 992                 struct platform_device *pdev)
 993 {
 994         /* non-DT devices get pdata from pdev */
 995         dd->pdata = pdev->dev.platform_data;
 996
 997         return 0;
 998 }
 999
1000 static int omap_des_probe(struct platform_device *pdev)
1001 {
1002         struct device *dev = &pdev->dev;
1003         struct omap_des_dev *dd;
1004         struct crypto_alg *algp;
1005         struct resource *res;
1006         int err = -ENOMEM, i, j, irq = -1;
1007         u32 reg;
1008
1009         dd = devm_kzalloc(dev, sizeof(struct omap_des_dev), GFP_KERNEL);
1010         if (dd == NULL) {
1011                 dev_err(dev, "unable to alloc data struct.\n");
1012                 goto err_data;
1013         }
1014         dd->dev = dev;
1015         platform_set_drvdata(pdev, dd);
1016
1017         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1018         if (!res) {
1019                 dev_err(dev, "no MEM resource info\n");
1020                 goto err_res;
1021         }
1022
1023         err = (dev->of_node) ? omap_des_get_of(dd, pdev) :
1024                                omap_des_get_pdev(dd, pdev);
1025         if (err)
1026                 goto err_res;
1027
1028         dd->io_base = devm_ioremap_resource(dev, res);
1029         if (IS_ERR(dd->io_base)) {
1030                 err = PTR_ERR(dd->io_base);
1031                 goto err_res;
1032         }
1033         dd->phys_base = res->start;
1034
1035         pm_runtime_enable(dev);
1036         pm_runtime_irq_safe(dev);
1037         err = pm_runtime_get_sync(dev);
1038         if (err < 0) {
1039                 pm_runtime_put_noidle(dev);
1040                 dev_err(dd->dev, "%s: failed to get_sync(%d)\n", __func__, err);
1041                 goto err_get;
1042         }
1043
1044         omap_des_dma_stop(dd);
1045
1046         reg = omap_des_read(dd, DES_REG_REV(dd));
1047
1048         pm_runtime_put_sync(dev);
1049
1050         dev_info(dev, "OMAP DES hw accel rev: %u.%u\n",
1051                  (reg & dd->pdata->major_mask) >> dd->pdata->major_shift,
1052                  (reg & dd->pdata->minor_mask) >> dd->pdata->minor_shift);
1053
1054         tasklet_init(&dd->done_task, omap_des_done_task, (unsigned long)dd);
1055
1056         err = omap_des_dma_init(dd);
1057         if (err == -EPROBE_DEFER) {
1058                 goto err_irq;
1059         } else if (err && DES_REG_IRQ_STATUS(dd) && DES_REG_IRQ_ENABLE(dd)) {
1060                 dd->pio_only = 1;
1061
1062                 irq = platform_get_irq(pdev, 0);
1063                 if (irq < 0) {
1064                         dev_err(dev, "can't get IRQ resource\n");
1065                         goto err_irq;
1066                 }
1067
1068                 err = devm_request_irq(dev, irq, omap_des_irq, 0,
1069                                 dev_name(dev), dd);
1070                 if (err) {
1071                         dev_err(dev, "Unable to grab omap-des IRQ\n");
1072                         goto err_irq;
1073                 }
1074         }
1075
1076
1077         INIT_LIST_HEAD(&dd->list);
1078         spin_lock(&list_lock);
1079         list_add_tail(&dd->list, &dev_list);
1080         spin_unlock(&list_lock);
1081
1082         /* Initialize des crypto engine */
1083         dd->engine = crypto_engine_alloc_init(dev, 1);
1084         if (!dd->engine) {
1085                 err = -ENOMEM;
1086                 goto err_engine;
1087         }
1088
1089         dd->engine->prepare_cipher_request = omap_des_prepare_req;
1090         dd->engine->cipher_one_request = omap_des_crypt_req;
1091         err = crypto_engine_start(dd->engine);
1092         if (err)
1093                 goto err_engine;
1094
1095         for (i = 0; i < dd->pdata->algs_info_size; i++) {
1096                 for (j = 0; j < dd->pdata->algs_info[i].size; j++) {
1097                         algp = &dd->pdata->algs_info[i].algs_list[j];
1098
1099                         pr_debug("reg alg: %s\n", algp->cra_name);
1100                         INIT_LIST_HEAD(&algp->cra_list);
1101
1102                         err = crypto_register_alg(algp);
1103                         if (err)
1104                                 goto err_algs;
1105
1106                         dd->pdata->algs_info[i].registered++;
1107                 }
1108         }
1109
1110         return 0;
1111
1112 err_algs:
1113         for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
1114                 for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
1115                         crypto_unregister_alg(
1116                                         &dd->pdata->algs_info[i].algs_list[j]);
1117
1118 err_engine:
1119         if (dd->engine)
1120                 crypto_engine_exit(dd->engine);
1121
1122         omap_des_dma_cleanup(dd);
1123 err_irq:
1124         tasklet_kill(&dd->done_task);
1125 err_get:
1126         pm_runtime_disable(dev);
1127 err_res:
1128         dd = NULL;
1129 err_data:
1130         dev_err(dev, "initialization failed.\n");
1131         return err;
1132 }
1133
1134 static int omap_des_remove(struct platform_device *pdev)
1135 {
1136         struct omap_des_dev *dd = platform_get_drvdata(pdev);
1137         int i, j;
1138
1139         if (!dd)
1140                 return -ENODEV;
1141
1142         spin_lock(&list_lock);
1143         list_del(&dd->list);
1144         spin_unlock(&list_lock);
1145
1146         for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
1147                 for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
1148                         crypto_unregister_alg(
1149                                         &dd->pdata->algs_info[i].algs_list[j]);
1150
1151         tasklet_kill(&dd->done_task);
1152         omap_des_dma_cleanup(dd);
1153         pm_runtime_disable(dd->dev);
1154         dd = NULL;
1155
1156         return 0;
1157 }
1158
1159 #ifdef CONFIG_PM_SLEEP
1160 static int omap_des_suspend(struct device *dev)
1161 {
1162         pm_runtime_put_sync(dev);
1163         return 0;
1164 }
1165
1166 static int omap_des_resume(struct device *dev)
1167 {
1168         int err;
1169
1170         err = pm_runtime_get_sync(dev);
1171         if (err < 0) {
1172                 pm_runtime_put_noidle(dev);
1173                 dev_err(dev, "%s: failed to get_sync(%d)\n", __func__, err);
1174                 return err;
1175         }
1176         return 0;
1177 }
1178 #endif
1179
1180 static SIMPLE_DEV_PM_OPS(omap_des_pm_ops, omap_des_suspend, omap_des_resume);
1181
1182 static struct platform_driver omap_des_driver = {
1183         .probe  = omap_des_probe,
1184         .remove = omap_des_remove,
1185         .driver = {
1186                 .name   = "omap-des",
1187                 .pm     = &omap_des_pm_ops,
1188                 .of_match_table = of_match_ptr(omap_des_of_match),
1189         },
1190 };
1191
1192 module_platform_driver(omap_des_driver);
1193
1194 MODULE_DESCRIPTION("OMAP DES hw acceleration support.");
1195 MODULE_LICENSE("GPL v2");
1196 MODULE_AUTHOR("Joel Fernandes <joelf@ti.com>");