drivers/dma/omap-dma.c

   1 /*
   2  * OMAP DMAengine support
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  */
   8 #include <linux/delay.h>
   9 #include <linux/dmaengine.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/dmapool.h>
  12 #include <linux/err.h>
  13 #include <linux/init.h>
  14 #include <linux/interrupt.h>
  15 #include <linux/list.h>
  16 #include <linux/module.h>
  17 #include <linux/omap-dma.h>
  18 #include <linux/platform_device.h>
  19 #include <linux/slab.h>
  20 #include <linux/spinlock.h>
  21 #include <linux/of_dma.h>
  22 #include <linux/of_device.h>
  23
  24 #include "virt-dma.h"
  25
  26 #define OMAP_SDMA_REQUESTS      127
  27 #define OMAP_SDMA_CHANNELS      32
  28
  29 struct omap_dmadev {
  30         struct dma_device ddev;
  31         spinlock_t lock;
  32         void __iomem *base;
  33         const struct omap_dma_reg *reg_map;
  34         struct omap_system_dma_plat_info *plat;
  35         bool legacy;
  36         bool ll123_supported;
  37         struct dma_pool *desc_pool;
  38         unsigned dma_requests;
  39         spinlock_t irq_lock;
  40         uint32_t irq_enable_mask;
  41         struct omap_chan **lch_map;
  42 };
  43
  44 struct omap_chan {
  45         struct virt_dma_chan vc;
  46         void __iomem *channel_base;
  47         const struct omap_dma_reg *reg_map;
  48         uint32_t ccr;
  49
  50         struct dma_slave_config cfg;
  51         unsigned dma_sig;
  52         bool cyclic;
  53         bool paused;
  54         bool running;
  55
  56         int dma_ch;
  57         struct omap_desc *desc;
  58         unsigned sgidx;
  59 };
  60
  61 #define DESC_NXT_SV_REFRESH     (0x1 << 24)
  62 #define DESC_NXT_SV_REUSE       (0x2 << 24)
  63 #define DESC_NXT_DV_REFRESH     (0x1 << 26)
  64 #define DESC_NXT_DV_REUSE       (0x2 << 26)
  65 #define DESC_NTYPE_TYPE2        (0x2 << 29)
  66
  67 /* Type 2 descriptor with Source or Destination address update */
  68 struct omap_type2_desc {
  69         uint32_t next_desc;
  70         uint32_t en;
  71         uint32_t addr; /* src or dst */
  72         uint16_t fn;
  73         uint16_t cicr;
  74         int16_t cdei;
  75         int16_t csei;
  76         int32_t cdfi;
  77         int32_t csfi;
  78 } __packed;
  79
  80 struct omap_sg {
  81         dma_addr_t addr;
  82         uint32_t en;            /* number of elements (24-bit) */
  83         uint32_t fn;            /* number of frames (16-bit) */
  84         int32_t fi;             /* for double indexing */
  85         int16_t ei;             /* for double indexing */
  86
  87         /* Linked list */
  88         struct omap_type2_desc *t2_desc;
  89         dma_addr_t t2_desc_paddr;
  90 };
  91
  92 struct omap_desc {
  93         struct virt_dma_desc vd;
  94         bool using_ll;
  95         enum dma_transfer_direction dir;
  96         dma_addr_t dev_addr;
  97
  98         int32_t fi;             /* for OMAP_DMA_SYNC_PACKET / double indexing */
  99         int16_t ei;             /* for double indexing */
 100         uint8_t es;             /* CSDP_DATA_TYPE_xxx */
 101         uint32_t ccr;           /* CCR value */
 102         uint16_t clnk_ctrl;     /* CLNK_CTRL value */
 103         uint16_t cicr;          /* CICR value */
 104         uint32_t csdp;          /* CSDP value */
 105
 106         unsigned sglen;
 107         struct omap_sg sg[0];
 108 };
 109
 110 enum {
 111         CAPS_0_SUPPORT_LL123    = BIT(20),      /* Linked List type1/2/3 */
 112         CAPS_0_SUPPORT_LL4      = BIT(21),      /* Linked List type4 */
 113
 114         CCR_FS                  = BIT(5),
 115         CCR_READ_PRIORITY       = BIT(6),
 116         CCR_ENABLE              = BIT(7),
 117         CCR_AUTO_INIT           = BIT(8),       /* OMAP1 only */
 118         CCR_REPEAT              = BIT(9),       /* OMAP1 only */
 119         CCR_OMAP31_DISABLE      = BIT(10),      /* OMAP1 only */
 120         CCR_SUSPEND_SENSITIVE   = BIT(8),       /* OMAP2+ only */
 121         CCR_RD_ACTIVE           = BIT(9),       /* OMAP2+ only */
 122         CCR_WR_ACTIVE           = BIT(10),      /* OMAP2+ only */
 123         CCR_SRC_AMODE_CONSTANT  = 0 << 12,
 124         CCR_SRC_AMODE_POSTINC   = 1 << 12,
 125         CCR_SRC_AMODE_SGLIDX    = 2 << 12,
 126         CCR_SRC_AMODE_DBLIDX    = 3 << 12,
 127         CCR_DST_AMODE_CONSTANT  = 0 << 14,
 128         CCR_DST_AMODE_POSTINC   = 1 << 14,
 129         CCR_DST_AMODE_SGLIDX    = 2 << 14,
 130         CCR_DST_AMODE_DBLIDX    = 3 << 14,
 131         CCR_CONSTANT_FILL       = BIT(16),
 132         CCR_TRANSPARENT_COPY    = BIT(17),
 133         CCR_BS                  = BIT(18),
 134         CCR_SUPERVISOR          = BIT(22),
 135         CCR_PREFETCH            = BIT(23),
 136         CCR_TRIGGER_SRC         = BIT(24),
 137         CCR_BUFFERING_DISABLE   = BIT(25),
 138         CCR_WRITE_PRIORITY      = BIT(26),
 139         CCR_SYNC_ELEMENT        = 0,
 140         CCR_SYNC_FRAME          = CCR_FS,
 141         CCR_SYNC_BLOCK          = CCR_BS,
 142         CCR_SYNC_PACKET         = CCR_BS | CCR_FS,
 143
 144         CSDP_DATA_TYPE_8        = 0,
 145         CSDP_DATA_TYPE_16       = 1,
 146         CSDP_DATA_TYPE_32       = 2,
 147         CSDP_SRC_PORT_EMIFF     = 0 << 2, /* OMAP1 only */
 148         CSDP_SRC_PORT_EMIFS     = 1 << 2, /* OMAP1 only */
 149         CSDP_SRC_PORT_OCP_T1    = 2 << 2, /* OMAP1 only */
 150         CSDP_SRC_PORT_TIPB      = 3 << 2, /* OMAP1 only */
 151         CSDP_SRC_PORT_OCP_T2    = 4 << 2, /* OMAP1 only */
 152         CSDP_SRC_PORT_MPUI      = 5 << 2, /* OMAP1 only */
 153         CSDP_SRC_PACKED         = BIT(6),
 154         CSDP_SRC_BURST_1        = 0 << 7,
 155         CSDP_SRC_BURST_16       = 1 << 7,
 156         CSDP_SRC_BURST_32       = 2 << 7,
 157         CSDP_SRC_BURST_64       = 3 << 7,
 158         CSDP_DST_PORT_EMIFF     = 0 << 9, /* OMAP1 only */
 159         CSDP_DST_PORT_EMIFS     = 1 << 9, /* OMAP1 only */
 160         CSDP_DST_PORT_OCP_T1    = 2 << 9, /* OMAP1 only */
 161         CSDP_DST_PORT_TIPB      = 3 << 9, /* OMAP1 only */
 162         CSDP_DST_PORT_OCP_T2    = 4 << 9, /* OMAP1 only */
 163         CSDP_DST_PORT_MPUI      = 5 << 9, /* OMAP1 only */
 164         CSDP_DST_PACKED         = BIT(13),
 165         CSDP_DST_BURST_1        = 0 << 14,
 166         CSDP_DST_BURST_16       = 1 << 14,
 167         CSDP_DST_BURST_32       = 2 << 14,
 168         CSDP_DST_BURST_64       = 3 << 14,
 169
 170         CICR_TOUT_IE            = BIT(0),       /* OMAP1 only */
 171         CICR_DROP_IE            = BIT(1),
 172         CICR_HALF_IE            = BIT(2),
 173         CICR_FRAME_IE           = BIT(3),
 174         CICR_LAST_IE            = BIT(4),
 175         CICR_BLOCK_IE           = BIT(5),
 176         CICR_PKT_IE             = BIT(7),       /* OMAP2+ only */
 177         CICR_TRANS_ERR_IE       = BIT(8),       /* OMAP2+ only */
 178         CICR_SUPERVISOR_ERR_IE  = BIT(10),      /* OMAP2+ only */
 179         CICR_MISALIGNED_ERR_IE  = BIT(11),      /* OMAP2+ only */
 180         CICR_DRAIN_IE           = BIT(12),      /* OMAP2+ only */
 181         CICR_SUPER_BLOCK_IE     = BIT(14),      /* OMAP2+ only */
 182
 183         CLNK_CTRL_ENABLE_LNK    = BIT(15),
 184
 185         CDP_DST_VALID_INC       = 0 << 0,
 186         CDP_DST_VALID_RELOAD    = 1 << 0,
 187         CDP_DST_VALID_REUSE     = 2 << 0,
 188         CDP_SRC_VALID_INC       = 0 << 2,
 189         CDP_SRC_VALID_RELOAD    = 1 << 2,
 190         CDP_SRC_VALID_REUSE     = 2 << 2,
 191         CDP_NTYPE_TYPE1         = 1 << 4,
 192         CDP_NTYPE_TYPE2         = 2 << 4,
 193         CDP_NTYPE_TYPE3         = 3 << 4,
 194         CDP_TMODE_NORMAL        = 0 << 8,
 195         CDP_TMODE_LLIST         = 1 << 8,
 196         CDP_FAST                = BIT(10),
 197 };
 198
 199 static const unsigned es_bytes[] = {
 200         [CSDP_DATA_TYPE_8] = 1,
 201         [CSDP_DATA_TYPE_16] = 2,
 202         [CSDP_DATA_TYPE_32] = 4,
 203 };
 204
 205 static struct of_dma_filter_info omap_dma_info = {
 206         .filter_fn = omap_dma_filter_fn,
 207 };
 208
 209 static inline struct omap_dmadev *to_omap_dma_dev(struct dma_device *d)
 210 {
 211         return container_of(d, struct omap_dmadev, ddev);
 212 }
 213
 214 static inline struct omap_chan *to_omap_dma_chan(struct dma_chan *c)
 215 {
 216         return container_of(c, struct omap_chan, vc.chan);
 217 }
 218
 219 static inline struct omap_desc *to_omap_dma_desc(struct dma_async_tx_descriptor *t)
 220 {
 221         return container_of(t, struct omap_desc, vd.tx);
 222 }
 223
 224 static void omap_dma_desc_free(struct virt_dma_desc *vd)
 225 {
 226         struct omap_desc *d = to_omap_dma_desc(&vd->tx);
 227
 228         if (d->using_ll) {
 229                 struct omap_dmadev *od = to_omap_dma_dev(vd->tx.chan->device);
 230                 int i;
 231
 232                 for (i = 0; i < d->sglen; i++) {
 233                         if (d->sg[i].t2_desc)
 234                                 dma_pool_free(od->desc_pool, d->sg[i].t2_desc,
 235                                               d->sg[i].t2_desc_paddr);
 236                 }
 237         }
 238
 239         kfree(d);
 240 }
 241
 242 static void omap_dma_fill_type2_desc(struct omap_desc *d, int idx,
 243                                      enum dma_transfer_direction dir, bool last)
 244 {
 245         struct omap_sg *sg = &d->sg[idx];
 246         struct omap_type2_desc *t2_desc = sg->t2_desc;
 247
 248         if (idx)
 249                 d->sg[idx - 1].t2_desc->next_desc = sg->t2_desc_paddr;
 250         if (last)
 251                 t2_desc->next_desc = 0xfffffffc;
 252
 253         t2_desc->en = sg->en;
 254         t2_desc->addr = sg->addr;
 255         t2_desc->fn = sg->fn & 0xffff;
 256         t2_desc->cicr = d->cicr;
 257         if (!last)
 258                 t2_desc->cicr &= ~CICR_BLOCK_IE;
 259
 260         switch (dir) {
 261         case DMA_DEV_TO_MEM:
 262                 t2_desc->cdei = sg->ei;
 263                 t2_desc->csei = d->ei;
 264                 t2_desc->cdfi = sg->fi;
 265                 t2_desc->csfi = d->fi;
 266
 267                 t2_desc->en |= DESC_NXT_DV_REFRESH;
 268                 t2_desc->en |= DESC_NXT_SV_REUSE;
 269                 break;
 270         case DMA_MEM_TO_DEV:
 271                 t2_desc->cdei = d->ei;
 272                 t2_desc->csei = sg->ei;
 273                 t2_desc->cdfi = d->fi;
 274                 t2_desc->csfi = sg->fi;
 275
 276                 t2_desc->en |= DESC_NXT_SV_REFRESH;
 277                 t2_desc->en |= DESC_NXT_DV_REUSE;
 278                 break;
 279         default:
 280                 return;
 281         }
 282
 283         t2_desc->en |= DESC_NTYPE_TYPE2;
 284 }
 285
 286 static void omap_dma_write(uint32_t val, unsigned type, void __iomem *addr)
 287 {
 288         switch (type) {
 289         case OMAP_DMA_REG_16BIT:
 290                 writew_relaxed(val, addr);
 291                 break;
 292         case OMAP_DMA_REG_2X16BIT:
 293                 writew_relaxed(val, addr);
 294                 writew_relaxed(val >> 16, addr + 2);
 295                 break;
 296         case OMAP_DMA_REG_32BIT:
 297                 writel_relaxed(val, addr);
 298                 break;
 299         default:
 300                 WARN_ON(1);
 301         }
 302 }
 303
 304 static unsigned omap_dma_read(unsigned type, void __iomem *addr)
 305 {
 306         unsigned val;
 307
 308         switch (type) {
 309         case OMAP_DMA_REG_16BIT:
 310                 val = readw_relaxed(addr);
 311                 break;
 312         case OMAP_DMA_REG_2X16BIT:
 313                 val = readw_relaxed(addr);
 314                 val |= readw_relaxed(addr + 2) << 16;
 315                 break;
 316         case OMAP_DMA_REG_32BIT:
 317                 val = readl_relaxed(addr);
 318                 break;
 319         default:
 320                 WARN_ON(1);
 321                 val = 0;
 322         }
 323
 324         return val;
 325 }
 326
 327 static void omap_dma_glbl_write(struct omap_dmadev *od, unsigned reg, unsigned val)
 328 {
 329         const struct omap_dma_reg *r = od->reg_map + reg;
 330
 331         WARN_ON(r->stride);
 332
 333         omap_dma_write(val, r->type, od->base + r->offset);
 334 }
 335
 336 static unsigned omap_dma_glbl_read(struct omap_dmadev *od, unsigned reg)
 337 {
 338         const struct omap_dma_reg *r = od->reg_map + reg;
 339
 340         WARN_ON(r->stride);
 341
 342         return omap_dma_read(r->type, od->base + r->offset);
 343 }
 344
 345 static void omap_dma_chan_write(struct omap_chan *c, unsigned reg, unsigned val)
 346 {
 347         const struct omap_dma_reg *r = c->reg_map + reg;
 348
 349         omap_dma_write(val, r->type, c->channel_base + r->offset);
 350 }
 351
 352 static unsigned omap_dma_chan_read(struct omap_chan *c, unsigned reg)
 353 {
 354         const struct omap_dma_reg *r = c->reg_map + reg;
 355
 356         return omap_dma_read(r->type, c->channel_base + r->offset);
 357 }
 358
 359 static void omap_dma_clear_csr(struct omap_chan *c)
 360 {
 361         if (dma_omap1())
 362                 omap_dma_chan_read(c, CSR);
 363         else
 364                 omap_dma_chan_write(c, CSR, ~0);
 365 }
 366
 367 static unsigned omap_dma_get_csr(struct omap_chan *c)
 368 {
 369         unsigned val = omap_dma_chan_read(c, CSR);
 370
 371         if (!dma_omap1())
 372                 omap_dma_chan_write(c, CSR, val);
 373
 374         return val;
 375 }
 376
 377 static void omap_dma_assign(struct omap_dmadev *od, struct omap_chan *c,
 378         unsigned lch)
 379 {
 380         c->channel_base = od->base + od->plat->channel_stride * lch;
 381
 382         od->lch_map[lch] = c;
 383 }
 384
 385 static void omap_dma_start(struct omap_chan *c, struct omap_desc *d)
 386 {
 387         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 388         uint16_t cicr = d->cicr;
 389
 390         if (__dma_omap15xx(od->plat->dma_attr))
 391                 omap_dma_chan_write(c, CPC, 0);
 392         else
 393                 omap_dma_chan_write(c, CDAC, 0);
 394
 395         omap_dma_clear_csr(c);
 396
 397         if (d->using_ll) {
 398                 uint32_t cdp = CDP_TMODE_LLIST | CDP_NTYPE_TYPE2 | CDP_FAST;
 399
 400                 if (d->dir == DMA_DEV_TO_MEM)
 401                         cdp |= (CDP_DST_VALID_RELOAD | CDP_SRC_VALID_REUSE);
 402                 else
 403                         cdp |= (CDP_DST_VALID_REUSE | CDP_SRC_VALID_RELOAD);
 404                 omap_dma_chan_write(c, CDP, cdp);
 405
 406                 omap_dma_chan_write(c, CNDP, d->sg[0].t2_desc_paddr);
 407                 omap_dma_chan_write(c, CCDN, 0);
 408                 omap_dma_chan_write(c, CCFN, 0xffff);
 409                 omap_dma_chan_write(c, CCEN, 0xffffff);
 410
 411                 cicr &= ~CICR_BLOCK_IE;
 412         } else if (od->ll123_supported) {
 413                 omap_dma_chan_write(c, CDP, 0);
 414         }
 415
 416         /* Enable interrupts */
 417         omap_dma_chan_write(c, CICR, cicr);
 418
 419         /* Enable channel */
 420         omap_dma_chan_write(c, CCR, d->ccr | CCR_ENABLE);
 421
 422         c->running = true;
 423 }
 424
 425 static void omap_dma_stop(struct omap_chan *c)
 426 {
 427         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 428         uint32_t val;
 429
 430         /* disable irq */
 431         omap_dma_chan_write(c, CICR, 0);
 432
 433         omap_dma_clear_csr(c);
 434
 435         val = omap_dma_chan_read(c, CCR);
 436         if (od->plat->errata & DMA_ERRATA_i541 && val & CCR_TRIGGER_SRC) {
 437                 uint32_t sysconfig;
 438                 unsigned i;
 439
 440                 sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG);
 441                 val = sysconfig & ~DMA_SYSCONFIG_MIDLEMODE_MASK;
 442                 val |= DMA_SYSCONFIG_MIDLEMODE(DMA_IDLEMODE_NO_IDLE);
 443                 omap_dma_glbl_write(od, OCP_SYSCONFIG, val);
 444
 445                 val = omap_dma_chan_read(c, CCR);
 446                 val &= ~CCR_ENABLE;
 447                 omap_dma_chan_write(c, CCR, val);
 448
 449                 /* Wait for sDMA FIFO to drain */
 450                 for (i = 0; ; i++) {
 451                         val = omap_dma_chan_read(c, CCR);
 452                         if (!(val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE)))
 453                                 break;
 454
 455                         if (i > 100)
 456                                 break;
 457
 458                         udelay(5);
 459                 }
 460
 461                 if (val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE))
 462                         dev_err(c->vc.chan.device->dev,
 463                                 "DMA drain did not complete on lch %d\n",
 464                                 c->dma_ch);
 465
 466                 omap_dma_glbl_write(od, OCP_SYSCONFIG, sysconfig);
 467         } else {
 468                 val &= ~CCR_ENABLE;
 469                 omap_dma_chan_write(c, CCR, val);
 470         }
 471
 472         mb();
 473
 474         if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) {
 475                 val = omap_dma_chan_read(c, CLNK_CTRL);
 476
 477                 if (dma_omap1())
 478                         val |= 1 << 14; /* set the STOP_LNK bit */
 479                 else
 480                         val &= ~CLNK_CTRL_ENABLE_LNK;
 481
 482                 omap_dma_chan_write(c, CLNK_CTRL, val);
 483         }
 484
 485         c->running = false;
 486 }
 487
 488 static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d)
 489 {
 490         struct omap_sg *sg = d->sg + c->sgidx;
 491         unsigned cxsa, cxei, cxfi;
 492
 493         if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
 494                 cxsa = CDSA;
 495                 cxei = CDEI;
 496                 cxfi = CDFI;
 497         } else {
 498                 cxsa = CSSA;
 499                 cxei = CSEI;
 500                 cxfi = CSFI;
 501         }
 502
 503         omap_dma_chan_write(c, cxsa, sg->addr);
 504         omap_dma_chan_write(c, cxei, sg->ei);
 505         omap_dma_chan_write(c, cxfi, sg->fi);
 506         omap_dma_chan_write(c, CEN, sg->en);
 507         omap_dma_chan_write(c, CFN, sg->fn);
 508
 509         omap_dma_start(c, d);
 510         c->sgidx++;
 511 }
 512
 513 static void omap_dma_start_desc(struct omap_chan *c)
 514 {
 515         struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
 516         struct omap_desc *d;
 517         unsigned cxsa, cxei, cxfi;
 518
 519         if (!vd) {
 520                 c->desc = NULL;
 521                 return;
 522         }
 523
 524         list_del(&vd->node);
 525
 526         c->desc = d = to_omap_dma_desc(&vd->tx);
 527         c->sgidx = 0;
 528
 529         /*
 530          * This provides the necessary barrier to ensure data held in
 531          * DMA coherent memory is visible to the DMA engine prior to
 532          * the transfer starting.
 533          */
 534         mb();
 535
 536         omap_dma_chan_write(c, CCR, d->ccr);
 537         if (dma_omap1())
 538                 omap_dma_chan_write(c, CCR2, d->ccr >> 16);
 539
 540         if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
 541                 cxsa = CSSA;
 542                 cxei = CSEI;
 543                 cxfi = CSFI;
 544         } else {
 545                 cxsa = CDSA;
 546                 cxei = CDEI;
 547                 cxfi = CDFI;
 548         }
 549
 550         omap_dma_chan_write(c, cxsa, d->dev_addr);
 551         omap_dma_chan_write(c, cxei, d->ei);
 552         omap_dma_chan_write(c, cxfi, d->fi);
 553         omap_dma_chan_write(c, CSDP, d->csdp);
 554         omap_dma_chan_write(c, CLNK_CTRL, d->clnk_ctrl);
 555
 556         omap_dma_start_sg(c, d);
 557 }
 558
 559 static void omap_dma_callback(int ch, u16 status, void *data)
 560 {
 561         struct omap_chan *c = data;
 562         struct omap_desc *d;
 563         unsigned long flags;
 564
 565         spin_lock_irqsave(&c->vc.lock, flags);
 566         d = c->desc;
 567         if (d) {
 568                 if (c->cyclic) {
 569                         vchan_cyclic_callback(&d->vd);
 570                 } else if (d->using_ll || c->sgidx == d->sglen) {
 571                         omap_dma_start_desc(c);
 572                         vchan_cookie_complete(&d->vd);
 573                 } else {
 574                         omap_dma_start_sg(c, d);
 575                 }
 576         }
 577         spin_unlock_irqrestore(&c->vc.lock, flags);
 578 }
 579
 580 static irqreturn_t omap_dma_irq(int irq, void *devid)
 581 {
 582         struct omap_dmadev *od = devid;
 583         unsigned status, channel;
 584
 585         spin_lock(&od->irq_lock);
 586
 587         status = omap_dma_glbl_read(od, IRQSTATUS_L1);
 588         status &= od->irq_enable_mask;
 589         if (status == 0) {
 590                 spin_unlock(&od->irq_lock);
 591                 return IRQ_NONE;
 592         }
 593
 594         while ((channel = ffs(status)) != 0) {
 595                 unsigned mask, csr;
 596                 struct omap_chan *c;
 597
 598                 channel -= 1;
 599                 mask = BIT(channel);
 600                 status &= ~mask;
 601
 602                 c = od->lch_map[channel];
 603                 if (c == NULL) {
 604                         /* This should never happen */
 605                         dev_err(od->ddev.dev, "invalid channel %u\n", channel);
 606                         continue;
 607                 }
 608
 609                 csr = omap_dma_get_csr(c);
 610                 omap_dma_glbl_write(od, IRQSTATUS_L1, mask);
 611
 612                 omap_dma_callback(channel, csr, c);
 613         }
 614
 615         spin_unlock(&od->irq_lock);
 616
 617         return IRQ_HANDLED;
 618 }
 619
 620 static int omap_dma_alloc_chan_resources(struct dma_chan *chan)
 621 {
 622         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 623         struct omap_chan *c = to_omap_dma_chan(chan);
 624         struct device *dev = od->ddev.dev;
 625         int ret;
 626
 627         if (od->legacy) {
 628                 ret = omap_request_dma(c->dma_sig, "DMA engine",
 629                                        omap_dma_callback, c, &c->dma_ch);
 630         } else {
 631                 ret = omap_request_dma(c->dma_sig, "DMA engine", NULL, NULL,
 632                                        &c->dma_ch);
 633         }
 634
 635         dev_dbg(dev, "allocating channel %u for %u\n", c->dma_ch, c->dma_sig);
 636
 637         if (ret >= 0) {
 638                 omap_dma_assign(od, c, c->dma_ch);
 639
 640                 if (!od->legacy) {
 641                         unsigned val;
 642
 643                         spin_lock_irq(&od->irq_lock);
 644                         val = BIT(c->dma_ch);
 645                         omap_dma_glbl_write(od, IRQSTATUS_L1, val);
 646                         od->irq_enable_mask |= val;
 647                         omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
 648
 649                         val = omap_dma_glbl_read(od, IRQENABLE_L0);
 650                         val &= ~BIT(c->dma_ch);
 651                         omap_dma_glbl_write(od, IRQENABLE_L0, val);
 652                         spin_unlock_irq(&od->irq_lock);
 653                 }
 654         }
 655
 656         if (dma_omap1()) {
 657                 if (__dma_omap16xx(od->plat->dma_attr)) {
 658                         c->ccr = CCR_OMAP31_DISABLE;
 659                         /* Duplicate what plat-omap/dma.c does */
 660                         c->ccr |= c->dma_ch + 1;
 661                 } else {
 662                         c->ccr = c->dma_sig & 0x1f;
 663                 }
 664         } else {
 665                 c->ccr = c->dma_sig & 0x1f;
 666                 c->ccr |= (c->dma_sig & ~0x1f) << 14;
 667         }
 668         if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING)
 669                 c->ccr |= CCR_BUFFERING_DISABLE;
 670
 671         return ret;
 672 }
 673
 674 static void omap_dma_free_chan_resources(struct dma_chan *chan)
 675 {
 676         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 677         struct omap_chan *c = to_omap_dma_chan(chan);
 678
 679         if (!od->legacy) {
 680                 spin_lock_irq(&od->irq_lock);
 681                 od->irq_enable_mask &= ~BIT(c->dma_ch);
 682                 omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
 683                 spin_unlock_irq(&od->irq_lock);
 684         }
 685
 686         c->channel_base = NULL;
 687         od->lch_map[c->dma_ch] = NULL;
 688         vchan_free_chan_resources(&c->vc);
 689         omap_free_dma(c->dma_ch);
 690
 691         dev_dbg(od->ddev.dev, "freeing channel %u used for %u\n", c->dma_ch,
 692                 c->dma_sig);
 693         c->dma_sig = 0;
 694 }
 695
 696 static size_t omap_dma_sg_size(struct omap_sg *sg)
 697 {
 698         return sg->en * sg->fn;
 699 }
 700
 701 static size_t omap_dma_desc_size(struct omap_desc *d)
 702 {
 703         unsigned i;
 704         size_t size;
 705
 706         for (size = i = 0; i < d->sglen; i++)
 707                 size += omap_dma_sg_size(&d->sg[i]);
 708
 709         return size * es_bytes[d->es];
 710 }
 711
 712 static size_t omap_dma_desc_size_pos(struct omap_desc *d, dma_addr_t addr)
 713 {
 714         unsigned i;
 715         size_t size, es_size = es_bytes[d->es];
 716
 717         for (size = i = 0; i < d->sglen; i++) {
 718                 size_t this_size = omap_dma_sg_size(&d->sg[i]) * es_size;
 719
 720                 if (size)
 721                         size += this_size;
 722                 else if (addr >= d->sg[i].addr &&
 723                          addr < d->sg[i].addr + this_size)
 724                         size += d->sg[i].addr + this_size - addr;
 725         }
 726         return size;
 727 }
 728
 729 /*
 730  * OMAP 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is
 731  * read before the DMA controller finished disabling the channel.
 732  */
 733 static uint32_t omap_dma_chan_read_3_3(struct omap_chan *c, unsigned reg)
 734 {
 735         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 736         uint32_t val;
 737
 738         val = omap_dma_chan_read(c, reg);
 739         if (val == 0 && od->plat->errata & DMA_ERRATA_3_3)
 740                 val = omap_dma_chan_read(c, reg);
 741
 742         return val;
 743 }
 744
 745 static dma_addr_t omap_dma_get_src_pos(struct omap_chan *c)
 746 {
 747         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 748         dma_addr_t addr, cdac;
 749
 750         if (__dma_omap15xx(od->plat->dma_attr)) {
 751                 addr = omap_dma_chan_read(c, CPC);
 752         } else {
 753                 addr = omap_dma_chan_read_3_3(c, CSAC);
 754                 cdac = omap_dma_chan_read_3_3(c, CDAC);
 755
 756                 /*
 757                  * CDAC == 0 indicates that the DMA transfer on the channel has
 758                  * not been started (no data has been transferred so far).
 759                  * Return the programmed source start address in this case.
 760                  */
 761                 if (cdac == 0)
 762                         addr = omap_dma_chan_read(c, CSSA);
 763         }
 764
 765         if (dma_omap1())
 766                 addr |= omap_dma_chan_read(c, CSSA) & 0xffff0000;
 767
 768         return addr;
 769 }
 770
 771 static dma_addr_t omap_dma_get_dst_pos(struct omap_chan *c)
 772 {
 773         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 774         dma_addr_t addr;
 775
 776         if (__dma_omap15xx(od->plat->dma_attr)) {
 777                 addr = omap_dma_chan_read(c, CPC);
 778         } else {
 779                 addr = omap_dma_chan_read_3_3(c, CDAC);
 780
 781                 /*
 782                  * CDAC == 0 indicates that the DMA transfer on the channel
 783                  * has not been started (no data has been transferred so
 784                  * far).  Return the programmed destination start address in
 785                  * this case.
 786                  */
 787                 if (addr == 0)
 788                         addr = omap_dma_chan_read(c, CDSA);
 789         }
 790
 791         if (dma_omap1())
 792                 addr |= omap_dma_chan_read(c, CDSA) & 0xffff0000;
 793
 794         return addr;
 795 }
 796
 797 static enum dma_status omap_dma_tx_status(struct dma_chan *chan,
 798         dma_cookie_t cookie, struct dma_tx_state *txstate)
 799 {
 800         struct omap_chan *c = to_omap_dma_chan(chan);
 801         struct virt_dma_desc *vd;
 802         enum dma_status ret;
 803         unsigned long flags;
 804
 805         ret = dma_cookie_status(chan, cookie, txstate);
 806
 807         if (!c->paused && c->running) {
 808                 uint32_t ccr = omap_dma_chan_read(c, CCR);
 809                 /*
 810                  * The channel is no longer active, set the return value
 811                  * accordingly
 812                  */
 813                 if (!(ccr & CCR_ENABLE))
 814                         ret = DMA_COMPLETE;
 815         }
 816
 817         if (ret == DMA_COMPLETE || !txstate)
 818                 return ret;
 819
 820         spin_lock_irqsave(&c->vc.lock, flags);
 821         vd = vchan_find_desc(&c->vc, cookie);
 822         if (vd) {
 823                 txstate->residue = omap_dma_desc_size(to_omap_dma_desc(&vd->tx));
 824         } else if (c->desc && c->desc->vd.tx.cookie == cookie) {
 825                 struct omap_desc *d = c->desc;
 826                 dma_addr_t pos;
 827
 828                 if (d->dir == DMA_MEM_TO_DEV)
 829                         pos = omap_dma_get_src_pos(c);
 830                 else if (d->dir == DMA_DEV_TO_MEM  || d->dir == DMA_MEM_TO_MEM)
 831                         pos = omap_dma_get_dst_pos(c);
 832                 else
 833                         pos = 0;
 834
 835                 txstate->residue = omap_dma_desc_size_pos(d, pos);
 836         } else {
 837                 txstate->residue = 0;
 838         }
 839         spin_unlock_irqrestore(&c->vc.lock, flags);
 840
 841         return ret;
 842 }
 843
 844 static void omap_dma_issue_pending(struct dma_chan *chan)
 845 {
 846         struct omap_chan *c = to_omap_dma_chan(chan);
 847         unsigned long flags;
 848
 849         spin_lock_irqsave(&c->vc.lock, flags);
 850         if (vchan_issue_pending(&c->vc) && !c->desc)
 851                 omap_dma_start_desc(c);
 852         spin_unlock_irqrestore(&c->vc.lock, flags);
 853 }
 854
 855 static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
 856         struct dma_chan *chan, struct scatterlist *sgl, unsigned sglen,
 857         enum dma_transfer_direction dir, unsigned long tx_flags, void *context)
 858 {
 859         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 860         struct omap_chan *c = to_omap_dma_chan(chan);
 861         enum dma_slave_buswidth dev_width;
 862         struct scatterlist *sgent;
 863         struct omap_desc *d;
 864         dma_addr_t dev_addr;
 865         unsigned i, es, en, frame_bytes;
 866         bool ll_failed = false;
 867         u32 burst;
 868
 869         if (dir == DMA_DEV_TO_MEM) {
 870                 dev_addr = c->cfg.src_addr;
 871                 dev_width = c->cfg.src_addr_width;
 872                 burst = c->cfg.src_maxburst;
 873         } else if (dir == DMA_MEM_TO_DEV) {
 874                 dev_addr = c->cfg.dst_addr;
 875                 dev_width = c->cfg.dst_addr_width;
 876                 burst = c->cfg.dst_maxburst;
 877         } else {
 878                 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
 879                 return NULL;
 880         }
 881
 882         /* Bus width translates to the element size (ES) */
 883         switch (dev_width) {
 884         case DMA_SLAVE_BUSWIDTH_1_BYTE:
 885                 es = CSDP_DATA_TYPE_8;
 886                 break;
 887         case DMA_SLAVE_BUSWIDTH_2_BYTES:
 888                 es = CSDP_DATA_TYPE_16;
 889                 break;
 890         case DMA_SLAVE_BUSWIDTH_4_BYTES:
 891                 es = CSDP_DATA_TYPE_32;
 892                 break;
 893         default: /* not reached */
 894                 return NULL;
 895         }
 896
 897         /* Now allocate and setup the descriptor. */
 898         d = kzalloc(sizeof(*d) + sglen * sizeof(d->sg[0]), GFP_ATOMIC);
 899         if (!d)
 900                 return NULL;
 901
 902         d->dir = dir;
 903         d->dev_addr = dev_addr;
 904         d->es = es;
 905
 906         d->ccr = c->ccr | CCR_SYNC_FRAME;
 907         if (dir == DMA_DEV_TO_MEM)
 908                 d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT;
 909         else
 910                 d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC;
 911
 912         d->cicr = CICR_DROP_IE | CICR_BLOCK_IE;
 913         d->csdp = es;
 914
 915         if (dma_omap1()) {
 916                 d->cicr |= CICR_TOUT_IE;
 917
 918                 if (dir == DMA_DEV_TO_MEM)
 919                         d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_TIPB;
 920                 else
 921                         d->csdp |= CSDP_DST_PORT_TIPB | CSDP_SRC_PORT_EMIFF;
 922         } else {
 923                 if (dir == DMA_DEV_TO_MEM)
 924                         d->ccr |= CCR_TRIGGER_SRC;
 925
 926                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
 927         }
 928         if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS)
 929                 d->clnk_ctrl = c->dma_ch;
 930
 931         /*
 932          * Build our scatterlist entries: each contains the address,
 933          * the number of elements (EN) in each frame, and the number of
 934          * frames (FN).  Number of bytes for this entry = ES * EN * FN.
 935          *
 936          * Burst size translates to number of elements with frame sync.
 937          * Note: DMA engine defines burst to be the number of dev-width
 938          * transfers.
 939          */
 940         en = burst;
 941         frame_bytes = es_bytes[es] * en;
 942
 943         if (sglen >= 2)
 944                 d->using_ll = od->ll123_supported;
 945
 946         for_each_sg(sgl, sgent, sglen, i) {
 947                 struct omap_sg *osg = &d->sg[i];
 948
 949                 osg->addr = sg_dma_address(sgent);
 950                 osg->en = en;
 951                 osg->fn = sg_dma_len(sgent) / frame_bytes;
 952
 953                 if (d->using_ll) {
 954                         osg->t2_desc = dma_pool_alloc(od->desc_pool, GFP_ATOMIC,
 955                                                       &osg->t2_desc_paddr);
 956                         if (!osg->t2_desc) {
 957                                 dev_err(chan->device->dev,
 958                                         "t2_desc[%d] allocation failed\n", i);
 959                                 ll_failed = true;
 960                                 d->using_ll = false;
 961                                 continue;
 962                         }
 963
 964                         omap_dma_fill_type2_desc(d, i, dir, (i == sglen - 1));
 965                 }
 966         }
 967
 968         d->sglen = sglen;
 969
 970         /* Release the dma_pool entries if one allocation failed */
 971         if (ll_failed) {
 972                 for (i = 0; i < d->sglen; i++) {
 973                         struct omap_sg *osg = &d->sg[i];
 974
 975                         if (osg->t2_desc) {
 976                                 dma_pool_free(od->desc_pool, osg->t2_desc,
 977                                               osg->t2_desc_paddr);
 978                                 osg->t2_desc = NULL;
 979                         }
 980                 }
 981         }
 982
 983         return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
 984 }
 985
 986 static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic(
 987         struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
 988         size_t period_len, enum dma_transfer_direction dir, unsigned long flags)
 989 {
 990         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 991         struct omap_chan *c = to_omap_dma_chan(chan);
 992         enum dma_slave_buswidth dev_width;
 993         struct omap_desc *d;
 994         dma_addr_t dev_addr;
 995         unsigned es;
 996         u32 burst;
 997
 998         if (dir == DMA_DEV_TO_MEM) {
 999                 dev_addr = c->cfg.src_addr;
1000                 dev_width = c->cfg.src_addr_width;
1001                 burst = c->cfg.src_maxburst;
1002         } else if (dir == DMA_MEM_TO_DEV) {
1003                 dev_addr = c->cfg.dst_addr;
1004                 dev_width = c->cfg.dst_addr_width;
1005                 burst = c->cfg.dst_maxburst;
1006         } else {
1007                 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
1008                 return NULL;
1009         }
1010
1011         /* Bus width translates to the element size (ES) */
1012         switch (dev_width) {
1013         case DMA_SLAVE_BUSWIDTH_1_BYTE:
1014                 es = CSDP_DATA_TYPE_8;
1015                 break;
1016         case DMA_SLAVE_BUSWIDTH_2_BYTES:
1017                 es = CSDP_DATA_TYPE_16;
1018                 break;
1019         case DMA_SLAVE_BUSWIDTH_4_BYTES:
1020                 es = CSDP_DATA_TYPE_32;
1021                 break;
1022         default: /* not reached */
1023                 return NULL;
1024         }
1025
1026         /* Now allocate and setup the descriptor. */
1027         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1028         if (!d)
1029                 return NULL;
1030
1031         d->dir = dir;
1032         d->dev_addr = dev_addr;
1033         d->fi = burst;
1034         d->es = es;
1035         d->sg[0].addr = buf_addr;
1036         d->sg[0].en = period_len / es_bytes[es];
1037         d->sg[0].fn = buf_len / period_len;
1038         d->sglen = 1;
1039
1040         d->ccr = c->ccr;
1041         if (dir == DMA_DEV_TO_MEM)
1042                 d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT;
1043         else
1044                 d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC;
1045
1046         d->cicr = CICR_DROP_IE;
1047         if (flags & DMA_PREP_INTERRUPT)
1048                 d->cicr |= CICR_FRAME_IE;
1049
1050         d->csdp = es;
1051
1052         if (dma_omap1()) {
1053                 d->cicr |= CICR_TOUT_IE;
1054
1055                 if (dir == DMA_DEV_TO_MEM)
1056                         d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_MPUI;
1057                 else
1058                         d->csdp |= CSDP_DST_PORT_MPUI | CSDP_SRC_PORT_EMIFF;
1059         } else {
1060                 if (burst)
1061                         d->ccr |= CCR_SYNC_PACKET;
1062                 else
1063                         d->ccr |= CCR_SYNC_ELEMENT;
1064
1065                 if (dir == DMA_DEV_TO_MEM) {
1066                         d->ccr |= CCR_TRIGGER_SRC;
1067                         d->csdp |= CSDP_DST_PACKED;
1068                 } else {
1069                         d->csdp |= CSDP_SRC_PACKED;
1070                 }
1071
1072                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1073
1074                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1075         }
1076
1077         if (__dma_omap15xx(od->plat->dma_attr))
1078                 d->ccr |= CCR_AUTO_INIT | CCR_REPEAT;
1079         else
1080                 d->clnk_ctrl = c->dma_ch | CLNK_CTRL_ENABLE_LNK;
1081
1082         c->cyclic = true;
1083
1084         return vchan_tx_prep(&c->vc, &d->vd, flags);
1085 }
1086
1087 static struct dma_async_tx_descriptor *omap_dma_prep_dma_memcpy(
1088         struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
1089         size_t len, unsigned long tx_flags)
1090 {
1091         struct omap_chan *c = to_omap_dma_chan(chan);
1092         struct omap_desc *d;
1093         uint8_t data_type;
1094
1095         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1096         if (!d)
1097                 return NULL;
1098
1099         data_type = __ffs((src | dest | len));
1100         if (data_type > CSDP_DATA_TYPE_32)
1101                 data_type = CSDP_DATA_TYPE_32;
1102
1103         d->dir = DMA_MEM_TO_MEM;
1104         d->dev_addr = src;
1105         d->fi = 0;
1106         d->es = data_type;
1107         d->sg[0].en = len / BIT(data_type);
1108         d->sg[0].fn = 1;
1109         d->sg[0].addr = dest;
1110         d->sglen = 1;
1111         d->ccr = c->ccr;
1112         d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_POSTINC;
1113
1114         d->cicr = CICR_DROP_IE | CICR_FRAME_IE;
1115
1116         d->csdp = data_type;
1117
1118         if (dma_omap1()) {
1119                 d->cicr |= CICR_TOUT_IE;
1120                 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
1121         } else {
1122                 d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
1123                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1124                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1125         }
1126
1127         return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
1128 }
1129
1130 static struct dma_async_tx_descriptor *omap_dma_prep_dma_interleaved(
1131         struct dma_chan *chan, struct dma_interleaved_template *xt,
1132         unsigned long flags)
1133 {
1134         struct omap_chan *c = to_omap_dma_chan(chan);
1135         struct omap_desc *d;
1136         struct omap_sg *sg;
1137         uint8_t data_type;
1138         size_t src_icg, dst_icg;
1139
1140         /* Slave mode is not supported */
1141         if (is_slave_direction(xt->dir))
1142                 return NULL;
1143
1144         if (xt->frame_size != 1 || xt->numf == 0)
1145                 return NULL;
1146
1147         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1148         if (!d)
1149                 return NULL;
1150
1151         data_type = __ffs((xt->src_start | xt->dst_start | xt->sgl[0].size));
1152         if (data_type > CSDP_DATA_TYPE_32)
1153                 data_type = CSDP_DATA_TYPE_32;
1154
1155         sg = &d->sg[0];
1156         d->dir = DMA_MEM_TO_MEM;
1157         d->dev_addr = xt->src_start;
1158         d->es = data_type;
1159         sg->en = xt->sgl[0].size / BIT(data_type);
1160         sg->fn = xt->numf;
1161         sg->addr = xt->dst_start;
1162         d->sglen = 1;
1163         d->ccr = c->ccr;
1164
1165         src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]);
1166         dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]);
1167         if (src_icg) {
1168                 d->ccr |= CCR_SRC_AMODE_DBLIDX;
1169                 d->ei = 1;
1170                 d->fi = src_icg;
1171         } else if (xt->src_inc) {
1172                 d->ccr |= CCR_SRC_AMODE_POSTINC;
1173                 d->fi = 0;
1174         } else {
1175                 dev_err(chan->device->dev,
1176                         "%s: SRC constant addressing is not supported\n",
1177                         __func__);
1178                 kfree(d);
1179                 return NULL;
1180         }
1181
1182         if (dst_icg) {
1183                 d->ccr |= CCR_DST_AMODE_DBLIDX;
1184                 sg->ei = 1;
1185                 sg->fi = dst_icg;
1186         } else if (xt->dst_inc) {
1187                 d->ccr |= CCR_DST_AMODE_POSTINC;
1188                 sg->fi = 0;
1189         } else {
1190                 dev_err(chan->device->dev,
1191                         "%s: DST constant addressing is not supported\n",
1192                         __func__);
1193                 kfree(d);
1194                 return NULL;
1195         }
1196
1197         d->cicr = CICR_DROP_IE | CICR_FRAME_IE;
1198
1199         d->csdp = data_type;
1200
1201         if (dma_omap1()) {
1202                 d->cicr |= CICR_TOUT_IE;
1203                 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
1204         } else {
1205                 d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
1206                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1207                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1208         }
1209
1210         return vchan_tx_prep(&c->vc, &d->vd, flags);
1211 }
1212
1213 static int omap_dma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg)
1214 {
1215         struct omap_chan *c = to_omap_dma_chan(chan);
1216
1217         if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
1218             cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
1219                 return -EINVAL;
1220
1221         memcpy(&c->cfg, cfg, sizeof(c->cfg));
1222
1223         return 0;
1224 }
1225
1226 static int omap_dma_terminate_all(struct dma_chan *chan)
1227 {
1228         struct omap_chan *c = to_omap_dma_chan(chan);
1229         unsigned long flags;
1230         LIST_HEAD(head);
1231
1232         spin_lock_irqsave(&c->vc.lock, flags);
1233
1234         /*
1235          * Stop DMA activity: we assume the callback will not be called
1236          * after omap_dma_stop() returns (even if it does, it will see
1237          * c->desc is NULL and exit.)
1238          */
1239         if (c->desc) {
1240                 omap_dma_desc_free(&c->desc->vd);
1241                 c->desc = NULL;
1242                 /* Avoid stopping the dma twice */
1243                 if (!c->paused)
1244                         omap_dma_stop(c);
1245         }
1246
1247         if (c->cyclic) {
1248                 c->cyclic = false;
1249                 c->paused = false;
1250         }
1251
1252         vchan_get_all_descriptors(&c->vc, &head);
1253         spin_unlock_irqrestore(&c->vc.lock, flags);
1254         vchan_dma_desc_free_list(&c->vc, &head);
1255
1256         return 0;
1257 }
1258
1259 static void omap_dma_synchronize(struct dma_chan *chan)
1260 {
1261         struct omap_chan *c = to_omap_dma_chan(chan);
1262
1263         vchan_synchronize(&c->vc);
1264 }
1265
1266 static int omap_dma_pause(struct dma_chan *chan)
1267 {
1268         struct omap_chan *c = to_omap_dma_chan(chan);
1269
1270         /* Pause/Resume only allowed with cyclic mode */
1271         if (!c->cyclic)
1272                 return -EINVAL;
1273
1274         if (!c->paused) {
1275                 omap_dma_stop(c);
1276                 c->paused = true;
1277         }
1278
1279         return 0;
1280 }
1281
1282 static int omap_dma_resume(struct dma_chan *chan)
1283 {
1284         struct omap_chan *c = to_omap_dma_chan(chan);
1285
1286         /* Pause/Resume only allowed with cyclic mode */
1287         if (!c->cyclic)
1288                 return -EINVAL;
1289
1290         if (c->paused) {
1291                 mb();
1292
1293                 /* Restore channel link register */
1294                 omap_dma_chan_write(c, CLNK_CTRL, c->desc->clnk_ctrl);
1295
1296                 omap_dma_start(c, c->desc);
1297                 c->paused = false;
1298         }
1299
1300         return 0;
1301 }
1302
1303 static int omap_dma_chan_init(struct omap_dmadev *od)
1304 {
1305         struct omap_chan *c;
1306
1307         c = kzalloc(sizeof(*c), GFP_KERNEL);
1308         if (!c)
1309                 return -ENOMEM;
1310
1311         c->reg_map = od->reg_map;
1312         c->vc.desc_free = omap_dma_desc_free;
1313         vchan_init(&c->vc, &od->ddev);
1314
1315         return 0;
1316 }
1317
1318 static void omap_dma_free(struct omap_dmadev *od)
1319 {
1320         while (!list_empty(&od->ddev.channels)) {
1321                 struct omap_chan *c = list_first_entry(&od->ddev.channels,
1322                         struct omap_chan, vc.chan.device_node);
1323
1324                 list_del(&c->vc.chan.device_node);
1325                 tasklet_kill(&c->vc.task);
1326                 kfree(c);
1327         }
1328 }
1329
1330 #define OMAP_DMA_BUSWIDTHS      (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
1331                                  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
1332                                  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
1333
1334 static int omap_dma_probe(struct platform_device *pdev)
1335 {
1336         struct omap_dmadev *od;
1337         struct resource *res;
1338         int rc, i, irq;
1339
1340         od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
1341         if (!od)
1342                 return -ENOMEM;
1343
1344         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1345         od->base = devm_ioremap_resource(&pdev->dev, res);
1346         if (IS_ERR(od->base))
1347                 return PTR_ERR(od->base);
1348
1349         od->plat = omap_get_plat_info();
1350         if (!od->plat)
1351                 return -EPROBE_DEFER;
1352
1353         od->reg_map = od->plat->reg_map;
1354
1355         dma_cap_set(DMA_SLAVE, od->ddev.cap_mask);
1356         dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask);
1357         dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask);
1358         dma_cap_set(DMA_INTERLEAVE, od->ddev.cap_mask);
1359         od->ddev.device_alloc_chan_resources = omap_dma_alloc_chan_resources;
1360         od->ddev.device_free_chan_resources = omap_dma_free_chan_resources;
1361         od->ddev.device_tx_status = omap_dma_tx_status;
1362         od->ddev.device_issue_pending = omap_dma_issue_pending;
1363         od->ddev.device_prep_slave_sg = omap_dma_prep_slave_sg;
1364         od->ddev.device_prep_dma_cyclic = omap_dma_prep_dma_cyclic;
1365         od->ddev.device_prep_dma_memcpy = omap_dma_prep_dma_memcpy;
1366         od->ddev.device_prep_interleaved_dma = omap_dma_prep_dma_interleaved;
1367         od->ddev.device_config = omap_dma_slave_config;
1368         od->ddev.device_pause = omap_dma_pause;
1369         od->ddev.device_resume = omap_dma_resume;
1370         od->ddev.device_terminate_all = omap_dma_terminate_all;
1371         od->ddev.device_synchronize = omap_dma_synchronize;
1372         od->ddev.src_addr_widths = OMAP_DMA_BUSWIDTHS;
1373         od->ddev.dst_addr_widths = OMAP_DMA_BUSWIDTHS;
1374         od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1375         od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1376         od->ddev.dev = &pdev->dev;
1377         INIT_LIST_HEAD(&od->ddev.channels);
1378         spin_lock_init(&od->lock);
1379         spin_lock_init(&od->irq_lock);
1380
1381         if (!pdev->dev.of_node) {
1382                 od->dma_requests = od->plat->dma_attr->lch_count;
1383                 if (unlikely(!od->dma_requests))
1384                         od->dma_requests = OMAP_SDMA_REQUESTS;
1385         } else if (of_property_read_u32(pdev->dev.of_node, "dma-requests",
1386                                         &od->dma_requests)) {
1387                 dev_info(&pdev->dev,
1388                          "Missing dma-requests property, using %u.\n",
1389                          OMAP_SDMA_REQUESTS);
1390                 od->dma_requests = OMAP_SDMA_REQUESTS;
1391         }
1392
1393         od->lch_map = devm_kcalloc(&pdev->dev, od->dma_requests,
1394                                    sizeof(*od->lch_map), GFP_KERNEL);
1395         if (!od->lch_map)
1396                 return -ENOMEM;
1397
1398         for (i = 0; i < od->dma_requests; i++) {
1399                 rc = omap_dma_chan_init(od);
1400                 if (rc) {
1401                         omap_dma_free(od);
1402                         return rc;
1403                 }
1404         }
1405
1406         irq = platform_get_irq(pdev, 1);
1407         if (irq <= 0) {
1408                 dev_info(&pdev->dev, "failed to get L1 IRQ: %d\n", irq);
1409                 od->legacy = true;
1410         } else {
1411                 /* Disable all interrupts */
1412                 od->irq_enable_mask = 0;
1413                 omap_dma_glbl_write(od, IRQENABLE_L1, 0);
1414
1415                 rc = devm_request_irq(&pdev->dev, irq, omap_dma_irq,
1416                                       IRQF_SHARED, "omap-dma-engine", od);
1417                 if (rc)
1418                         return rc;
1419         }
1420
1421         if (omap_dma_glbl_read(od, CAPS_0) & CAPS_0_SUPPORT_LL123)
1422                 od->ll123_supported = true;
1423
1424         od->ddev.filter.map = od->plat->slave_map;
1425         od->ddev.filter.mapcnt = od->plat->slavecnt;
1426         od->ddev.filter.fn = omap_dma_filter_fn;
1427
1428         if (od->ll123_supported) {
1429                 od->desc_pool = dma_pool_create(dev_name(&pdev->dev),
1430                                                 &pdev->dev,
1431                                                 sizeof(struct omap_type2_desc),
1432                                                 4, 0);
1433                 if (!od->desc_pool) {
1434                         dev_err(&pdev->dev,
1435                                 "unable to allocate descriptor pool\n");
1436                         od->ll123_supported = false;
1437                 }
1438         }
1439
1440         rc = dma_async_device_register(&od->ddev);
1441         if (rc) {
1442                 pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n",
1443                         rc);
1444                 omap_dma_free(od);
1445                 return rc;
1446         }
1447
1448         platform_set_drvdata(pdev, od);
1449
1450         if (pdev->dev.of_node) {
1451                 omap_dma_info.dma_cap = od->ddev.cap_mask;
1452
1453                 /* Device-tree DMA controller registration */
1454                 rc = of_dma_controller_register(pdev->dev.of_node,
1455                                 of_dma_simple_xlate, &omap_dma_info);
1456                 if (rc) {
1457                         pr_warn("OMAP-DMA: failed to register DMA controller\n");
1458                         dma_async_device_unregister(&od->ddev);
1459                         omap_dma_free(od);
1460                 }
1461         }
1462
1463         dev_info(&pdev->dev, "OMAP DMA engine driver%s\n",
1464                  od->ll123_supported ? " (LinkedList1/2/3 supported)" : "");
1465
1466         return rc;
1467 }
1468
1469 static int omap_dma_remove(struct platform_device *pdev)
1470 {
1471         struct omap_dmadev *od = platform_get_drvdata(pdev);
1472         int irq;
1473
1474         if (pdev->dev.of_node)
1475                 of_dma_controller_free(pdev->dev.of_node);
1476
1477         irq = platform_get_irq(pdev, 1);
1478         devm_free_irq(&pdev->dev, irq, od);
1479
1480         dma_async_device_unregister(&od->ddev);
1481
1482         if (!od->legacy) {
1483                 /* Disable all interrupts */
1484                 omap_dma_glbl_write(od, IRQENABLE_L0, 0);
1485         }
1486
1487         if (od->ll123_supported)
1488                 dma_pool_destroy(od->desc_pool);
1489
1490         omap_dma_free(od);
1491
1492         return 0;
1493 }
1494
1495 static const struct of_device_id omap_dma_match[] = {
1496         { .compatible = "ti,omap2420-sdma", },
1497         { .compatible = "ti,omap2430-sdma", },
1498         { .compatible = "ti,omap3430-sdma", },
1499         { .compatible = "ti,omap3630-sdma", },
1500         { .compatible = "ti,omap4430-sdma", },
1501         {},
1502 };
1503 MODULE_DEVICE_TABLE(of, omap_dma_match);
1504
1505 static struct platform_driver omap_dma_driver = {
1506         .probe  = omap_dma_probe,
1507         .remove = omap_dma_remove,
1508         .driver = {
1509                 .name = "omap-dma-engine",
1510                 .of_match_table = of_match_ptr(omap_dma_match),
1511         },
1512 };
1513
1514 bool omap_dma_filter_fn(struct dma_chan *chan, void *param)
1515 {
1516         if (chan->device->dev->driver == &omap_dma_driver.driver) {
1517                 struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1518                 struct omap_chan *c = to_omap_dma_chan(chan);
1519                 unsigned req = *(unsigned *)param;
1520
1521                 if (req <= od->dma_requests) {
1522                         c->dma_sig = req;
1523                         return true;
1524                 }
1525         }
1526         return false;
1527 }
1528 EXPORT_SYMBOL_GPL(omap_dma_filter_fn);
1529
1530 static int omap_dma_init(void)
1531 {
1532         return platform_driver_register(&omap_dma_driver);
1533 }
1534 subsys_initcall(omap_dma_init);
1535
1536 static void __exit omap_dma_exit(void)
1537 {
1538         platform_driver_unregister(&omap_dma_driver);
1539 }
1540 module_exit(omap_dma_exit);
1541
1542 MODULE_AUTHOR("Russell King");
1543 MODULE_LICENSE("GPL");