drivers/dma/omap-dma.c

   1 /*
   2  * OMAP DMAengine support
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  */
   8 #include <linux/delay.h>
   9 #include <linux/dmaengine.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/err.h>
  12 #include <linux/init.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/list.h>
  15 #include <linux/module.h>
  16 #include <linux/omap-dma.h>
  17 #include <linux/platform_device.h>
  18 #include <linux/slab.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/of_dma.h>
  21 #include <linux/of_device.h>
  22
  23 #include "virt-dma.h"
  24
  25 #define OMAP_SDMA_REQUESTS      127
  26 #define OMAP_SDMA_CHANNELS      32
  27
  28 struct omap_dmadev {
  29         struct dma_device ddev;
  30         spinlock_t lock;
  31         void __iomem *base;
  32         const struct omap_dma_reg *reg_map;
  33         struct omap_system_dma_plat_info *plat;
  34         bool legacy;
  35         unsigned dma_requests;
  36         spinlock_t irq_lock;
  37         uint32_t irq_enable_mask;
  38         struct omap_chan *lch_map[OMAP_SDMA_CHANNELS];
  39 };
  40
  41 struct omap_chan {
  42         struct virt_dma_chan vc;
  43         void __iomem *channel_base;
  44         const struct omap_dma_reg *reg_map;
  45         uint32_t ccr;
  46
  47         struct dma_slave_config cfg;
  48         unsigned dma_sig;
  49         bool cyclic;
  50         bool paused;
  51         bool running;
  52
  53         int dma_ch;
  54         struct omap_desc *desc;
  55         unsigned sgidx;
  56 };
  57
  58 struct omap_sg {
  59         dma_addr_t addr;
  60         uint32_t en;            /* number of elements (24-bit) */
  61         uint32_t fn;            /* number of frames (16-bit) */
  62         int32_t fi;             /* for double indexing */
  63         int16_t ei;             /* for double indexing */
  64 };
  65
  66 struct omap_desc {
  67         struct virt_dma_desc vd;
  68         enum dma_transfer_direction dir;
  69         dma_addr_t dev_addr;
  70
  71         int32_t fi;             /* for OMAP_DMA_SYNC_PACKET / double indexing */
  72         int16_t ei;             /* for double indexing */
  73         uint8_t es;             /* CSDP_DATA_TYPE_xxx */
  74         uint32_t ccr;           /* CCR value */
  75         uint16_t clnk_ctrl;     /* CLNK_CTRL value */
  76         uint16_t cicr;          /* CICR value */
  77         uint32_t csdp;          /* CSDP value */
  78
  79         unsigned sglen;
  80         struct omap_sg sg[0];
  81 };
  82
  83 enum {
  84         CCR_FS                  = BIT(5),
  85         CCR_READ_PRIORITY       = BIT(6),
  86         CCR_ENABLE              = BIT(7),
  87         CCR_AUTO_INIT           = BIT(8),       /* OMAP1 only */
  88         CCR_REPEAT              = BIT(9),       /* OMAP1 only */
  89         CCR_OMAP31_DISABLE      = BIT(10),      /* OMAP1 only */
  90         CCR_SUSPEND_SENSITIVE   = BIT(8),       /* OMAP2+ only */
  91         CCR_RD_ACTIVE           = BIT(9),       /* OMAP2+ only */
  92         CCR_WR_ACTIVE           = BIT(10),      /* OMAP2+ only */
  93         CCR_SRC_AMODE_CONSTANT  = 0 << 12,
  94         CCR_SRC_AMODE_POSTINC   = 1 << 12,
  95         CCR_SRC_AMODE_SGLIDX    = 2 << 12,
  96         CCR_SRC_AMODE_DBLIDX    = 3 << 12,
  97         CCR_DST_AMODE_CONSTANT  = 0 << 14,
  98         CCR_DST_AMODE_POSTINC   = 1 << 14,
  99         CCR_DST_AMODE_SGLIDX    = 2 << 14,
 100         CCR_DST_AMODE_DBLIDX    = 3 << 14,
 101         CCR_CONSTANT_FILL       = BIT(16),
 102         CCR_TRANSPARENT_COPY    = BIT(17),
 103         CCR_BS                  = BIT(18),
 104         CCR_SUPERVISOR          = BIT(22),
 105         CCR_PREFETCH            = BIT(23),
 106         CCR_TRIGGER_SRC         = BIT(24),
 107         CCR_BUFFERING_DISABLE   = BIT(25),
 108         CCR_WRITE_PRIORITY      = BIT(26),
 109         CCR_SYNC_ELEMENT        = 0,
 110         CCR_SYNC_FRAME          = CCR_FS,
 111         CCR_SYNC_BLOCK          = CCR_BS,
 112         CCR_SYNC_PACKET         = CCR_BS | CCR_FS,
 113
 114         CSDP_DATA_TYPE_8        = 0,
 115         CSDP_DATA_TYPE_16       = 1,
 116         CSDP_DATA_TYPE_32       = 2,
 117         CSDP_SRC_PORT_EMIFF     = 0 << 2, /* OMAP1 only */
 118         CSDP_SRC_PORT_EMIFS     = 1 << 2, /* OMAP1 only */
 119         CSDP_SRC_PORT_OCP_T1    = 2 << 2, /* OMAP1 only */
 120         CSDP_SRC_PORT_TIPB      = 3 << 2, /* OMAP1 only */
 121         CSDP_SRC_PORT_OCP_T2    = 4 << 2, /* OMAP1 only */
 122         CSDP_SRC_PORT_MPUI      = 5 << 2, /* OMAP1 only */
 123         CSDP_SRC_PACKED         = BIT(6),
 124         CSDP_SRC_BURST_1        = 0 << 7,
 125         CSDP_SRC_BURST_16       = 1 << 7,
 126         CSDP_SRC_BURST_32       = 2 << 7,
 127         CSDP_SRC_BURST_64       = 3 << 7,
 128         CSDP_DST_PORT_EMIFF     = 0 << 9, /* OMAP1 only */
 129         CSDP_DST_PORT_EMIFS     = 1 << 9, /* OMAP1 only */
 130         CSDP_DST_PORT_OCP_T1    = 2 << 9, /* OMAP1 only */
 131         CSDP_DST_PORT_TIPB      = 3 << 9, /* OMAP1 only */
 132         CSDP_DST_PORT_OCP_T2    = 4 << 9, /* OMAP1 only */
 133         CSDP_DST_PORT_MPUI      = 5 << 9, /* OMAP1 only */
 134         CSDP_DST_PACKED         = BIT(13),
 135         CSDP_DST_BURST_1        = 0 << 14,
 136         CSDP_DST_BURST_16       = 1 << 14,
 137         CSDP_DST_BURST_32       = 2 << 14,
 138         CSDP_DST_BURST_64       = 3 << 14,
 139
 140         CICR_TOUT_IE            = BIT(0),       /* OMAP1 only */
 141         CICR_DROP_IE            = BIT(1),
 142         CICR_HALF_IE            = BIT(2),
 143         CICR_FRAME_IE           = BIT(3),
 144         CICR_LAST_IE            = BIT(4),
 145         CICR_BLOCK_IE           = BIT(5),
 146         CICR_PKT_IE             = BIT(7),       /* OMAP2+ only */
 147         CICR_TRANS_ERR_IE       = BIT(8),       /* OMAP2+ only */
 148         CICR_SUPERVISOR_ERR_IE  = BIT(10),      /* OMAP2+ only */
 149         CICR_MISALIGNED_ERR_IE  = BIT(11),      /* OMAP2+ only */
 150         CICR_DRAIN_IE           = BIT(12),      /* OMAP2+ only */
 151         CICR_SUPER_BLOCK_IE     = BIT(14),      /* OMAP2+ only */
 152
 153         CLNK_CTRL_ENABLE_LNK    = BIT(15),
 154 };
 155
 156 static const unsigned es_bytes[] = {
 157         [CSDP_DATA_TYPE_8] = 1,
 158         [CSDP_DATA_TYPE_16] = 2,
 159         [CSDP_DATA_TYPE_32] = 4,
 160 };
 161
 162 static struct of_dma_filter_info omap_dma_info = {
 163         .filter_fn = omap_dma_filter_fn,
 164 };
 165
 166 static inline struct omap_dmadev *to_omap_dma_dev(struct dma_device *d)
 167 {
 168         return container_of(d, struct omap_dmadev, ddev);
 169 }
 170
 171 static inline struct omap_chan *to_omap_dma_chan(struct dma_chan *c)
 172 {
 173         return container_of(c, struct omap_chan, vc.chan);
 174 }
 175
 176 static inline struct omap_desc *to_omap_dma_desc(struct dma_async_tx_descriptor *t)
 177 {
 178         return container_of(t, struct omap_desc, vd.tx);
 179 }
 180
 181 static void omap_dma_desc_free(struct virt_dma_desc *vd)
 182 {
 183         kfree(container_of(vd, struct omap_desc, vd));
 184 }
 185
 186 static void omap_dma_write(uint32_t val, unsigned type, void __iomem *addr)
 187 {
 188         switch (type) {
 189         case OMAP_DMA_REG_16BIT:
 190                 writew_relaxed(val, addr);
 191                 break;
 192         case OMAP_DMA_REG_2X16BIT:
 193                 writew_relaxed(val, addr);
 194                 writew_relaxed(val >> 16, addr + 2);
 195                 break;
 196         case OMAP_DMA_REG_32BIT:
 197                 writel_relaxed(val, addr);
 198                 break;
 199         default:
 200                 WARN_ON(1);
 201         }
 202 }
 203
 204 static unsigned omap_dma_read(unsigned type, void __iomem *addr)
 205 {
 206         unsigned val;
 207
 208         switch (type) {
 209         case OMAP_DMA_REG_16BIT:
 210                 val = readw_relaxed(addr);
 211                 break;
 212         case OMAP_DMA_REG_2X16BIT:
 213                 val = readw_relaxed(addr);
 214                 val |= readw_relaxed(addr + 2) << 16;
 215                 break;
 216         case OMAP_DMA_REG_32BIT:
 217                 val = readl_relaxed(addr);
 218                 break;
 219         default:
 220                 WARN_ON(1);
 221                 val = 0;
 222         }
 223
 224         return val;
 225 }
 226
 227 static void omap_dma_glbl_write(struct omap_dmadev *od, unsigned reg, unsigned val)
 228 {
 229         const struct omap_dma_reg *r = od->reg_map + reg;
 230
 231         WARN_ON(r->stride);
 232
 233         omap_dma_write(val, r->type, od->base + r->offset);
 234 }
 235
 236 static unsigned omap_dma_glbl_read(struct omap_dmadev *od, unsigned reg)
 237 {
 238         const struct omap_dma_reg *r = od->reg_map + reg;
 239
 240         WARN_ON(r->stride);
 241
 242         return omap_dma_read(r->type, od->base + r->offset);
 243 }
 244
 245 static void omap_dma_chan_write(struct omap_chan *c, unsigned reg, unsigned val)
 246 {
 247         const struct omap_dma_reg *r = c->reg_map + reg;
 248
 249         omap_dma_write(val, r->type, c->channel_base + r->offset);
 250 }
 251
 252 static unsigned omap_dma_chan_read(struct omap_chan *c, unsigned reg)
 253 {
 254         const struct omap_dma_reg *r = c->reg_map + reg;
 255
 256         return omap_dma_read(r->type, c->channel_base + r->offset);
 257 }
 258
 259 static void omap_dma_clear_csr(struct omap_chan *c)
 260 {
 261         if (dma_omap1())
 262                 omap_dma_chan_read(c, CSR);
 263         else
 264                 omap_dma_chan_write(c, CSR, ~0);
 265 }
 266
 267 static unsigned omap_dma_get_csr(struct omap_chan *c)
 268 {
 269         unsigned val = omap_dma_chan_read(c, CSR);
 270
 271         if (!dma_omap1())
 272                 omap_dma_chan_write(c, CSR, val);
 273
 274         return val;
 275 }
 276
 277 static void omap_dma_assign(struct omap_dmadev *od, struct omap_chan *c,
 278         unsigned lch)
 279 {
 280         c->channel_base = od->base + od->plat->channel_stride * lch;
 281
 282         od->lch_map[lch] = c;
 283 }
 284
 285 static void omap_dma_start(struct omap_chan *c, struct omap_desc *d)
 286 {
 287         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 288
 289         if (__dma_omap15xx(od->plat->dma_attr))
 290                 omap_dma_chan_write(c, CPC, 0);
 291         else
 292                 omap_dma_chan_write(c, CDAC, 0);
 293
 294         omap_dma_clear_csr(c);
 295
 296         /* Enable interrupts */
 297         omap_dma_chan_write(c, CICR, d->cicr);
 298
 299         /* Enable channel */
 300         omap_dma_chan_write(c, CCR, d->ccr | CCR_ENABLE);
 301
 302         c->running = true;
 303 }
 304
 305 static void omap_dma_stop(struct omap_chan *c)
 306 {
 307         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 308         uint32_t val;
 309
 310         /* disable irq */
 311         omap_dma_chan_write(c, CICR, 0);
 312
 313         omap_dma_clear_csr(c);
 314
 315         val = omap_dma_chan_read(c, CCR);
 316         if (od->plat->errata & DMA_ERRATA_i541 && val & CCR_TRIGGER_SRC) {
 317                 uint32_t sysconfig;
 318                 unsigned i;
 319
 320                 sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG);
 321                 val = sysconfig & ~DMA_SYSCONFIG_MIDLEMODE_MASK;
 322                 val |= DMA_SYSCONFIG_MIDLEMODE(DMA_IDLEMODE_NO_IDLE);
 323                 omap_dma_glbl_write(od, OCP_SYSCONFIG, val);
 324
 325                 val = omap_dma_chan_read(c, CCR);
 326                 val &= ~CCR_ENABLE;
 327                 omap_dma_chan_write(c, CCR, val);
 328
 329                 /* Wait for sDMA FIFO to drain */
 330                 for (i = 0; ; i++) {
 331                         val = omap_dma_chan_read(c, CCR);
 332                         if (!(val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE)))
 333                                 break;
 334
 335                         if (i > 100)
 336                                 break;
 337
 338                         udelay(5);
 339                 }
 340
 341                 if (val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE))
 342                         dev_err(c->vc.chan.device->dev,
 343                                 "DMA drain did not complete on lch %d\n",
 344                                 c->dma_ch);
 345
 346                 omap_dma_glbl_write(od, OCP_SYSCONFIG, sysconfig);
 347         } else {
 348                 val &= ~CCR_ENABLE;
 349                 omap_dma_chan_write(c, CCR, val);
 350         }
 351
 352         mb();
 353
 354         if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) {
 355                 val = omap_dma_chan_read(c, CLNK_CTRL);
 356
 357                 if (dma_omap1())
 358                         val |= 1 << 14; /* set the STOP_LNK bit */
 359                 else
 360                         val &= ~CLNK_CTRL_ENABLE_LNK;
 361
 362                 omap_dma_chan_write(c, CLNK_CTRL, val);
 363         }
 364
 365         c->running = false;
 366 }
 367
 368 static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d)
 369 {
 370         struct omap_sg *sg = d->sg + c->sgidx;
 371         unsigned cxsa, cxei, cxfi;
 372
 373         if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
 374                 cxsa = CDSA;
 375                 cxei = CDEI;
 376                 cxfi = CDFI;
 377         } else {
 378                 cxsa = CSSA;
 379                 cxei = CSEI;
 380                 cxfi = CSFI;
 381         }
 382
 383         omap_dma_chan_write(c, cxsa, sg->addr);
 384         omap_dma_chan_write(c, cxei, sg->ei);
 385         omap_dma_chan_write(c, cxfi, sg->fi);
 386         omap_dma_chan_write(c, CEN, sg->en);
 387         omap_dma_chan_write(c, CFN, sg->fn);
 388
 389         omap_dma_start(c, d);
 390         c->sgidx++;
 391 }
 392
 393 static void omap_dma_start_desc(struct omap_chan *c)
 394 {
 395         struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
 396         struct omap_desc *d;
 397         unsigned cxsa, cxei, cxfi;
 398
 399         if (!vd) {
 400                 c->desc = NULL;
 401                 return;
 402         }
 403
 404         list_del(&vd->node);
 405
 406         c->desc = d = to_omap_dma_desc(&vd->tx);
 407         c->sgidx = 0;
 408
 409         /*
 410          * This provides the necessary barrier to ensure data held in
 411          * DMA coherent memory is visible to the DMA engine prior to
 412          * the transfer starting.
 413          */
 414         mb();
 415
 416         omap_dma_chan_write(c, CCR, d->ccr);
 417         if (dma_omap1())
 418                 omap_dma_chan_write(c, CCR2, d->ccr >> 16);
 419
 420         if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
 421                 cxsa = CSSA;
 422                 cxei = CSEI;
 423                 cxfi = CSFI;
 424         } else {
 425                 cxsa = CDSA;
 426                 cxei = CDEI;
 427                 cxfi = CDFI;
 428         }
 429
 430         omap_dma_chan_write(c, cxsa, d->dev_addr);
 431         omap_dma_chan_write(c, cxei, d->ei);
 432         omap_dma_chan_write(c, cxfi, d->fi);
 433         omap_dma_chan_write(c, CSDP, d->csdp);
 434         omap_dma_chan_write(c, CLNK_CTRL, d->clnk_ctrl);
 435
 436         omap_dma_start_sg(c, d);
 437 }
 438
 439 static void omap_dma_callback(int ch, u16 status, void *data)
 440 {
 441         struct omap_chan *c = data;
 442         struct omap_desc *d;
 443         unsigned long flags;
 444
 445         spin_lock_irqsave(&c->vc.lock, flags);
 446         d = c->desc;
 447         if (d) {
 448                 if (c->cyclic) {
 449                         vchan_cyclic_callback(&d->vd);
 450                 } else if (c->sgidx == d->sglen) {
 451                         omap_dma_start_desc(c);
 452                         vchan_cookie_complete(&d->vd);
 453                 } else {
 454                         omap_dma_start_sg(c, d);
 455                 }
 456         }
 457         spin_unlock_irqrestore(&c->vc.lock, flags);
 458 }
 459
 460 static irqreturn_t omap_dma_irq(int irq, void *devid)
 461 {
 462         struct omap_dmadev *od = devid;
 463         unsigned status, channel;
 464
 465         spin_lock(&od->irq_lock);
 466
 467         status = omap_dma_glbl_read(od, IRQSTATUS_L1);
 468         status &= od->irq_enable_mask;
 469         if (status == 0) {
 470                 spin_unlock(&od->irq_lock);
 471                 return IRQ_NONE;
 472         }
 473
 474         while ((channel = ffs(status)) != 0) {
 475                 unsigned mask, csr;
 476                 struct omap_chan *c;
 477
 478                 channel -= 1;
 479                 mask = BIT(channel);
 480                 status &= ~mask;
 481
 482                 c = od->lch_map[channel];
 483                 if (c == NULL) {
 484                         /* This should never happen */
 485                         dev_err(od->ddev.dev, "invalid channel %u\n", channel);
 486                         continue;
 487                 }
 488
 489                 csr = omap_dma_get_csr(c);
 490                 omap_dma_glbl_write(od, IRQSTATUS_L1, mask);
 491
 492                 omap_dma_callback(channel, csr, c);
 493         }
 494
 495         spin_unlock(&od->irq_lock);
 496
 497         return IRQ_HANDLED;
 498 }
 499
 500 static int omap_dma_alloc_chan_resources(struct dma_chan *chan)
 501 {
 502         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 503         struct omap_chan *c = to_omap_dma_chan(chan);
 504         int ret;
 505
 506         if (od->legacy) {
 507                 ret = omap_request_dma(c->dma_sig, "DMA engine",
 508                                        omap_dma_callback, c, &c->dma_ch);
 509         } else {
 510                 ret = omap_request_dma(c->dma_sig, "DMA engine", NULL, NULL,
 511                                        &c->dma_ch);
 512         }
 513
 514         dev_dbg(od->ddev.dev, "allocating channel %u for %u\n",
 515                 c->dma_ch, c->dma_sig);
 516
 517         if (ret >= 0) {
 518                 omap_dma_assign(od, c, c->dma_ch);
 519
 520                 if (!od->legacy) {
 521                         unsigned val;
 522
 523                         spin_lock_irq(&od->irq_lock);
 524                         val = BIT(c->dma_ch);
 525                         omap_dma_glbl_write(od, IRQSTATUS_L1, val);
 526                         od->irq_enable_mask |= val;
 527                         omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
 528
 529                         val = omap_dma_glbl_read(od, IRQENABLE_L0);
 530                         val &= ~BIT(c->dma_ch);
 531                         omap_dma_glbl_write(od, IRQENABLE_L0, val);
 532                         spin_unlock_irq(&od->irq_lock);
 533                 }
 534         }
 535
 536         if (dma_omap1()) {
 537                 if (__dma_omap16xx(od->plat->dma_attr)) {
 538                         c->ccr = CCR_OMAP31_DISABLE;
 539                         /* Duplicate what plat-omap/dma.c does */
 540                         c->ccr |= c->dma_ch + 1;
 541                 } else {
 542                         c->ccr = c->dma_sig & 0x1f;
 543                 }
 544         } else {
 545                 c->ccr = c->dma_sig & 0x1f;
 546                 c->ccr |= (c->dma_sig & ~0x1f) << 14;
 547         }
 548         if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING)
 549                 c->ccr |= CCR_BUFFERING_DISABLE;
 550
 551         return ret;
 552 }
 553
 554 static void omap_dma_free_chan_resources(struct dma_chan *chan)
 555 {
 556         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 557         struct omap_chan *c = to_omap_dma_chan(chan);
 558
 559         if (!od->legacy) {
 560                 spin_lock_irq(&od->irq_lock);
 561                 od->irq_enable_mask &= ~BIT(c->dma_ch);
 562                 omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
 563                 spin_unlock_irq(&od->irq_lock);
 564         }
 565
 566         c->channel_base = NULL;
 567         od->lch_map[c->dma_ch] = NULL;
 568         vchan_free_chan_resources(&c->vc);
 569         omap_free_dma(c->dma_ch);
 570
 571         dev_dbg(od->ddev.dev, "freeing channel for %u\n", c->dma_sig);
 572         c->dma_sig = 0;
 573 }
 574
 575 static size_t omap_dma_sg_size(struct omap_sg *sg)
 576 {
 577         return sg->en * sg->fn;
 578 }
 579
 580 static size_t omap_dma_desc_size(struct omap_desc *d)
 581 {
 582         unsigned i;
 583         size_t size;
 584
 585         for (size = i = 0; i < d->sglen; i++)
 586                 size += omap_dma_sg_size(&d->sg[i]);
 587
 588         return size * es_bytes[d->es];
 589 }
 590
 591 static size_t omap_dma_desc_size_pos(struct omap_desc *d, dma_addr_t addr)
 592 {
 593         unsigned i;
 594         size_t size, es_size = es_bytes[d->es];
 595
 596         for (size = i = 0; i < d->sglen; i++) {
 597                 size_t this_size = omap_dma_sg_size(&d->sg[i]) * es_size;
 598
 599                 if (size)
 600                         size += this_size;
 601                 else if (addr >= d->sg[i].addr &&
 602                          addr < d->sg[i].addr + this_size)
 603                         size += d->sg[i].addr + this_size - addr;
 604         }
 605         return size;
 606 }
 607
 608 /*
 609  * OMAP 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is
 610  * read before the DMA controller finished disabling the channel.
 611  */
 612 static uint32_t omap_dma_chan_read_3_3(struct omap_chan *c, unsigned reg)
 613 {
 614         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 615         uint32_t val;
 616
 617         val = omap_dma_chan_read(c, reg);
 618         if (val == 0 && od->plat->errata & DMA_ERRATA_3_3)
 619                 val = omap_dma_chan_read(c, reg);
 620
 621         return val;
 622 }
 623
 624 static dma_addr_t omap_dma_get_src_pos(struct omap_chan *c)
 625 {
 626         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 627         dma_addr_t addr, cdac;
 628
 629         if (__dma_omap15xx(od->plat->dma_attr)) {
 630                 addr = omap_dma_chan_read(c, CPC);
 631         } else {
 632                 addr = omap_dma_chan_read_3_3(c, CSAC);
 633                 cdac = omap_dma_chan_read_3_3(c, CDAC);
 634
 635                 /*
 636                  * CDAC == 0 indicates that the DMA transfer on the channel has
 637                  * not been started (no data has been transferred so far).
 638                  * Return the programmed source start address in this case.
 639                  */
 640                 if (cdac == 0)
 641                         addr = omap_dma_chan_read(c, CSSA);
 642         }
 643
 644         if (dma_omap1())
 645                 addr |= omap_dma_chan_read(c, CSSA) & 0xffff0000;
 646
 647         return addr;
 648 }
 649
 650 static dma_addr_t omap_dma_get_dst_pos(struct omap_chan *c)
 651 {
 652         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 653         dma_addr_t addr;
 654
 655         if (__dma_omap15xx(od->plat->dma_attr)) {
 656                 addr = omap_dma_chan_read(c, CPC);
 657         } else {
 658                 addr = omap_dma_chan_read_3_3(c, CDAC);
 659
 660                 /*
 661                  * CDAC == 0 indicates that the DMA transfer on the channel
 662                  * has not been started (no data has been transferred so
 663                  * far).  Return the programmed destination start address in
 664                  * this case.
 665                  */
 666                 if (addr == 0)
 667                         addr = omap_dma_chan_read(c, CDSA);
 668         }
 669
 670         if (dma_omap1())
 671                 addr |= omap_dma_chan_read(c, CDSA) & 0xffff0000;
 672
 673         return addr;
 674 }
 675
 676 static enum dma_status omap_dma_tx_status(struct dma_chan *chan,
 677         dma_cookie_t cookie, struct dma_tx_state *txstate)
 678 {
 679         struct omap_chan *c = to_omap_dma_chan(chan);
 680         struct virt_dma_desc *vd;
 681         enum dma_status ret;
 682         unsigned long flags;
 683
 684         ret = dma_cookie_status(chan, cookie, txstate);
 685
 686         if (!c->paused && c->running) {
 687                 uint32_t ccr = omap_dma_chan_read(c, CCR);
 688                 /*
 689                  * The channel is no longer active, set the return value
 690                  * accordingly
 691                  */
 692                 if (!(ccr & CCR_ENABLE))
 693                         ret = DMA_COMPLETE;
 694         }
 695
 696         if (ret == DMA_COMPLETE || !txstate)
 697                 return ret;
 698
 699         spin_lock_irqsave(&c->vc.lock, flags);
 700         vd = vchan_find_desc(&c->vc, cookie);
 701         if (vd) {
 702                 txstate->residue = omap_dma_desc_size(to_omap_dma_desc(&vd->tx));
 703         } else if (c->desc && c->desc->vd.tx.cookie == cookie) {
 704                 struct omap_desc *d = c->desc;
 705                 dma_addr_t pos;
 706
 707                 if (d->dir == DMA_MEM_TO_DEV)
 708                         pos = omap_dma_get_src_pos(c);
 709                 else if (d->dir == DMA_DEV_TO_MEM  || d->dir == DMA_MEM_TO_MEM)
 710                         pos = omap_dma_get_dst_pos(c);
 711                 else
 712                         pos = 0;
 713
 714                 txstate->residue = omap_dma_desc_size_pos(d, pos);
 715         } else {
 716                 txstate->residue = 0;
 717         }
 718         spin_unlock_irqrestore(&c->vc.lock, flags);
 719
 720         return ret;
 721 }
 722
 723 static void omap_dma_issue_pending(struct dma_chan *chan)
 724 {
 725         struct omap_chan *c = to_omap_dma_chan(chan);
 726         unsigned long flags;
 727
 728         spin_lock_irqsave(&c->vc.lock, flags);
 729         if (vchan_issue_pending(&c->vc) && !c->desc)
 730                 omap_dma_start_desc(c);
 731         spin_unlock_irqrestore(&c->vc.lock, flags);
 732 }
 733
 734 static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
 735         struct dma_chan *chan, struct scatterlist *sgl, unsigned sglen,
 736         enum dma_transfer_direction dir, unsigned long tx_flags, void *context)
 737 {
 738         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 739         struct omap_chan *c = to_omap_dma_chan(chan);
 740         enum dma_slave_buswidth dev_width;
 741         struct scatterlist *sgent;
 742         struct omap_desc *d;
 743         dma_addr_t dev_addr;
 744         unsigned i, es, en, frame_bytes;
 745         u32 burst;
 746
 747         if (dir == DMA_DEV_TO_MEM) {
 748                 dev_addr = c->cfg.src_addr;
 749                 dev_width = c->cfg.src_addr_width;
 750                 burst = c->cfg.src_maxburst;
 751         } else if (dir == DMA_MEM_TO_DEV) {
 752                 dev_addr = c->cfg.dst_addr;
 753                 dev_width = c->cfg.dst_addr_width;
 754                 burst = c->cfg.dst_maxburst;
 755         } else {
 756                 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
 757                 return NULL;
 758         }
 759
 760         /* Bus width translates to the element size (ES) */
 761         switch (dev_width) {
 762         case DMA_SLAVE_BUSWIDTH_1_BYTE:
 763                 es = CSDP_DATA_TYPE_8;
 764                 break;
 765         case DMA_SLAVE_BUSWIDTH_2_BYTES:
 766                 es = CSDP_DATA_TYPE_16;
 767                 break;
 768         case DMA_SLAVE_BUSWIDTH_4_BYTES:
 769                 es = CSDP_DATA_TYPE_32;
 770                 break;
 771         default: /* not reached */
 772                 return NULL;
 773         }
 774
 775         /* Now allocate and setup the descriptor. */
 776         d = kzalloc(sizeof(*d) + sglen * sizeof(d->sg[0]), GFP_ATOMIC);
 777         if (!d)
 778                 return NULL;
 779
 780         d->dir = dir;
 781         d->dev_addr = dev_addr;
 782         d->es = es;
 783
 784         d->ccr = c->ccr | CCR_SYNC_FRAME;
 785         if (dir == DMA_DEV_TO_MEM)
 786                 d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT;
 787         else
 788                 d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC;
 789
 790         d->cicr = CICR_DROP_IE | CICR_BLOCK_IE;
 791         d->csdp = es;
 792
 793         if (dma_omap1()) {
 794                 d->cicr |= CICR_TOUT_IE;
 795
 796                 if (dir == DMA_DEV_TO_MEM)
 797                         d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_TIPB;
 798                 else
 799                         d->csdp |= CSDP_DST_PORT_TIPB | CSDP_SRC_PORT_EMIFF;
 800         } else {
 801                 if (dir == DMA_DEV_TO_MEM)
 802                         d->ccr |= CCR_TRIGGER_SRC;
 803
 804                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
 805         }
 806         if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS)
 807                 d->clnk_ctrl = c->dma_ch;
 808
 809         /*
 810          * Build our scatterlist entries: each contains the address,
 811          * the number of elements (EN) in each frame, and the number of
 812          * frames (FN).  Number of bytes for this entry = ES * EN * FN.
 813          *
 814          * Burst size translates to number of elements with frame sync.
 815          * Note: DMA engine defines burst to be the number of dev-width
 816          * transfers.
 817          */
 818         en = burst;
 819         frame_bytes = es_bytes[es] * en;
 820         for_each_sg(sgl, sgent, sglen, i) {
 821                 d->sg[i].addr = sg_dma_address(sgent);
 822                 d->sg[i].en = en;
 823                 d->sg[i].fn = sg_dma_len(sgent) / frame_bytes;
 824         }
 825
 826         d->sglen = sglen;
 827
 828         return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
 829 }
 830
 831 static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic(
 832         struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
 833         size_t period_len, enum dma_transfer_direction dir, unsigned long flags)
 834 {
 835         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 836         struct omap_chan *c = to_omap_dma_chan(chan);
 837         enum dma_slave_buswidth dev_width;
 838         struct omap_desc *d;
 839         dma_addr_t dev_addr;
 840         unsigned es;
 841         u32 burst;
 842
 843         if (dir == DMA_DEV_TO_MEM) {
 844                 dev_addr = c->cfg.src_addr;
 845                 dev_width = c->cfg.src_addr_width;
 846                 burst = c->cfg.src_maxburst;
 847         } else if (dir == DMA_MEM_TO_DEV) {
 848                 dev_addr = c->cfg.dst_addr;
 849                 dev_width = c->cfg.dst_addr_width;
 850                 burst = c->cfg.dst_maxburst;
 851         } else {
 852                 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
 853                 return NULL;
 854         }
 855
 856         /* Bus width translates to the element size (ES) */
 857         switch (dev_width) {
 858         case DMA_SLAVE_BUSWIDTH_1_BYTE:
 859                 es = CSDP_DATA_TYPE_8;
 860                 break;
 861         case DMA_SLAVE_BUSWIDTH_2_BYTES:
 862                 es = CSDP_DATA_TYPE_16;
 863                 break;
 864         case DMA_SLAVE_BUSWIDTH_4_BYTES:
 865                 es = CSDP_DATA_TYPE_32;
 866                 break;
 867         default: /* not reached */
 868                 return NULL;
 869         }
 870
 871         /* Now allocate and setup the descriptor. */
 872         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
 873         if (!d)
 874                 return NULL;
 875
 876         d->dir = dir;
 877         d->dev_addr = dev_addr;
 878         d->fi = burst;
 879         d->es = es;
 880         d->sg[0].addr = buf_addr;
 881         d->sg[0].en = period_len / es_bytes[es];
 882         d->sg[0].fn = buf_len / period_len;
 883         d->sglen = 1;
 884
 885         d->ccr = c->ccr;
 886         if (dir == DMA_DEV_TO_MEM)
 887                 d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT;
 888         else
 889                 d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC;
 890
 891         d->cicr = CICR_DROP_IE;
 892         if (flags & DMA_PREP_INTERRUPT)
 893                 d->cicr |= CICR_FRAME_IE;
 894
 895         d->csdp = es;
 896
 897         if (dma_omap1()) {
 898                 d->cicr |= CICR_TOUT_IE;
 899
 900                 if (dir == DMA_DEV_TO_MEM)
 901                         d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_MPUI;
 902                 else
 903                         d->csdp |= CSDP_DST_PORT_MPUI | CSDP_SRC_PORT_EMIFF;
 904         } else {
 905                 if (burst)
 906                         d->ccr |= CCR_SYNC_PACKET;
 907                 else
 908                         d->ccr |= CCR_SYNC_ELEMENT;
 909
 910                 if (dir == DMA_DEV_TO_MEM) {
 911                         d->ccr |= CCR_TRIGGER_SRC;
 912                         d->csdp |= CSDP_DST_PACKED;
 913                 } else {
 914                         d->csdp |= CSDP_SRC_PACKED;
 915                 }
 916
 917                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
 918
 919                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
 920         }
 921
 922         if (__dma_omap15xx(od->plat->dma_attr))
 923                 d->ccr |= CCR_AUTO_INIT | CCR_REPEAT;
 924         else
 925                 d->clnk_ctrl = c->dma_ch | CLNK_CTRL_ENABLE_LNK;
 926
 927         c->cyclic = true;
 928
 929         return vchan_tx_prep(&c->vc, &d->vd, flags);
 930 }
 931
 932 static struct dma_async_tx_descriptor *omap_dma_prep_dma_memcpy(
 933         struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 934         size_t len, unsigned long tx_flags)
 935 {
 936         struct omap_chan *c = to_omap_dma_chan(chan);
 937         struct omap_desc *d;
 938         uint8_t data_type;
 939
 940         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
 941         if (!d)
 942                 return NULL;
 943
 944         data_type = __ffs((src | dest | len));
 945         if (data_type > CSDP_DATA_TYPE_32)
 946                 data_type = CSDP_DATA_TYPE_32;
 947
 948         d->dir = DMA_MEM_TO_MEM;
 949         d->dev_addr = src;
 950         d->fi = 0;
 951         d->es = data_type;
 952         d->sg[0].en = len / BIT(data_type);
 953         d->sg[0].fn = 1;
 954         d->sg[0].addr = dest;
 955         d->sglen = 1;
 956         d->ccr = c->ccr;
 957         d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_POSTINC;
 958
 959         d->cicr = CICR_DROP_IE | CICR_FRAME_IE;
 960
 961         d->csdp = data_type;
 962
 963         if (dma_omap1()) {
 964                 d->cicr |= CICR_TOUT_IE;
 965                 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
 966         } else {
 967                 d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
 968                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
 969                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
 970         }
 971
 972         return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
 973 }
 974
 975 static struct dma_async_tx_descriptor *omap_dma_prep_dma_interleaved(
 976         struct dma_chan *chan, struct dma_interleaved_template *xt,
 977         unsigned long flags)
 978 {
 979         struct omap_chan *c = to_omap_dma_chan(chan);
 980         struct omap_desc *d;
 981         struct omap_sg *sg;
 982         uint8_t data_type;
 983         size_t src_icg, dst_icg;
 984
 985         /* Slave mode is not supported */
 986         if (is_slave_direction(xt->dir))
 987                 return NULL;
 988
 989         if (xt->frame_size != 1 || xt->numf == 0)
 990                 return NULL;
 991
 992         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
 993         if (!d)
 994                 return NULL;
 995
 996         data_type = __ffs((xt->src_start | xt->dst_start | xt->sgl[0].size));
 997         if (data_type > CSDP_DATA_TYPE_32)
 998                 data_type = CSDP_DATA_TYPE_32;
 999
1000         sg = &d->sg[0];
1001         d->dir = DMA_MEM_TO_MEM;
1002         d->dev_addr = xt->src_start;
1003         d->es = data_type;
1004         sg->en = xt->sgl[0].size / BIT(data_type);
1005         sg->fn = xt->numf;
1006         sg->addr = xt->dst_start;
1007         d->sglen = 1;
1008         d->ccr = c->ccr;
1009
1010         src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]);
1011         dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]);
1012         if (src_icg) {
1013                 d->ccr |= CCR_SRC_AMODE_DBLIDX;
1014                 d->ei = 1;
1015                 d->fi = src_icg;
1016         } else if (xt->src_inc) {
1017                 d->ccr |= CCR_SRC_AMODE_POSTINC;
1018                 d->fi = 0;
1019         } else {
1020                 dev_err(chan->device->dev,
1021                         "%s: SRC constant addressing is not supported\n",
1022                         __func__);
1023                 kfree(d);
1024                 return NULL;
1025         }
1026
1027         if (dst_icg) {
1028                 d->ccr |= CCR_DST_AMODE_DBLIDX;
1029                 sg->ei = 1;
1030                 sg->fi = dst_icg;
1031         } else if (xt->dst_inc) {
1032                 d->ccr |= CCR_DST_AMODE_POSTINC;
1033                 sg->fi = 0;
1034         } else {
1035                 dev_err(chan->device->dev,
1036                         "%s: DST constant addressing is not supported\n",
1037                         __func__);
1038                 kfree(d);
1039                 return NULL;
1040         }
1041
1042         d->cicr = CICR_DROP_IE | CICR_FRAME_IE;
1043
1044         d->csdp = data_type;
1045
1046         if (dma_omap1()) {
1047                 d->cicr |= CICR_TOUT_IE;
1048                 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
1049         } else {
1050                 d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
1051                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1052                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1053         }
1054
1055         return vchan_tx_prep(&c->vc, &d->vd, flags);
1056 }
1057
1058 static int omap_dma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg)
1059 {
1060         struct omap_chan *c = to_omap_dma_chan(chan);
1061
1062         if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
1063             cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
1064                 return -EINVAL;
1065
1066         memcpy(&c->cfg, cfg, sizeof(c->cfg));
1067
1068         return 0;
1069 }
1070
1071 static int omap_dma_terminate_all(struct dma_chan *chan)
1072 {
1073         struct omap_chan *c = to_omap_dma_chan(chan);
1074         unsigned long flags;
1075         LIST_HEAD(head);
1076
1077         spin_lock_irqsave(&c->vc.lock, flags);
1078
1079         /*
1080          * Stop DMA activity: we assume the callback will not be called
1081          * after omap_dma_stop() returns (even if it does, it will see
1082          * c->desc is NULL and exit.)
1083          */
1084         if (c->desc) {
1085                 omap_dma_desc_free(&c->desc->vd);
1086                 c->desc = NULL;
1087                 /* Avoid stopping the dma twice */
1088                 if (!c->paused)
1089                         omap_dma_stop(c);
1090         }
1091
1092         if (c->cyclic) {
1093                 c->cyclic = false;
1094                 c->paused = false;
1095         }
1096
1097         vchan_get_all_descriptors(&c->vc, &head);
1098         spin_unlock_irqrestore(&c->vc.lock, flags);
1099         vchan_dma_desc_free_list(&c->vc, &head);
1100
1101         return 0;
1102 }
1103
1104 static void omap_dma_synchronize(struct dma_chan *chan)
1105 {
1106         struct omap_chan *c = to_omap_dma_chan(chan);
1107
1108         vchan_synchronize(&c->vc);
1109 }
1110
1111 static int omap_dma_pause(struct dma_chan *chan)
1112 {
1113         struct omap_chan *c = to_omap_dma_chan(chan);
1114
1115         /* Pause/Resume only allowed with cyclic mode */
1116         if (!c->cyclic)
1117                 return -EINVAL;
1118
1119         if (!c->paused) {
1120                 omap_dma_stop(c);
1121                 c->paused = true;
1122         }
1123
1124         return 0;
1125 }
1126
1127 static int omap_dma_resume(struct dma_chan *chan)
1128 {
1129         struct omap_chan *c = to_omap_dma_chan(chan);
1130
1131         /* Pause/Resume only allowed with cyclic mode */
1132         if (!c->cyclic)
1133                 return -EINVAL;
1134
1135         if (c->paused) {
1136                 mb();
1137
1138                 /* Restore channel link register */
1139                 omap_dma_chan_write(c, CLNK_CTRL, c->desc->clnk_ctrl);
1140
1141                 omap_dma_start(c, c->desc);
1142                 c->paused = false;
1143         }
1144
1145         return 0;
1146 }
1147
1148 static int omap_dma_chan_init(struct omap_dmadev *od)
1149 {
1150         struct omap_chan *c;
1151
1152         c = kzalloc(sizeof(*c), GFP_KERNEL);
1153         if (!c)
1154                 return -ENOMEM;
1155
1156         c->reg_map = od->reg_map;
1157         c->vc.desc_free = omap_dma_desc_free;
1158         vchan_init(&c->vc, &od->ddev);
1159
1160         return 0;
1161 }
1162
1163 static void omap_dma_free(struct omap_dmadev *od)
1164 {
1165         while (!list_empty(&od->ddev.channels)) {
1166                 struct omap_chan *c = list_first_entry(&od->ddev.channels,
1167                         struct omap_chan, vc.chan.device_node);
1168
1169                 list_del(&c->vc.chan.device_node);
1170                 tasklet_kill(&c->vc.task);
1171                 kfree(c);
1172         }
1173 }
1174
1175 #define OMAP_DMA_BUSWIDTHS      (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
1176                                  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
1177                                  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
1178
1179 static int omap_dma_probe(struct platform_device *pdev)
1180 {
1181         struct omap_dmadev *od;
1182         struct resource *res;
1183         int rc, i, irq;
1184
1185         od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
1186         if (!od)
1187                 return -ENOMEM;
1188
1189         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1190         od->base = devm_ioremap_resource(&pdev->dev, res);
1191         if (IS_ERR(od->base))
1192                 return PTR_ERR(od->base);
1193
1194         od->plat = omap_get_plat_info();
1195         if (!od->plat)
1196                 return -EPROBE_DEFER;
1197
1198         od->reg_map = od->plat->reg_map;
1199
1200         dma_cap_set(DMA_SLAVE, od->ddev.cap_mask);
1201         dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask);
1202         dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask);
1203         dma_cap_set(DMA_INTERLEAVE, od->ddev.cap_mask);
1204         od->ddev.device_alloc_chan_resources = omap_dma_alloc_chan_resources;
1205         od->ddev.device_free_chan_resources = omap_dma_free_chan_resources;
1206         od->ddev.device_tx_status = omap_dma_tx_status;
1207         od->ddev.device_issue_pending = omap_dma_issue_pending;
1208         od->ddev.device_prep_slave_sg = omap_dma_prep_slave_sg;
1209         od->ddev.device_prep_dma_cyclic = omap_dma_prep_dma_cyclic;
1210         od->ddev.device_prep_dma_memcpy = omap_dma_prep_dma_memcpy;
1211         od->ddev.device_prep_interleaved_dma = omap_dma_prep_dma_interleaved;
1212         od->ddev.device_config = omap_dma_slave_config;
1213         od->ddev.device_pause = omap_dma_pause;
1214         od->ddev.device_resume = omap_dma_resume;
1215         od->ddev.device_terminate_all = omap_dma_terminate_all;
1216         od->ddev.device_synchronize = omap_dma_synchronize;
1217         od->ddev.src_addr_widths = OMAP_DMA_BUSWIDTHS;
1218         od->ddev.dst_addr_widths = OMAP_DMA_BUSWIDTHS;
1219         od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1220         od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1221         od->ddev.dev = &pdev->dev;
1222         INIT_LIST_HEAD(&od->ddev.channels);
1223         spin_lock_init(&od->lock);
1224         spin_lock_init(&od->irq_lock);
1225
1226         od->dma_requests = OMAP_SDMA_REQUESTS;
1227         if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
1228                                                       "dma-requests",
1229                                                       &od->dma_requests)) {
1230                 dev_info(&pdev->dev,
1231                          "Missing dma-requests property, using %u.\n",
1232                          OMAP_SDMA_REQUESTS);
1233         }
1234
1235         for (i = 0; i < OMAP_SDMA_CHANNELS; i++) {
1236                 rc = omap_dma_chan_init(od);
1237                 if (rc) {
1238                         omap_dma_free(od);
1239                         return rc;
1240                 }
1241         }
1242
1243         irq = platform_get_irq(pdev, 1);
1244         if (irq <= 0) {
1245                 dev_info(&pdev->dev, "failed to get L1 IRQ: %d\n", irq);
1246                 od->legacy = true;
1247         } else {
1248                 /* Disable all interrupts */
1249                 od->irq_enable_mask = 0;
1250                 omap_dma_glbl_write(od, IRQENABLE_L1, 0);
1251
1252                 rc = devm_request_irq(&pdev->dev, irq, omap_dma_irq,
1253                                       IRQF_SHARED, "omap-dma-engine", od);
1254                 if (rc)
1255                         return rc;
1256         }
1257
1258         od->ddev.filter.map = od->plat->slave_map;
1259         od->ddev.filter.mapcnt = od->plat->slavecnt;
1260         od->ddev.filter.fn = omap_dma_filter_fn;
1261
1262         rc = dma_async_device_register(&od->ddev);
1263         if (rc) {
1264                 pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n",
1265                         rc);
1266                 omap_dma_free(od);
1267                 return rc;
1268         }
1269
1270         platform_set_drvdata(pdev, od);
1271
1272         if (pdev->dev.of_node) {
1273                 omap_dma_info.dma_cap = od->ddev.cap_mask;
1274
1275                 /* Device-tree DMA controller registration */
1276                 rc = of_dma_controller_register(pdev->dev.of_node,
1277                                 of_dma_simple_xlate, &omap_dma_info);
1278                 if (rc) {
1279                         pr_warn("OMAP-DMA: failed to register DMA controller\n");
1280                         dma_async_device_unregister(&od->ddev);
1281                         omap_dma_free(od);
1282                 }
1283         }
1284
1285         dev_info(&pdev->dev, "OMAP DMA engine driver\n");
1286
1287         return rc;
1288 }
1289
1290 static int omap_dma_remove(struct platform_device *pdev)
1291 {
1292         struct omap_dmadev *od = platform_get_drvdata(pdev);
1293         int irq;
1294
1295         if (pdev->dev.of_node)
1296                 of_dma_controller_free(pdev->dev.of_node);
1297
1298         irq = platform_get_irq(pdev, 1);
1299         devm_free_irq(&pdev->dev, irq, od);
1300
1301         dma_async_device_unregister(&od->ddev);
1302
1303         if (!od->legacy) {
1304                 /* Disable all interrupts */
1305                 omap_dma_glbl_write(od, IRQENABLE_L0, 0);
1306         }
1307
1308         omap_dma_free(od);
1309
1310         return 0;
1311 }
1312
1313 static const struct of_device_id omap_dma_match[] = {
1314         { .compatible = "ti,omap2420-sdma", },
1315         { .compatible = "ti,omap2430-sdma", },
1316         { .compatible = "ti,omap3430-sdma", },
1317         { .compatible = "ti,omap3630-sdma", },
1318         { .compatible = "ti,omap4430-sdma", },
1319         {},
1320 };
1321 MODULE_DEVICE_TABLE(of, omap_dma_match);
1322
1323 static struct platform_driver omap_dma_driver = {
1324         .probe  = omap_dma_probe,
1325         .remove = omap_dma_remove,
1326         .driver = {
1327                 .name = "omap-dma-engine",
1328                 .of_match_table = of_match_ptr(omap_dma_match),
1329         },
1330 };
1331
1332 bool omap_dma_filter_fn(struct dma_chan *chan, void *param)
1333 {
1334         if (chan->device->dev->driver == &omap_dma_driver.driver) {
1335                 struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1336                 struct omap_chan *c = to_omap_dma_chan(chan);
1337                 unsigned req = *(unsigned *)param;
1338
1339                 if (req <= od->dma_requests) {
1340                         c->dma_sig = req;
1341                         return true;
1342                 }
1343         }
1344         return false;
1345 }
1346 EXPORT_SYMBOL_GPL(omap_dma_filter_fn);
1347
1348 static int omap_dma_init(void)
1349 {
1350         return platform_driver_register(&omap_dma_driver);
1351 }
1352 subsys_initcall(omap_dma_init);
1353
1354 static void __exit omap_dma_exit(void)
1355 {
1356         platform_driver_unregister(&omap_dma_driver);
1357 }
1358 module_exit(omap_dma_exit);
1359
1360 MODULE_AUTHOR("Russell King");
1361 MODULE_LICENSE("GPL");