drivers/dma/omap-dma.c

   1 /*
   2  * OMAP DMAengine support
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  */
   8 #include <linux/delay.h>
   9 #include <linux/dmaengine.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/err.h>
  12 #include <linux/init.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/list.h>
  15 #include <linux/module.h>
  16 #include <linux/omap-dma.h>
  17 #include <linux/platform_device.h>
  18 #include <linux/slab.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/of_dma.h>
  21 #include <linux/of_device.h>
  22
  23 #include "virt-dma.h"
  24
  25 #define OMAP_SDMA_REQUESTS      127
  26 #define OMAP_SDMA_CHANNELS      32
  27
  28 struct omap_dmadev {
  29         struct dma_device ddev;
  30         spinlock_t lock;
  31         void __iomem *base;
  32         const struct omap_dma_reg *reg_map;
  33         struct omap_system_dma_plat_info *plat;
  34         bool legacy;
  35         unsigned dma_requests;
  36         spinlock_t irq_lock;
  37         uint32_t irq_enable_mask;
  38         struct omap_chan *lch_map[OMAP_SDMA_CHANNELS];
  39 };
  40
  41 struct omap_chan {
  42         struct virt_dma_chan vc;
  43         void __iomem *channel_base;
  44         const struct omap_dma_reg *reg_map;
  45         uint32_t ccr;
  46
  47         struct dma_slave_config cfg;
  48         unsigned dma_sig;
  49         bool cyclic;
  50         bool paused;
  51         bool running;
  52
  53         int dma_ch;
  54         struct omap_desc *desc;
  55         unsigned sgidx;
  56 };
  57
  58 struct omap_sg {
  59         dma_addr_t addr;
  60         uint32_t en;            /* number of elements (24-bit) */
  61         uint32_t fn;            /* number of frames (16-bit) */
  62         int32_t fi;             /* for double indexing */
  63         int16_t ei;             /* for double indexing */
  64 };
  65
  66 struct omap_desc {
  67         struct virt_dma_desc vd;
  68         enum dma_transfer_direction dir;
  69         dma_addr_t dev_addr;
  70
  71         int32_t fi;             /* for OMAP_DMA_SYNC_PACKET / double indexing */
  72         int16_t ei;             /* for double indexing */
  73         uint8_t es;             /* CSDP_DATA_TYPE_xxx */
  74         uint32_t ccr;           /* CCR value */
  75         uint16_t clnk_ctrl;     /* CLNK_CTRL value */
  76         uint16_t cicr;          /* CICR value */
  77         uint32_t csdp;          /* CSDP value */
  78
  79         unsigned sglen;
  80         struct omap_sg sg[0];
  81 };
  82
  83 enum {
  84         CCR_FS                  = BIT(5),
  85         CCR_READ_PRIORITY       = BIT(6),
  86         CCR_ENABLE              = BIT(7),
  87         CCR_AUTO_INIT           = BIT(8),       /* OMAP1 only */
  88         CCR_REPEAT              = BIT(9),       /* OMAP1 only */
  89         CCR_OMAP31_DISABLE      = BIT(10),      /* OMAP1 only */
  90         CCR_SUSPEND_SENSITIVE   = BIT(8),       /* OMAP2+ only */
  91         CCR_RD_ACTIVE           = BIT(9),       /* OMAP2+ only */
  92         CCR_WR_ACTIVE           = BIT(10),      /* OMAP2+ only */
  93         CCR_SRC_AMODE_CONSTANT  = 0 << 12,
  94         CCR_SRC_AMODE_POSTINC   = 1 << 12,
  95         CCR_SRC_AMODE_SGLIDX    = 2 << 12,
  96         CCR_SRC_AMODE_DBLIDX    = 3 << 12,
  97         CCR_DST_AMODE_CONSTANT  = 0 << 14,
  98         CCR_DST_AMODE_POSTINC   = 1 << 14,
  99         CCR_DST_AMODE_SGLIDX    = 2 << 14,
 100         CCR_DST_AMODE_DBLIDX    = 3 << 14,
 101         CCR_CONSTANT_FILL       = BIT(16),
 102         CCR_TRANSPARENT_COPY    = BIT(17),
 103         CCR_BS                  = BIT(18),
 104         CCR_SUPERVISOR          = BIT(22),
 105         CCR_PREFETCH            = BIT(23),
 106         CCR_TRIGGER_SRC         = BIT(24),
 107         CCR_BUFFERING_DISABLE   = BIT(25),
 108         CCR_WRITE_PRIORITY      = BIT(26),
 109         CCR_SYNC_ELEMENT        = 0,
 110         CCR_SYNC_FRAME          = CCR_FS,
 111         CCR_SYNC_BLOCK          = CCR_BS,
 112         CCR_SYNC_PACKET         = CCR_BS | CCR_FS,
 113
 114         CSDP_DATA_TYPE_8        = 0,
 115         CSDP_DATA_TYPE_16       = 1,
 116         CSDP_DATA_TYPE_32       = 2,
 117         CSDP_SRC_PORT_EMIFF     = 0 << 2, /* OMAP1 only */
 118         CSDP_SRC_PORT_EMIFS     = 1 << 2, /* OMAP1 only */
 119         CSDP_SRC_PORT_OCP_T1    = 2 << 2, /* OMAP1 only */
 120         CSDP_SRC_PORT_TIPB      = 3 << 2, /* OMAP1 only */
 121         CSDP_SRC_PORT_OCP_T2    = 4 << 2, /* OMAP1 only */
 122         CSDP_SRC_PORT_MPUI      = 5 << 2, /* OMAP1 only */
 123         CSDP_SRC_PACKED         = BIT(6),
 124         CSDP_SRC_BURST_1        = 0 << 7,
 125         CSDP_SRC_BURST_16       = 1 << 7,
 126         CSDP_SRC_BURST_32       = 2 << 7,
 127         CSDP_SRC_BURST_64       = 3 << 7,
 128         CSDP_DST_PORT_EMIFF     = 0 << 9, /* OMAP1 only */
 129         CSDP_DST_PORT_EMIFS     = 1 << 9, /* OMAP1 only */
 130         CSDP_DST_PORT_OCP_T1    = 2 << 9, /* OMAP1 only */
 131         CSDP_DST_PORT_TIPB      = 3 << 9, /* OMAP1 only */
 132         CSDP_DST_PORT_OCP_T2    = 4 << 9, /* OMAP1 only */
 133         CSDP_DST_PORT_MPUI      = 5 << 9, /* OMAP1 only */
 134         CSDP_DST_PACKED         = BIT(13),
 135         CSDP_DST_BURST_1        = 0 << 14,
 136         CSDP_DST_BURST_16       = 1 << 14,
 137         CSDP_DST_BURST_32       = 2 << 14,
 138         CSDP_DST_BURST_64       = 3 << 14,
 139
 140         CICR_TOUT_IE            = BIT(0),       /* OMAP1 only */
 141         CICR_DROP_IE            = BIT(1),
 142         CICR_HALF_IE            = BIT(2),
 143         CICR_FRAME_IE           = BIT(3),
 144         CICR_LAST_IE            = BIT(4),
 145         CICR_BLOCK_IE           = BIT(5),
 146         CICR_PKT_IE             = BIT(7),       /* OMAP2+ only */
 147         CICR_TRANS_ERR_IE       = BIT(8),       /* OMAP2+ only */
 148         CICR_SUPERVISOR_ERR_IE  = BIT(10),      /* OMAP2+ only */
 149         CICR_MISALIGNED_ERR_IE  = BIT(11),      /* OMAP2+ only */
 150         CICR_DRAIN_IE           = BIT(12),      /* OMAP2+ only */
 151         CICR_SUPER_BLOCK_IE     = BIT(14),      /* OMAP2+ only */
 152
 153         CLNK_CTRL_ENABLE_LNK    = BIT(15),
 154 };
 155
 156 static const unsigned es_bytes[] = {
 157         [CSDP_DATA_TYPE_8] = 1,
 158         [CSDP_DATA_TYPE_16] = 2,
 159         [CSDP_DATA_TYPE_32] = 4,
 160 };
 161
 162 static struct of_dma_filter_info omap_dma_info = {
 163         .filter_fn = omap_dma_filter_fn,
 164 };
 165
 166 static inline struct omap_dmadev *to_omap_dma_dev(struct dma_device *d)
 167 {
 168         return container_of(d, struct omap_dmadev, ddev);
 169 }
 170
 171 static inline struct omap_chan *to_omap_dma_chan(struct dma_chan *c)
 172 {
 173         return container_of(c, struct omap_chan, vc.chan);
 174 }
 175
 176 static inline struct omap_desc *to_omap_dma_desc(struct dma_async_tx_descriptor *t)
 177 {
 178         return container_of(t, struct omap_desc, vd.tx);
 179 }
 180
 181 static void omap_dma_desc_free(struct virt_dma_desc *vd)
 182 {
 183         kfree(container_of(vd, struct omap_desc, vd));
 184 }
 185
 186 static void omap_dma_write(uint32_t val, unsigned type, void __iomem *addr)
 187 {
 188         switch (type) {
 189         case OMAP_DMA_REG_16BIT:
 190                 writew_relaxed(val, addr);
 191                 break;
 192         case OMAP_DMA_REG_2X16BIT:
 193                 writew_relaxed(val, addr);
 194                 writew_relaxed(val >> 16, addr + 2);
 195                 break;
 196         case OMAP_DMA_REG_32BIT:
 197                 writel_relaxed(val, addr);
 198                 break;
 199         default:
 200                 WARN_ON(1);
 201         }
 202 }
 203
 204 static unsigned omap_dma_read(unsigned type, void __iomem *addr)
 205 {
 206         unsigned val;
 207
 208         switch (type) {
 209         case OMAP_DMA_REG_16BIT:
 210                 val = readw_relaxed(addr);
 211                 break;
 212         case OMAP_DMA_REG_2X16BIT:
 213                 val = readw_relaxed(addr);
 214                 val |= readw_relaxed(addr + 2) << 16;
 215                 break;
 216         case OMAP_DMA_REG_32BIT:
 217                 val = readl_relaxed(addr);
 218                 break;
 219         default:
 220                 WARN_ON(1);
 221                 val = 0;
 222         }
 223
 224         return val;
 225 }
 226
 227 static void omap_dma_glbl_write(struct omap_dmadev *od, unsigned reg, unsigned val)
 228 {
 229         const struct omap_dma_reg *r = od->reg_map + reg;
 230
 231         WARN_ON(r->stride);
 232
 233         omap_dma_write(val, r->type, od->base + r->offset);
 234 }
 235
 236 static unsigned omap_dma_glbl_read(struct omap_dmadev *od, unsigned reg)
 237 {
 238         const struct omap_dma_reg *r = od->reg_map + reg;
 239
 240         WARN_ON(r->stride);
 241
 242         return omap_dma_read(r->type, od->base + r->offset);
 243 }
 244
 245 static void omap_dma_chan_write(struct omap_chan *c, unsigned reg, unsigned val)
 246 {
 247         const struct omap_dma_reg *r = c->reg_map + reg;
 248
 249         omap_dma_write(val, r->type, c->channel_base + r->offset);
 250 }
 251
 252 static unsigned omap_dma_chan_read(struct omap_chan *c, unsigned reg)
 253 {
 254         const struct omap_dma_reg *r = c->reg_map + reg;
 255
 256         return omap_dma_read(r->type, c->channel_base + r->offset);
 257 }
 258
 259 static void omap_dma_clear_csr(struct omap_chan *c)
 260 {
 261         if (dma_omap1())
 262                 omap_dma_chan_read(c, CSR);
 263         else
 264                 omap_dma_chan_write(c, CSR, ~0);
 265 }
 266
 267 static unsigned omap_dma_get_csr(struct omap_chan *c)
 268 {
 269         unsigned val = omap_dma_chan_read(c, CSR);
 270
 271         if (!dma_omap1())
 272                 omap_dma_chan_write(c, CSR, val);
 273
 274         return val;
 275 }
 276
 277 static void omap_dma_assign(struct omap_dmadev *od, struct omap_chan *c,
 278         unsigned lch)
 279 {
 280         c->channel_base = od->base + od->plat->channel_stride * lch;
 281
 282         od->lch_map[lch] = c;
 283 }
 284
 285 static void omap_dma_start(struct omap_chan *c, struct omap_desc *d)
 286 {
 287         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 288
 289         if (__dma_omap15xx(od->plat->dma_attr))
 290                 omap_dma_chan_write(c, CPC, 0);
 291         else
 292                 omap_dma_chan_write(c, CDAC, 0);
 293
 294         omap_dma_clear_csr(c);
 295
 296         /* Enable interrupts */
 297         omap_dma_chan_write(c, CICR, d->cicr);
 298
 299         /* Enable channel */
 300         omap_dma_chan_write(c, CCR, d->ccr | CCR_ENABLE);
 301
 302         c->running = true;
 303 }
 304
 305 static void omap_dma_stop(struct omap_chan *c)
 306 {
 307         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 308         uint32_t val;
 309
 310         /* disable irq */
 311         omap_dma_chan_write(c, CICR, 0);
 312
 313         omap_dma_clear_csr(c);
 314
 315         val = omap_dma_chan_read(c, CCR);
 316         if (od->plat->errata & DMA_ERRATA_i541 && val & CCR_TRIGGER_SRC) {
 317                 uint32_t sysconfig;
 318                 unsigned i;
 319
 320                 sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG);
 321                 val = sysconfig & ~DMA_SYSCONFIG_MIDLEMODE_MASK;
 322                 val |= DMA_SYSCONFIG_MIDLEMODE(DMA_IDLEMODE_NO_IDLE);
 323                 omap_dma_glbl_write(od, OCP_SYSCONFIG, val);
 324
 325                 val = omap_dma_chan_read(c, CCR);
 326                 val &= ~CCR_ENABLE;
 327                 omap_dma_chan_write(c, CCR, val);
 328
 329                 /* Wait for sDMA FIFO to drain */
 330                 for (i = 0; ; i++) {
 331                         val = omap_dma_chan_read(c, CCR);
 332                         if (!(val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE)))
 333                                 break;
 334
 335                         if (i > 100)
 336                                 break;
 337
 338                         udelay(5);
 339                 }
 340
 341                 if (val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE))
 342                         dev_err(c->vc.chan.device->dev,
 343                                 "DMA drain did not complete on lch %d\n",
 344                                 c->dma_ch);
 345
 346                 omap_dma_glbl_write(od, OCP_SYSCONFIG, sysconfig);
 347         } else {
 348                 val &= ~CCR_ENABLE;
 349                 omap_dma_chan_write(c, CCR, val);
 350         }
 351
 352         mb();
 353
 354         if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) {
 355                 val = omap_dma_chan_read(c, CLNK_CTRL);
 356
 357                 if (dma_omap1())
 358                         val |= 1 << 14; /* set the STOP_LNK bit */
 359                 else
 360                         val &= ~CLNK_CTRL_ENABLE_LNK;
 361
 362                 omap_dma_chan_write(c, CLNK_CTRL, val);
 363         }
 364
 365         c->running = false;
 366 }
 367
 368 static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d,
 369         unsigned idx)
 370 {
 371         struct omap_sg *sg = d->sg + idx;
 372         unsigned cxsa, cxei, cxfi;
 373
 374         if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
 375                 cxsa = CDSA;
 376                 cxei = CDEI;
 377                 cxfi = CDFI;
 378         } else {
 379                 cxsa = CSSA;
 380                 cxei = CSEI;
 381                 cxfi = CSFI;
 382         }
 383
 384         omap_dma_chan_write(c, cxsa, sg->addr);
 385         omap_dma_chan_write(c, cxei, sg->ei);
 386         omap_dma_chan_write(c, cxfi, sg->fi);
 387         omap_dma_chan_write(c, CEN, sg->en);
 388         omap_dma_chan_write(c, CFN, sg->fn);
 389
 390         omap_dma_start(c, d);
 391 }
 392
 393 static void omap_dma_start_desc(struct omap_chan *c)
 394 {
 395         struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
 396         struct omap_desc *d;
 397         unsigned cxsa, cxei, cxfi;
 398
 399         if (!vd) {
 400                 c->desc = NULL;
 401                 return;
 402         }
 403
 404         list_del(&vd->node);
 405
 406         c->desc = d = to_omap_dma_desc(&vd->tx);
 407         c->sgidx = 0;
 408
 409         /*
 410          * This provides the necessary barrier to ensure data held in
 411          * DMA coherent memory is visible to the DMA engine prior to
 412          * the transfer starting.
 413          */
 414         mb();
 415
 416         omap_dma_chan_write(c, CCR, d->ccr);
 417         if (dma_omap1())
 418                 omap_dma_chan_write(c, CCR2, d->ccr >> 16);
 419
 420         if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
 421                 cxsa = CSSA;
 422                 cxei = CSEI;
 423                 cxfi = CSFI;
 424         } else {
 425                 cxsa = CDSA;
 426                 cxei = CDEI;
 427                 cxfi = CDFI;
 428         }
 429
 430         omap_dma_chan_write(c, cxsa, d->dev_addr);
 431         omap_dma_chan_write(c, cxei, d->ei);
 432         omap_dma_chan_write(c, cxfi, d->fi);
 433         omap_dma_chan_write(c, CSDP, d->csdp);
 434         omap_dma_chan_write(c, CLNK_CTRL, d->clnk_ctrl);
 435
 436         omap_dma_start_sg(c, d, 0);
 437 }
 438
 439 static void omap_dma_callback(int ch, u16 status, void *data)
 440 {
 441         struct omap_chan *c = data;
 442         struct omap_desc *d;
 443         unsigned long flags;
 444
 445         spin_lock_irqsave(&c->vc.lock, flags);
 446         d = c->desc;
 447         if (d) {
 448                 if (!c->cyclic) {
 449                         if (++c->sgidx < d->sglen) {
 450                                 omap_dma_start_sg(c, d, c->sgidx);
 451                         } else {
 452                                 omap_dma_start_desc(c);
 453                                 vchan_cookie_complete(&d->vd);
 454                         }
 455                 } else {
 456                         vchan_cyclic_callback(&d->vd);
 457                 }
 458         }
 459         spin_unlock_irqrestore(&c->vc.lock, flags);
 460 }
 461
 462 static irqreturn_t omap_dma_irq(int irq, void *devid)
 463 {
 464         struct omap_dmadev *od = devid;
 465         unsigned status, channel;
 466
 467         spin_lock(&od->irq_lock);
 468
 469         status = omap_dma_glbl_read(od, IRQSTATUS_L1);
 470         status &= od->irq_enable_mask;
 471         if (status == 0) {
 472                 spin_unlock(&od->irq_lock);
 473                 return IRQ_NONE;
 474         }
 475
 476         while ((channel = ffs(status)) != 0) {
 477                 unsigned mask, csr;
 478                 struct omap_chan *c;
 479
 480                 channel -= 1;
 481                 mask = BIT(channel);
 482                 status &= ~mask;
 483
 484                 c = od->lch_map[channel];
 485                 if (c == NULL) {
 486                         /* This should never happen */
 487                         dev_err(od->ddev.dev, "invalid channel %u\n", channel);
 488                         continue;
 489                 }
 490
 491                 csr = omap_dma_get_csr(c);
 492                 omap_dma_glbl_write(od, IRQSTATUS_L1, mask);
 493
 494                 omap_dma_callback(channel, csr, c);
 495         }
 496
 497         spin_unlock(&od->irq_lock);
 498
 499         return IRQ_HANDLED;
 500 }
 501
 502 static int omap_dma_alloc_chan_resources(struct dma_chan *chan)
 503 {
 504         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 505         struct omap_chan *c = to_omap_dma_chan(chan);
 506         int ret;
 507
 508         if (od->legacy) {
 509                 ret = omap_request_dma(c->dma_sig, "DMA engine",
 510                                        omap_dma_callback, c, &c->dma_ch);
 511         } else {
 512                 ret = omap_request_dma(c->dma_sig, "DMA engine", NULL, NULL,
 513                                        &c->dma_ch);
 514         }
 515
 516         dev_dbg(od->ddev.dev, "allocating channel %u for %u\n",
 517                 c->dma_ch, c->dma_sig);
 518
 519         if (ret >= 0) {
 520                 omap_dma_assign(od, c, c->dma_ch);
 521
 522                 if (!od->legacy) {
 523                         unsigned val;
 524
 525                         spin_lock_irq(&od->irq_lock);
 526                         val = BIT(c->dma_ch);
 527                         omap_dma_glbl_write(od, IRQSTATUS_L1, val);
 528                         od->irq_enable_mask |= val;
 529                         omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
 530
 531                         val = omap_dma_glbl_read(od, IRQENABLE_L0);
 532                         val &= ~BIT(c->dma_ch);
 533                         omap_dma_glbl_write(od, IRQENABLE_L0, val);
 534                         spin_unlock_irq(&od->irq_lock);
 535                 }
 536         }
 537
 538         if (dma_omap1()) {
 539                 if (__dma_omap16xx(od->plat->dma_attr)) {
 540                         c->ccr = CCR_OMAP31_DISABLE;
 541                         /* Duplicate what plat-omap/dma.c does */
 542                         c->ccr |= c->dma_ch + 1;
 543                 } else {
 544                         c->ccr = c->dma_sig & 0x1f;
 545                 }
 546         } else {
 547                 c->ccr = c->dma_sig & 0x1f;
 548                 c->ccr |= (c->dma_sig & ~0x1f) << 14;
 549         }
 550         if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING)
 551                 c->ccr |= CCR_BUFFERING_DISABLE;
 552
 553         return ret;
 554 }
 555
 556 static void omap_dma_free_chan_resources(struct dma_chan *chan)
 557 {
 558         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 559         struct omap_chan *c = to_omap_dma_chan(chan);
 560
 561         if (!od->legacy) {
 562                 spin_lock_irq(&od->irq_lock);
 563                 od->irq_enable_mask &= ~BIT(c->dma_ch);
 564                 omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
 565                 spin_unlock_irq(&od->irq_lock);
 566         }
 567
 568         c->channel_base = NULL;
 569         od->lch_map[c->dma_ch] = NULL;
 570         vchan_free_chan_resources(&c->vc);
 571         omap_free_dma(c->dma_ch);
 572
 573         dev_dbg(od->ddev.dev, "freeing channel for %u\n", c->dma_sig);
 574         c->dma_sig = 0;
 575 }
 576
 577 static size_t omap_dma_sg_size(struct omap_sg *sg)
 578 {
 579         return sg->en * sg->fn;
 580 }
 581
 582 static size_t omap_dma_desc_size(struct omap_desc *d)
 583 {
 584         unsigned i;
 585         size_t size;
 586
 587         for (size = i = 0; i < d->sglen; i++)
 588                 size += omap_dma_sg_size(&d->sg[i]);
 589
 590         return size * es_bytes[d->es];
 591 }
 592
 593 static size_t omap_dma_desc_size_pos(struct omap_desc *d, dma_addr_t addr)
 594 {
 595         unsigned i;
 596         size_t size, es_size = es_bytes[d->es];
 597
 598         for (size = i = 0; i < d->sglen; i++) {
 599                 size_t this_size = omap_dma_sg_size(&d->sg[i]) * es_size;
 600
 601                 if (size)
 602                         size += this_size;
 603                 else if (addr >= d->sg[i].addr &&
 604                          addr < d->sg[i].addr + this_size)
 605                         size += d->sg[i].addr + this_size - addr;
 606         }
 607         return size;
 608 }
 609
 610 /*
 611  * OMAP 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is
 612  * read before the DMA controller finished disabling the channel.
 613  */
 614 static uint32_t omap_dma_chan_read_3_3(struct omap_chan *c, unsigned reg)
 615 {
 616         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 617         uint32_t val;
 618
 619         val = omap_dma_chan_read(c, reg);
 620         if (val == 0 && od->plat->errata & DMA_ERRATA_3_3)
 621                 val = omap_dma_chan_read(c, reg);
 622
 623         return val;
 624 }
 625
 626 static dma_addr_t omap_dma_get_src_pos(struct omap_chan *c)
 627 {
 628         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 629         dma_addr_t addr, cdac;
 630
 631         if (__dma_omap15xx(od->plat->dma_attr)) {
 632                 addr = omap_dma_chan_read(c, CPC);
 633         } else {
 634                 addr = omap_dma_chan_read_3_3(c, CSAC);
 635                 cdac = omap_dma_chan_read_3_3(c, CDAC);
 636
 637                 /*
 638                  * CDAC == 0 indicates that the DMA transfer on the channel has
 639                  * not been started (no data has been transferred so far).
 640                  * Return the programmed source start address in this case.
 641                  */
 642                 if (cdac == 0)
 643                         addr = omap_dma_chan_read(c, CSSA);
 644         }
 645
 646         if (dma_omap1())
 647                 addr |= omap_dma_chan_read(c, CSSA) & 0xffff0000;
 648
 649         return addr;
 650 }
 651
 652 static dma_addr_t omap_dma_get_dst_pos(struct omap_chan *c)
 653 {
 654         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 655         dma_addr_t addr;
 656
 657         if (__dma_omap15xx(od->plat->dma_attr)) {
 658                 addr = omap_dma_chan_read(c, CPC);
 659         } else {
 660                 addr = omap_dma_chan_read_3_3(c, CDAC);
 661
 662                 /*
 663                  * CDAC == 0 indicates that the DMA transfer on the channel
 664                  * has not been started (no data has been transferred so
 665                  * far).  Return the programmed destination start address in
 666                  * this case.
 667                  */
 668                 if (addr == 0)
 669                         addr = omap_dma_chan_read(c, CDSA);
 670         }
 671
 672         if (dma_omap1())
 673                 addr |= omap_dma_chan_read(c, CDSA) & 0xffff0000;
 674
 675         return addr;
 676 }
 677
 678 static enum dma_status omap_dma_tx_status(struct dma_chan *chan,
 679         dma_cookie_t cookie, struct dma_tx_state *txstate)
 680 {
 681         struct omap_chan *c = to_omap_dma_chan(chan);
 682         struct virt_dma_desc *vd;
 683         enum dma_status ret;
 684         unsigned long flags;
 685
 686         ret = dma_cookie_status(chan, cookie, txstate);
 687
 688         if (!c->paused && c->running) {
 689                 uint32_t ccr = omap_dma_chan_read(c, CCR);
 690                 /*
 691                  * The channel is no longer active, set the return value
 692                  * accordingly
 693                  */
 694                 if (!(ccr & CCR_ENABLE))
 695                         ret = DMA_COMPLETE;
 696         }
 697
 698         if (ret == DMA_COMPLETE || !txstate)
 699                 return ret;
 700
 701         spin_lock_irqsave(&c->vc.lock, flags);
 702         vd = vchan_find_desc(&c->vc, cookie);
 703         if (vd) {
 704                 txstate->residue = omap_dma_desc_size(to_omap_dma_desc(&vd->tx));
 705         } else if (c->desc && c->desc->vd.tx.cookie == cookie) {
 706                 struct omap_desc *d = c->desc;
 707                 dma_addr_t pos;
 708
 709                 if (d->dir == DMA_MEM_TO_DEV)
 710                         pos = omap_dma_get_src_pos(c);
 711                 else if (d->dir == DMA_DEV_TO_MEM  || d->dir == DMA_MEM_TO_MEM)
 712                         pos = omap_dma_get_dst_pos(c);
 713                 else
 714                         pos = 0;
 715
 716                 txstate->residue = omap_dma_desc_size_pos(d, pos);
 717         } else {
 718                 txstate->residue = 0;
 719         }
 720         spin_unlock_irqrestore(&c->vc.lock, flags);
 721
 722         return ret;
 723 }
 724
 725 static void omap_dma_issue_pending(struct dma_chan *chan)
 726 {
 727         struct omap_chan *c = to_omap_dma_chan(chan);
 728         unsigned long flags;
 729
 730         spin_lock_irqsave(&c->vc.lock, flags);
 731         if (vchan_issue_pending(&c->vc) && !c->desc)
 732                 omap_dma_start_desc(c);
 733         spin_unlock_irqrestore(&c->vc.lock, flags);
 734 }
 735
 736 static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
 737         struct dma_chan *chan, struct scatterlist *sgl, unsigned sglen,
 738         enum dma_transfer_direction dir, unsigned long tx_flags, void *context)
 739 {
 740         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 741         struct omap_chan *c = to_omap_dma_chan(chan);
 742         enum dma_slave_buswidth dev_width;
 743         struct scatterlist *sgent;
 744         struct omap_desc *d;
 745         dma_addr_t dev_addr;
 746         unsigned i, es, en, frame_bytes;
 747         u32 burst;
 748
 749         if (dir == DMA_DEV_TO_MEM) {
 750                 dev_addr = c->cfg.src_addr;
 751                 dev_width = c->cfg.src_addr_width;
 752                 burst = c->cfg.src_maxburst;
 753         } else if (dir == DMA_MEM_TO_DEV) {
 754                 dev_addr = c->cfg.dst_addr;
 755                 dev_width = c->cfg.dst_addr_width;
 756                 burst = c->cfg.dst_maxburst;
 757         } else {
 758                 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
 759                 return NULL;
 760         }
 761
 762         /* Bus width translates to the element size (ES) */
 763         switch (dev_width) {
 764         case DMA_SLAVE_BUSWIDTH_1_BYTE:
 765                 es = CSDP_DATA_TYPE_8;
 766                 break;
 767         case DMA_SLAVE_BUSWIDTH_2_BYTES:
 768                 es = CSDP_DATA_TYPE_16;
 769                 break;
 770         case DMA_SLAVE_BUSWIDTH_4_BYTES:
 771                 es = CSDP_DATA_TYPE_32;
 772                 break;
 773         default: /* not reached */
 774                 return NULL;
 775         }
 776
 777         /* Now allocate and setup the descriptor. */
 778         d = kzalloc(sizeof(*d) + sglen * sizeof(d->sg[0]), GFP_ATOMIC);
 779         if (!d)
 780                 return NULL;
 781
 782         d->dir = dir;
 783         d->dev_addr = dev_addr;
 784         d->es = es;
 785
 786         d->ccr = c->ccr | CCR_SYNC_FRAME;
 787         if (dir == DMA_DEV_TO_MEM)
 788                 d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT;
 789         else
 790                 d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC;
 791
 792         d->cicr = CICR_DROP_IE | CICR_BLOCK_IE;
 793         d->csdp = es;
 794
 795         if (dma_omap1()) {
 796                 d->cicr |= CICR_TOUT_IE;
 797
 798                 if (dir == DMA_DEV_TO_MEM)
 799                         d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_TIPB;
 800                 else
 801                         d->csdp |= CSDP_DST_PORT_TIPB | CSDP_SRC_PORT_EMIFF;
 802         } else {
 803                 if (dir == DMA_DEV_TO_MEM)
 804                         d->ccr |= CCR_TRIGGER_SRC;
 805
 806                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
 807         }
 808         if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS)
 809                 d->clnk_ctrl = c->dma_ch;
 810
 811         /*
 812          * Build our scatterlist entries: each contains the address,
 813          * the number of elements (EN) in each frame, and the number of
 814          * frames (FN).  Number of bytes for this entry = ES * EN * FN.
 815          *
 816          * Burst size translates to number of elements with frame sync.
 817          * Note: DMA engine defines burst to be the number of dev-width
 818          * transfers.
 819          */
 820         en = burst;
 821         frame_bytes = es_bytes[es] * en;
 822         for_each_sg(sgl, sgent, sglen, i) {
 823                 d->sg[i].addr = sg_dma_address(sgent);
 824                 d->sg[i].en = en;
 825                 d->sg[i].fn = sg_dma_len(sgent) / frame_bytes;
 826         }
 827
 828         d->sglen = sglen;
 829
 830         return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
 831 }
 832
 833 static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic(
 834         struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
 835         size_t period_len, enum dma_transfer_direction dir, unsigned long flags)
 836 {
 837         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 838         struct omap_chan *c = to_omap_dma_chan(chan);
 839         enum dma_slave_buswidth dev_width;
 840         struct omap_desc *d;
 841         dma_addr_t dev_addr;
 842         unsigned es;
 843         u32 burst;
 844
 845         if (dir == DMA_DEV_TO_MEM) {
 846                 dev_addr = c->cfg.src_addr;
 847                 dev_width = c->cfg.src_addr_width;
 848                 burst = c->cfg.src_maxburst;
 849         } else if (dir == DMA_MEM_TO_DEV) {
 850                 dev_addr = c->cfg.dst_addr;
 851                 dev_width = c->cfg.dst_addr_width;
 852                 burst = c->cfg.dst_maxburst;
 853         } else {
 854                 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
 855                 return NULL;
 856         }
 857
 858         /* Bus width translates to the element size (ES) */
 859         switch (dev_width) {
 860         case DMA_SLAVE_BUSWIDTH_1_BYTE:
 861                 es = CSDP_DATA_TYPE_8;
 862                 break;
 863         case DMA_SLAVE_BUSWIDTH_2_BYTES:
 864                 es = CSDP_DATA_TYPE_16;
 865                 break;
 866         case DMA_SLAVE_BUSWIDTH_4_BYTES:
 867                 es = CSDP_DATA_TYPE_32;
 868                 break;
 869         default: /* not reached */
 870                 return NULL;
 871         }
 872
 873         /* Now allocate and setup the descriptor. */
 874         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
 875         if (!d)
 876                 return NULL;
 877
 878         d->dir = dir;
 879         d->dev_addr = dev_addr;
 880         d->fi = burst;
 881         d->es = es;
 882         d->sg[0].addr = buf_addr;
 883         d->sg[0].en = period_len / es_bytes[es];
 884         d->sg[0].fn = buf_len / period_len;
 885         d->sglen = 1;
 886
 887         d->ccr = c->ccr;
 888         if (dir == DMA_DEV_TO_MEM)
 889                 d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT;
 890         else
 891                 d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC;
 892
 893         d->cicr = CICR_DROP_IE;
 894         if (flags & DMA_PREP_INTERRUPT)
 895                 d->cicr |= CICR_FRAME_IE;
 896
 897         d->csdp = es;
 898
 899         if (dma_omap1()) {
 900                 d->cicr |= CICR_TOUT_IE;
 901
 902                 if (dir == DMA_DEV_TO_MEM)
 903                         d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_MPUI;
 904                 else
 905                         d->csdp |= CSDP_DST_PORT_MPUI | CSDP_SRC_PORT_EMIFF;
 906         } else {
 907                 if (burst)
 908                         d->ccr |= CCR_SYNC_PACKET;
 909                 else
 910                         d->ccr |= CCR_SYNC_ELEMENT;
 911
 912                 if (dir == DMA_DEV_TO_MEM) {
 913                         d->ccr |= CCR_TRIGGER_SRC;
 914                         d->csdp |= CSDP_DST_PACKED;
 915                 } else {
 916                         d->csdp |= CSDP_SRC_PACKED;
 917                 }
 918
 919                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
 920
 921                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
 922         }
 923
 924         if (__dma_omap15xx(od->plat->dma_attr))
 925                 d->ccr |= CCR_AUTO_INIT | CCR_REPEAT;
 926         else
 927                 d->clnk_ctrl = c->dma_ch | CLNK_CTRL_ENABLE_LNK;
 928
 929         c->cyclic = true;
 930
 931         return vchan_tx_prep(&c->vc, &d->vd, flags);
 932 }
 933
 934 static struct dma_async_tx_descriptor *omap_dma_prep_dma_memcpy(
 935         struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 936         size_t len, unsigned long tx_flags)
 937 {
 938         struct omap_chan *c = to_omap_dma_chan(chan);
 939         struct omap_desc *d;
 940         uint8_t data_type;
 941
 942         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
 943         if (!d)
 944                 return NULL;
 945
 946         data_type = __ffs((src | dest | len));
 947         if (data_type > CSDP_DATA_TYPE_32)
 948                 data_type = CSDP_DATA_TYPE_32;
 949
 950         d->dir = DMA_MEM_TO_MEM;
 951         d->dev_addr = src;
 952         d->fi = 0;
 953         d->es = data_type;
 954         d->sg[0].en = len / BIT(data_type);
 955         d->sg[0].fn = 1;
 956         d->sg[0].addr = dest;
 957         d->sglen = 1;
 958         d->ccr = c->ccr;
 959         d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_POSTINC;
 960
 961         d->cicr = CICR_DROP_IE | CICR_FRAME_IE;
 962
 963         d->csdp = data_type;
 964
 965         if (dma_omap1()) {
 966                 d->cicr |= CICR_TOUT_IE;
 967                 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
 968         } else {
 969                 d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
 970                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
 971                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
 972         }
 973
 974         return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
 975 }
 976
 977 static struct dma_async_tx_descriptor *omap_dma_prep_dma_interleaved(
 978         struct dma_chan *chan, struct dma_interleaved_template *xt,
 979         unsigned long flags)
 980 {
 981         struct omap_chan *c = to_omap_dma_chan(chan);
 982         struct omap_desc *d;
 983         struct omap_sg *sg;
 984         uint8_t data_type;
 985         size_t src_icg, dst_icg;
 986
 987         /* Slave mode is not supported */
 988         if (is_slave_direction(xt->dir))
 989                 return NULL;
 990
 991         if (xt->frame_size != 1 || xt->numf == 0)
 992                 return NULL;
 993
 994         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
 995         if (!d)
 996                 return NULL;
 997
 998         data_type = __ffs((xt->src_start | xt->dst_start | xt->sgl[0].size));
 999         if (data_type > CSDP_DATA_TYPE_32)
1000                 data_type = CSDP_DATA_TYPE_32;
1001
1002         sg = &d->sg[0];
1003         d->dir = DMA_MEM_TO_MEM;
1004         d->dev_addr = xt->src_start;
1005         d->es = data_type;
1006         sg->en = xt->sgl[0].size / BIT(data_type);
1007         sg->fn = xt->numf;
1008         sg->addr = xt->dst_start;
1009         d->sglen = 1;
1010         d->ccr = c->ccr;
1011
1012         src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]);
1013         dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]);
1014         if (src_icg) {
1015                 d->ccr |= CCR_SRC_AMODE_DBLIDX;
1016                 d->ei = 1;
1017                 d->fi = src_icg;
1018         } else if (xt->src_inc) {
1019                 d->ccr |= CCR_SRC_AMODE_POSTINC;
1020                 d->fi = 0;
1021         } else {
1022                 dev_err(chan->device->dev,
1023                         "%s: SRC constant addressing is not supported\n",
1024                         __func__);
1025                 kfree(d);
1026                 return NULL;
1027         }
1028
1029         if (dst_icg) {
1030                 d->ccr |= CCR_DST_AMODE_DBLIDX;
1031                 sg->ei = 1;
1032                 sg->fi = dst_icg;
1033         } else if (xt->dst_inc) {
1034                 d->ccr |= CCR_DST_AMODE_POSTINC;
1035                 sg->fi = 0;
1036         } else {
1037                 dev_err(chan->device->dev,
1038                         "%s: DST constant addressing is not supported\n",
1039                         __func__);
1040                 kfree(d);
1041                 return NULL;
1042         }
1043
1044         d->cicr = CICR_DROP_IE | CICR_FRAME_IE;
1045
1046         d->csdp = data_type;
1047
1048         if (dma_omap1()) {
1049                 d->cicr |= CICR_TOUT_IE;
1050                 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
1051         } else {
1052                 d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
1053                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1054                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1055         }
1056
1057         return vchan_tx_prep(&c->vc, &d->vd, flags);
1058 }
1059
1060 static int omap_dma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg)
1061 {
1062         struct omap_chan *c = to_omap_dma_chan(chan);
1063
1064         if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
1065             cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
1066                 return -EINVAL;
1067
1068         memcpy(&c->cfg, cfg, sizeof(c->cfg));
1069
1070         return 0;
1071 }
1072
1073 static int omap_dma_terminate_all(struct dma_chan *chan)
1074 {
1075         struct omap_chan *c = to_omap_dma_chan(chan);
1076         unsigned long flags;
1077         LIST_HEAD(head);
1078
1079         spin_lock_irqsave(&c->vc.lock, flags);
1080
1081         /*
1082          * Stop DMA activity: we assume the callback will not be called
1083          * after omap_dma_stop() returns (even if it does, it will see
1084          * c->desc is NULL and exit.)
1085          */
1086         if (c->desc) {
1087                 omap_dma_desc_free(&c->desc->vd);
1088                 c->desc = NULL;
1089                 /* Avoid stopping the dma twice */
1090                 if (!c->paused)
1091                         omap_dma_stop(c);
1092         }
1093
1094         if (c->cyclic) {
1095                 c->cyclic = false;
1096                 c->paused = false;
1097         }
1098
1099         vchan_get_all_descriptors(&c->vc, &head);
1100         spin_unlock_irqrestore(&c->vc.lock, flags);
1101         vchan_dma_desc_free_list(&c->vc, &head);
1102
1103         return 0;
1104 }
1105
1106 static void omap_dma_synchronize(struct dma_chan *chan)
1107 {
1108         struct omap_chan *c = to_omap_dma_chan(chan);
1109
1110         vchan_synchronize(&c->vc);
1111 }
1112
1113 static int omap_dma_pause(struct dma_chan *chan)
1114 {
1115         struct omap_chan *c = to_omap_dma_chan(chan);
1116
1117         /* Pause/Resume only allowed with cyclic mode */
1118         if (!c->cyclic)
1119                 return -EINVAL;
1120
1121         if (!c->paused) {
1122                 omap_dma_stop(c);
1123                 c->paused = true;
1124         }
1125
1126         return 0;
1127 }
1128
1129 static int omap_dma_resume(struct dma_chan *chan)
1130 {
1131         struct omap_chan *c = to_omap_dma_chan(chan);
1132
1133         /* Pause/Resume only allowed with cyclic mode */
1134         if (!c->cyclic)
1135                 return -EINVAL;
1136
1137         if (c->paused) {
1138                 mb();
1139
1140                 /* Restore channel link register */
1141                 omap_dma_chan_write(c, CLNK_CTRL, c->desc->clnk_ctrl);
1142
1143                 omap_dma_start(c, c->desc);
1144                 c->paused = false;
1145         }
1146
1147         return 0;
1148 }
1149
1150 static int omap_dma_chan_init(struct omap_dmadev *od)
1151 {
1152         struct omap_chan *c;
1153
1154         c = kzalloc(sizeof(*c), GFP_KERNEL);
1155         if (!c)
1156                 return -ENOMEM;
1157
1158         c->reg_map = od->reg_map;
1159         c->vc.desc_free = omap_dma_desc_free;
1160         vchan_init(&c->vc, &od->ddev);
1161
1162         return 0;
1163 }
1164
1165 static void omap_dma_free(struct omap_dmadev *od)
1166 {
1167         while (!list_empty(&od->ddev.channels)) {
1168                 struct omap_chan *c = list_first_entry(&od->ddev.channels,
1169                         struct omap_chan, vc.chan.device_node);
1170
1171                 list_del(&c->vc.chan.device_node);
1172                 tasklet_kill(&c->vc.task);
1173                 kfree(c);
1174         }
1175 }
1176
1177 #define OMAP_DMA_BUSWIDTHS      (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
1178                                  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
1179                                  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
1180
1181 static int omap_dma_probe(struct platform_device *pdev)
1182 {
1183         struct omap_dmadev *od;
1184         struct resource *res;
1185         int rc, i, irq;
1186
1187         od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
1188         if (!od)
1189                 return -ENOMEM;
1190
1191         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1192         od->base = devm_ioremap_resource(&pdev->dev, res);
1193         if (IS_ERR(od->base))
1194                 return PTR_ERR(od->base);
1195
1196         od->plat = omap_get_plat_info();
1197         if (!od->plat)
1198                 return -EPROBE_DEFER;
1199
1200         od->reg_map = od->plat->reg_map;
1201
1202         dma_cap_set(DMA_SLAVE, od->ddev.cap_mask);
1203         dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask);
1204         dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask);
1205         dma_cap_set(DMA_INTERLEAVE, od->ddev.cap_mask);
1206         od->ddev.device_alloc_chan_resources = omap_dma_alloc_chan_resources;
1207         od->ddev.device_free_chan_resources = omap_dma_free_chan_resources;
1208         od->ddev.device_tx_status = omap_dma_tx_status;
1209         od->ddev.device_issue_pending = omap_dma_issue_pending;
1210         od->ddev.device_prep_slave_sg = omap_dma_prep_slave_sg;
1211         od->ddev.device_prep_dma_cyclic = omap_dma_prep_dma_cyclic;
1212         od->ddev.device_prep_dma_memcpy = omap_dma_prep_dma_memcpy;
1213         od->ddev.device_prep_interleaved_dma = omap_dma_prep_dma_interleaved;
1214         od->ddev.device_config = omap_dma_slave_config;
1215         od->ddev.device_pause = omap_dma_pause;
1216         od->ddev.device_resume = omap_dma_resume;
1217         od->ddev.device_terminate_all = omap_dma_terminate_all;
1218         od->ddev.device_synchronize = omap_dma_synchronize;
1219         od->ddev.src_addr_widths = OMAP_DMA_BUSWIDTHS;
1220         od->ddev.dst_addr_widths = OMAP_DMA_BUSWIDTHS;
1221         od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1222         od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1223         od->ddev.dev = &pdev->dev;
1224         INIT_LIST_HEAD(&od->ddev.channels);
1225         spin_lock_init(&od->lock);
1226         spin_lock_init(&od->irq_lock);
1227
1228         od->dma_requests = OMAP_SDMA_REQUESTS;
1229         if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
1230                                                       "dma-requests",
1231                                                       &od->dma_requests)) {
1232                 dev_info(&pdev->dev,
1233                          "Missing dma-requests property, using %u.\n",
1234                          OMAP_SDMA_REQUESTS);
1235         }
1236
1237         for (i = 0; i < OMAP_SDMA_CHANNELS; i++) {
1238                 rc = omap_dma_chan_init(od);
1239                 if (rc) {
1240                         omap_dma_free(od);
1241                         return rc;
1242                 }
1243         }
1244
1245         irq = platform_get_irq(pdev, 1);
1246         if (irq <= 0) {
1247                 dev_info(&pdev->dev, "failed to get L1 IRQ: %d\n", irq);
1248                 od->legacy = true;
1249         } else {
1250                 /* Disable all interrupts */
1251                 od->irq_enable_mask = 0;
1252                 omap_dma_glbl_write(od, IRQENABLE_L1, 0);
1253
1254                 rc = devm_request_irq(&pdev->dev, irq, omap_dma_irq,
1255                                       IRQF_SHARED, "omap-dma-engine", od);
1256                 if (rc)
1257                         return rc;
1258         }
1259
1260         od->ddev.filter.map = od->plat->slave_map;
1261         od->ddev.filter.mapcnt = od->plat->slavecnt;
1262         od->ddev.filter.fn = omap_dma_filter_fn;
1263
1264         rc = dma_async_device_register(&od->ddev);
1265         if (rc) {
1266                 pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n",
1267                         rc);
1268                 omap_dma_free(od);
1269                 return rc;
1270         }
1271
1272         platform_set_drvdata(pdev, od);
1273
1274         if (pdev->dev.of_node) {
1275                 omap_dma_info.dma_cap = od->ddev.cap_mask;
1276
1277                 /* Device-tree DMA controller registration */
1278                 rc = of_dma_controller_register(pdev->dev.of_node,
1279                                 of_dma_simple_xlate, &omap_dma_info);
1280                 if (rc) {
1281                         pr_warn("OMAP-DMA: failed to register DMA controller\n");
1282                         dma_async_device_unregister(&od->ddev);
1283                         omap_dma_free(od);
1284                 }
1285         }
1286
1287         dev_info(&pdev->dev, "OMAP DMA engine driver\n");
1288
1289         return rc;
1290 }
1291
1292 static int omap_dma_remove(struct platform_device *pdev)
1293 {
1294         struct omap_dmadev *od = platform_get_drvdata(pdev);
1295         int irq;
1296
1297         if (pdev->dev.of_node)
1298                 of_dma_controller_free(pdev->dev.of_node);
1299
1300         irq = platform_get_irq(pdev, 1);
1301         devm_free_irq(&pdev->dev, irq, od);
1302
1303         dma_async_device_unregister(&od->ddev);
1304
1305         if (!od->legacy) {
1306                 /* Disable all interrupts */
1307                 omap_dma_glbl_write(od, IRQENABLE_L0, 0);
1308         }
1309
1310         omap_dma_free(od);
1311
1312         return 0;
1313 }
1314
1315 static const struct of_device_id omap_dma_match[] = {
1316         { .compatible = "ti,omap2420-sdma", },
1317         { .compatible = "ti,omap2430-sdma", },
1318         { .compatible = "ti,omap3430-sdma", },
1319         { .compatible = "ti,omap3630-sdma", },
1320         { .compatible = "ti,omap4430-sdma", },
1321         {},
1322 };
1323 MODULE_DEVICE_TABLE(of, omap_dma_match);
1324
1325 static struct platform_driver omap_dma_driver = {
1326         .probe  = omap_dma_probe,
1327         .remove = omap_dma_remove,
1328         .driver = {
1329                 .name = "omap-dma-engine",
1330                 .of_match_table = of_match_ptr(omap_dma_match),
1331         },
1332 };
1333
1334 bool omap_dma_filter_fn(struct dma_chan *chan, void *param)
1335 {
1336         if (chan->device->dev->driver == &omap_dma_driver.driver) {
1337                 struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1338                 struct omap_chan *c = to_omap_dma_chan(chan);
1339                 unsigned req = *(unsigned *)param;
1340
1341                 if (req <= od->dma_requests) {
1342                         c->dma_sig = req;
1343                         return true;
1344                 }
1345         }
1346         return false;
1347 }
1348 EXPORT_SYMBOL_GPL(omap_dma_filter_fn);
1349
1350 static int omap_dma_init(void)
1351 {
1352         return platform_driver_register(&omap_dma_driver);
1353 }
1354 subsys_initcall(omap_dma_init);
1355
1356 static void __exit omap_dma_exit(void)
1357 {
1358         platform_driver_unregister(&omap_dma_driver);
1359 }
1360 module_exit(omap_dma_exit);
1361
1362 MODULE_AUTHOR("Russell King");
1363 MODULE_LICENSE("GPL");