drivers/dma/sa11x0-dma.c

   1 /*
   2  * SA11x0 DMAengine support
   3  *
   4  * Copyright (C) 2012 Russell King
   5  *   Derived in part from arch/arm/mach-sa1100/dma.c,
   6  *   Copyright (C) 2000, 2001 by Nicolas Pitre
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 as
  10  * published by the Free Software Foundation.
  11  */
  12 #include <linux/sched.h>
  13 #include <linux/device.h>
  14 #include <linux/dmaengine.h>
  15 #include <linux/init.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/kernel.h>
  18 #include <linux/module.h>
  19 #include <linux/platform_device.h>
  20 #include <linux/sa11x0-dma.h>
  21 #include <linux/slab.h>
  22 #include <linux/spinlock.h>
  23
  24 #include "virt-dma.h"
  25
  26 #define NR_PHY_CHAN     6
  27 #define DMA_ALIGN       3
  28 #define DMA_MAX_SIZE    0x1fff
  29 #define DMA_CHUNK_SIZE  0x1000
  30
  31 #define DMA_DDAR        0x00
  32 #define DMA_DCSR_S      0x04
  33 #define DMA_DCSR_C      0x08
  34 #define DMA_DCSR_R      0x0c
  35 #define DMA_DBSA        0x10
  36 #define DMA_DBTA        0x14
  37 #define DMA_DBSB        0x18
  38 #define DMA_DBTB        0x1c
  39 #define DMA_SIZE        0x20
  40
  41 #define DCSR_RUN        (1 << 0)
  42 #define DCSR_IE         (1 << 1)
  43 #define DCSR_ERROR      (1 << 2)
  44 #define DCSR_DONEA      (1 << 3)
  45 #define DCSR_STRTA      (1 << 4)
  46 #define DCSR_DONEB      (1 << 5)
  47 #define DCSR_STRTB      (1 << 6)
  48 #define DCSR_BIU        (1 << 7)
  49
  50 #define DDAR_RW         (1 << 0)        /* 0 = W, 1 = R */
  51 #define DDAR_E          (1 << 1)        /* 0 = LE, 1 = BE */
  52 #define DDAR_BS         (1 << 2)        /* 0 = BS4, 1 = BS8 */
  53 #define DDAR_DW         (1 << 3)        /* 0 = 8b, 1 = 16b */
  54 #define DDAR_Ser0UDCTr  (0x0 << 4)
  55 #define DDAR_Ser0UDCRc  (0x1 << 4)
  56 #define DDAR_Ser1SDLCTr (0x2 << 4)
  57 #define DDAR_Ser1SDLCRc (0x3 << 4)
  58 #define DDAR_Ser1UARTTr (0x4 << 4)
  59 #define DDAR_Ser1UARTRc (0x5 << 4)
  60 #define DDAR_Ser2ICPTr  (0x6 << 4)
  61 #define DDAR_Ser2ICPRc  (0x7 << 4)
  62 #define DDAR_Ser3UARTTr (0x8 << 4)
  63 #define DDAR_Ser3UARTRc (0x9 << 4)
  64 #define DDAR_Ser4MCP0Tr (0xa << 4)
  65 #define DDAR_Ser4MCP0Rc (0xb << 4)
  66 #define DDAR_Ser4MCP1Tr (0xc << 4)
  67 #define DDAR_Ser4MCP1Rc (0xd << 4)
  68 #define DDAR_Ser4SSPTr  (0xe << 4)
  69 #define DDAR_Ser4SSPRc  (0xf << 4)
  70
  71 struct sa11x0_dma_sg {
  72         u32                     addr;
  73         u32                     len;
  74 };
  75
  76 struct sa11x0_dma_desc {
  77         struct virt_dma_desc    vd;
  78
  79         u32                     ddar;
  80         size_t                  size;
  81         unsigned                period;
  82         bool                    cyclic;
  83
  84         unsigned                sglen;
  85         struct sa11x0_dma_sg    sg[0];
  86 };
  87
  88 struct sa11x0_dma_phy;
  89
  90 struct sa11x0_dma_chan {
  91         struct virt_dma_chan    vc;
  92
  93         /* protected by c->vc.lock */
  94         struct sa11x0_dma_phy   *phy;
  95         enum dma_status         status;
  96
  97         /* protected by d->lock */
  98         struct list_head        node;
  99
 100         u32                     ddar;
 101         const char              *name;
 102 };
 103
 104 struct sa11x0_dma_phy {
 105         void __iomem            *base;
 106         struct sa11x0_dma_dev   *dev;
 107         unsigned                num;
 108
 109         struct sa11x0_dma_chan  *vchan;
 110
 111         /* Protected by c->vc.lock */
 112         unsigned                sg_load;
 113         struct sa11x0_dma_desc  *txd_load;
 114         unsigned                sg_done;
 115         struct sa11x0_dma_desc  *txd_done;
 116         u32                     dbs[2];
 117         u32                     dbt[2];
 118         u32                     dcsr;
 119 };
 120
 121 struct sa11x0_dma_dev {
 122         struct dma_device       slave;
 123         void __iomem            *base;
 124         spinlock_t              lock;
 125         struct tasklet_struct   task;
 126         struct list_head        chan_pending;
 127         struct sa11x0_dma_phy   phy[NR_PHY_CHAN];
 128 };
 129
 130 static struct sa11x0_dma_chan *to_sa11x0_dma_chan(struct dma_chan *chan)
 131 {
 132         return container_of(chan, struct sa11x0_dma_chan, vc.chan);
 133 }
 134
 135 static struct sa11x0_dma_dev *to_sa11x0_dma(struct dma_device *dmadev)
 136 {
 137         return container_of(dmadev, struct sa11x0_dma_dev, slave);
 138 }
 139
 140 static struct sa11x0_dma_desc *sa11x0_dma_next_desc(struct sa11x0_dma_chan *c)
 141 {
 142         struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
 143
 144         return vd ? container_of(vd, struct sa11x0_dma_desc, vd) : NULL;
 145 }
 146
 147 static void sa11x0_dma_free_desc(struct virt_dma_desc *vd)
 148 {
 149         kfree(container_of(vd, struct sa11x0_dma_desc, vd));
 150 }
 151
 152 static void sa11x0_dma_start_desc(struct sa11x0_dma_phy *p, struct sa11x0_dma_desc *txd)
 153 {
 154         list_del(&txd->vd.node);
 155         p->txd_load = txd;
 156         p->sg_load = 0;
 157
 158         dev_vdbg(p->dev->slave.dev, "pchan %u: txd %p[%x]: starting: DDAR:%x\n",
 159                 p->num, &txd->vd, txd->vd.tx.cookie, txd->ddar);
 160 }
 161
 162 static void noinline sa11x0_dma_start_sg(struct sa11x0_dma_phy *p,
 163         struct sa11x0_dma_chan *c)
 164 {
 165         struct sa11x0_dma_desc *txd = p->txd_load;
 166         struct sa11x0_dma_sg *sg;
 167         void __iomem *base = p->base;
 168         unsigned dbsx, dbtx;
 169         u32 dcsr;
 170
 171         if (!txd)
 172                 return;
 173
 174         dcsr = readl_relaxed(base + DMA_DCSR_R);
 175
 176         /* Don't try to load the next transfer if both buffers are started */
 177         if ((dcsr & (DCSR_STRTA | DCSR_STRTB)) == (DCSR_STRTA | DCSR_STRTB))
 178                 return;
 179
 180         if (p->sg_load == txd->sglen) {
 181                 if (!txd->cyclic) {
 182                         struct sa11x0_dma_desc *txn = sa11x0_dma_next_desc(c);
 183
 184                         /*
 185                          * We have reached the end of the current descriptor.
 186                          * Peek at the next descriptor, and if compatible with
 187                          * the current, start processing it.
 188                          */
 189                         if (txn && txn->ddar == txd->ddar) {
 190                                 txd = txn;
 191                                 sa11x0_dma_start_desc(p, txn);
 192                         } else {
 193                                 p->txd_load = NULL;
 194                                 return;
 195                         }
 196                 } else {
 197                         /* Cyclic: reset back to beginning */
 198                         p->sg_load = 0;
 199                 }
 200         }
 201
 202         sg = &txd->sg[p->sg_load++];
 203
 204         /* Select buffer to load according to channel status */
 205         if (((dcsr & (DCSR_BIU | DCSR_STRTB)) == (DCSR_BIU | DCSR_STRTB)) ||
 206             ((dcsr & (DCSR_BIU | DCSR_STRTA)) == 0)) {
 207                 dbsx = DMA_DBSA;
 208                 dbtx = DMA_DBTA;
 209                 dcsr = DCSR_STRTA | DCSR_IE | DCSR_RUN;
 210         } else {
 211                 dbsx = DMA_DBSB;
 212                 dbtx = DMA_DBTB;
 213                 dcsr = DCSR_STRTB | DCSR_IE | DCSR_RUN;
 214         }
 215
 216         writel_relaxed(sg->addr, base + dbsx);
 217         writel_relaxed(sg->len, base + dbtx);
 218         writel(dcsr, base + DMA_DCSR_S);
 219
 220         dev_dbg(p->dev->slave.dev, "pchan %u: load: DCSR:%02x DBS%c:%08x DBT%c:%08x\n",
 221                 p->num, dcsr,
 222                 'A' + (dbsx == DMA_DBSB), sg->addr,
 223                 'A' + (dbtx == DMA_DBTB), sg->len);
 224 }
 225
 226 static void noinline sa11x0_dma_complete(struct sa11x0_dma_phy *p,
 227         struct sa11x0_dma_chan *c)
 228 {
 229         struct sa11x0_dma_desc *txd = p->txd_done;
 230
 231         if (++p->sg_done == txd->sglen) {
 232                 if (!txd->cyclic) {
 233                         vchan_cookie_complete(&txd->vd);
 234
 235                         p->sg_done = 0;
 236                         p->txd_done = p->txd_load;
 237
 238                         if (!p->txd_done)
 239                                 tasklet_schedule(&p->dev->task);
 240                 } else {
 241                         if ((p->sg_done % txd->period) == 0)
 242                                 vchan_cyclic_callback(&txd->vd);
 243
 244                         /* Cyclic: reset back to beginning */
 245                         p->sg_done = 0;
 246                 }
 247         }
 248
 249         sa11x0_dma_start_sg(p, c);
 250 }
 251
 252 static irqreturn_t sa11x0_dma_irq(int irq, void *dev_id)
 253 {
 254         struct sa11x0_dma_phy *p = dev_id;
 255         struct sa11x0_dma_dev *d = p->dev;
 256         struct sa11x0_dma_chan *c;
 257         u32 dcsr;
 258
 259         dcsr = readl_relaxed(p->base + DMA_DCSR_R);
 260         if (!(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB)))
 261                 return IRQ_NONE;
 262
 263         /* Clear reported status bits */
 264         writel_relaxed(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB),
 265                 p->base + DMA_DCSR_C);
 266
 267         dev_dbg(d->slave.dev, "pchan %u: irq: DCSR:%02x\n", p->num, dcsr);
 268
 269         if (dcsr & DCSR_ERROR) {
 270                 dev_err(d->slave.dev, "pchan %u: error. DCSR:%02x DDAR:%08x DBSA:%08x DBTA:%08x DBSB:%08x DBTB:%08x\n",
 271                         p->num, dcsr,
 272                         readl_relaxed(p->base + DMA_DDAR),
 273                         readl_relaxed(p->base + DMA_DBSA),
 274                         readl_relaxed(p->base + DMA_DBTA),
 275                         readl_relaxed(p->base + DMA_DBSB),
 276                         readl_relaxed(p->base + DMA_DBTB));
 277         }
 278
 279         c = p->vchan;
 280         if (c) {
 281                 unsigned long flags;
 282
 283                 spin_lock_irqsave(&c->vc.lock, flags);
 284                 /*
 285                  * Now that we're holding the lock, check that the vchan
 286                  * really is associated with this pchan before touching the
 287                  * hardware.  This should always succeed, because we won't
 288                  * change p->vchan or c->phy while the channel is actively
 289                  * transferring.
 290                  */
 291                 if (c->phy == p) {
 292                         if (dcsr & DCSR_DONEA)
 293                                 sa11x0_dma_complete(p, c);
 294                         if (dcsr & DCSR_DONEB)
 295                                 sa11x0_dma_complete(p, c);
 296                 }
 297                 spin_unlock_irqrestore(&c->vc.lock, flags);
 298         }
 299
 300         return IRQ_HANDLED;
 301 }
 302
 303 static void sa11x0_dma_start_txd(struct sa11x0_dma_chan *c)
 304 {
 305         struct sa11x0_dma_desc *txd = sa11x0_dma_next_desc(c);
 306
 307         /* If the issued list is empty, we have no further txds to process */
 308         if (txd) {
 309                 struct sa11x0_dma_phy *p = c->phy;
 310
 311                 sa11x0_dma_start_desc(p, txd);
 312                 p->txd_done = txd;
 313                 p->sg_done = 0;
 314
 315                 /* The channel should not have any transfers started */
 316                 WARN_ON(readl_relaxed(p->base + DMA_DCSR_R) &
 317                                       (DCSR_STRTA | DCSR_STRTB));
 318
 319                 /* Clear the run and start bits before changing DDAR */
 320                 writel_relaxed(DCSR_RUN | DCSR_STRTA | DCSR_STRTB,
 321                                p->base + DMA_DCSR_C);
 322                 writel_relaxed(txd->ddar, p->base + DMA_DDAR);
 323
 324                 /* Try to start both buffers */
 325                 sa11x0_dma_start_sg(p, c);
 326                 sa11x0_dma_start_sg(p, c);
 327         }
 328 }
 329
 330 static void sa11x0_dma_tasklet(unsigned long arg)
 331 {
 332         struct sa11x0_dma_dev *d = (struct sa11x0_dma_dev *)arg;
 333         struct sa11x0_dma_phy *p;
 334         struct sa11x0_dma_chan *c;
 335         unsigned pch, pch_alloc = 0;
 336
 337         dev_dbg(d->slave.dev, "tasklet enter\n");
 338
 339         list_for_each_entry(c, &d->slave.channels, vc.chan.device_node) {
 340                 spin_lock_irq(&c->vc.lock);
 341                 p = c->phy;
 342                 if (p && !p->txd_done) {
 343                         sa11x0_dma_start_txd(c);
 344                         if (!p->txd_done) {
 345                                 /* No current txd associated with this channel */
 346                                 dev_dbg(d->slave.dev, "pchan %u: free\n", p->num);
 347
 348                                 /* Mark this channel free */
 349                                 c->phy = NULL;
 350                                 p->vchan = NULL;
 351                         }
 352                 }
 353                 spin_unlock_irq(&c->vc.lock);
 354         }
 355
 356         spin_lock_irq(&d->lock);
 357         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
 358                 p = &d->phy[pch];
 359
 360                 if (p->vchan == NULL && !list_empty(&d->chan_pending)) {
 361                         c = list_first_entry(&d->chan_pending,
 362                                 struct sa11x0_dma_chan, node);
 363                         list_del_init(&c->node);
 364
 365                         pch_alloc |= 1 << pch;
 366
 367                         /* Mark this channel allocated */
 368                         p->vchan = c;
 369
 370                         dev_dbg(d->slave.dev, "pchan %u: alloc vchan %p\n", pch, &c->vc);
 371                 }
 372         }
 373         spin_unlock_irq(&d->lock);
 374
 375         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
 376                 if (pch_alloc & (1 << pch)) {
 377                         p = &d->phy[pch];
 378                         c = p->vchan;
 379
 380                         spin_lock_irq(&c->vc.lock);
 381                         c->phy = p;
 382
 383                         sa11x0_dma_start_txd(c);
 384                         spin_unlock_irq(&c->vc.lock);
 385                 }
 386         }
 387
 388         dev_dbg(d->slave.dev, "tasklet exit\n");
 389 }
 390
 391
 392 static int sa11x0_dma_alloc_chan_resources(struct dma_chan *chan)
 393 {
 394         return 0;
 395 }
 396
 397 static void sa11x0_dma_free_chan_resources(struct dma_chan *chan)
 398 {
 399         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 400         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 401         unsigned long flags;
 402
 403         spin_lock_irqsave(&d->lock, flags);
 404         list_del_init(&c->node);
 405         spin_unlock_irqrestore(&d->lock, flags);
 406
 407         vchan_free_chan_resources(&c->vc);
 408 }
 409
 410 static dma_addr_t sa11x0_dma_pos(struct sa11x0_dma_phy *p)
 411 {
 412         unsigned reg;
 413         u32 dcsr;
 414
 415         dcsr = readl_relaxed(p->base + DMA_DCSR_R);
 416
 417         if ((dcsr & (DCSR_BIU | DCSR_STRTA)) == DCSR_STRTA ||
 418             (dcsr & (DCSR_BIU | DCSR_STRTB)) == DCSR_BIU)
 419                 reg = DMA_DBSA;
 420         else
 421                 reg = DMA_DBSB;
 422
 423         return readl_relaxed(p->base + reg);
 424 }
 425
 426 static enum dma_status sa11x0_dma_tx_status(struct dma_chan *chan,
 427         dma_cookie_t cookie, struct dma_tx_state *state)
 428 {
 429         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 430         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 431         struct sa11x0_dma_phy *p;
 432         struct virt_dma_desc *vd;
 433         unsigned long flags;
 434         enum dma_status ret;
 435
 436         ret = dma_cookie_status(&c->vc.chan, cookie, state);
 437         if (ret == DMA_COMPLETE)
 438                 return ret;
 439
 440         if (!state)
 441                 return c->status;
 442
 443         spin_lock_irqsave(&c->vc.lock, flags);
 444         p = c->phy;
 445
 446         /*
 447          * If the cookie is on our issue queue, then the residue is
 448          * its total size.
 449          */
 450         vd = vchan_find_desc(&c->vc, cookie);
 451         if (vd) {
 452                 state->residue = container_of(vd, struct sa11x0_dma_desc, vd)->size;
 453         } else if (!p) {
 454                 state->residue = 0;
 455         } else {
 456                 struct sa11x0_dma_desc *txd;
 457                 size_t bytes = 0;
 458
 459                 if (p->txd_done && p->txd_done->vd.tx.cookie == cookie)
 460                         txd = p->txd_done;
 461                 else if (p->txd_load && p->txd_load->vd.tx.cookie == cookie)
 462                         txd = p->txd_load;
 463                 else
 464                         txd = NULL;
 465
 466                 ret = c->status;
 467                 if (txd) {
 468                         dma_addr_t addr = sa11x0_dma_pos(p);
 469                         unsigned i;
 470
 471                         dev_vdbg(d->slave.dev, "tx_status: addr:%x\n", addr);
 472
 473                         for (i = 0; i < txd->sglen; i++) {
 474                                 dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x\n",
 475                                         i, txd->sg[i].addr, txd->sg[i].len);
 476                                 if (addr >= txd->sg[i].addr &&
 477                                     addr < txd->sg[i].addr + txd->sg[i].len) {
 478                                         unsigned len;
 479
 480                                         len = txd->sg[i].len -
 481                                                 (addr - txd->sg[i].addr);
 482                                         dev_vdbg(d->slave.dev, "tx_status: [%u] +%x\n",
 483                                                 i, len);
 484                                         bytes += len;
 485                                         i++;
 486                                         break;
 487                                 }
 488                         }
 489                         for (; i < txd->sglen; i++) {
 490                                 dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x ++\n",
 491                                         i, txd->sg[i].addr, txd->sg[i].len);
 492                                 bytes += txd->sg[i].len;
 493                         }
 494                 }
 495                 state->residue = bytes;
 496         }
 497         spin_unlock_irqrestore(&c->vc.lock, flags);
 498
 499         dev_vdbg(d->slave.dev, "tx_status: bytes 0x%zx\n", state->residue);
 500
 501         return ret;
 502 }
 503
 504 /*
 505  * Move pending txds to the issued list, and re-init pending list.
 506  * If not already pending, add this channel to the list of pending
 507  * channels and trigger the tasklet to run.
 508  */
 509 static void sa11x0_dma_issue_pending(struct dma_chan *chan)
 510 {
 511         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 512         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 513         unsigned long flags;
 514
 515         spin_lock_irqsave(&c->vc.lock, flags);
 516         if (vchan_issue_pending(&c->vc)) {
 517                 if (!c->phy) {
 518                         spin_lock(&d->lock);
 519                         if (list_empty(&c->node)) {
 520                                 list_add_tail(&c->node, &d->chan_pending);
 521                                 tasklet_schedule(&d->task);
 522                                 dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc);
 523                         }
 524                         spin_unlock(&d->lock);
 525                 }
 526         } else
 527                 dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc);
 528         spin_unlock_irqrestore(&c->vc.lock, flags);
 529 }
 530
 531 static struct dma_async_tx_descriptor *sa11x0_dma_prep_slave_sg(
 532         struct dma_chan *chan, struct scatterlist *sg, unsigned int sglen,
 533         enum dma_transfer_direction dir, unsigned long flags, void *context)
 534 {
 535         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 536         struct sa11x0_dma_desc *txd;
 537         struct scatterlist *sgent;
 538         unsigned i, j = sglen;
 539         size_t size = 0;
 540
 541         /* SA11x0 channels can only operate in their native direction */
 542         if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {
 543                 dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",
 544                         &c->vc, c->ddar, dir);
 545                 return NULL;
 546         }
 547
 548         /* Do not allow zero-sized txds */
 549         if (sglen == 0)
 550                 return NULL;
 551
 552         for_each_sg(sg, sgent, sglen, i) {
 553                 dma_addr_t addr = sg_dma_address(sgent);
 554                 unsigned int len = sg_dma_len(sgent);
 555
 556                 if (len > DMA_MAX_SIZE)
 557                         j += DIV_ROUND_UP(len, DMA_MAX_SIZE & ~DMA_ALIGN) - 1;
 558                 if (addr & DMA_ALIGN) {
 559                         dev_dbg(chan->device->dev, "vchan %p: bad buffer alignment: %08x\n",
 560                                 &c->vc, addr);
 561                         return NULL;
 562                 }
 563         }
 564
 565         txd = kzalloc(sizeof(*txd) + j * sizeof(txd->sg[0]), GFP_ATOMIC);
 566         if (!txd) {
 567                 dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc);
 568                 return NULL;
 569         }
 570
 571         j = 0;
 572         for_each_sg(sg, sgent, sglen, i) {
 573                 dma_addr_t addr = sg_dma_address(sgent);
 574                 unsigned len = sg_dma_len(sgent);
 575
 576                 size += len;
 577
 578                 do {
 579                         unsigned tlen = len;
 580
 581                         /*
 582                          * Check whether the transfer will fit.  If not, try
 583                          * to split the transfer up such that we end up with
 584                          * equal chunks - but make sure that we preserve the
 585                          * alignment.  This avoids small segments.
 586                          */
 587                         if (tlen > DMA_MAX_SIZE) {
 588                                 unsigned mult = DIV_ROUND_UP(tlen,
 589                                         DMA_MAX_SIZE & ~DMA_ALIGN);
 590
 591                                 tlen = (tlen / mult) & ~DMA_ALIGN;
 592                         }
 593
 594                         txd->sg[j].addr = addr;
 595                         txd->sg[j].len = tlen;
 596
 597                         addr += tlen;
 598                         len -= tlen;
 599                         j++;
 600                 } while (len);
 601         }
 602
 603         txd->ddar = c->ddar;
 604         txd->size = size;
 605         txd->sglen = j;
 606
 607         dev_dbg(chan->device->dev, "vchan %p: txd %p: size %u nr %u\n",
 608                 &c->vc, &txd->vd, txd->size, txd->sglen);
 609
 610         return vchan_tx_prep(&c->vc, &txd->vd, flags);
 611 }
 612
 613 static struct dma_async_tx_descriptor *sa11x0_dma_prep_dma_cyclic(
 614         struct dma_chan *chan, dma_addr_t addr, size_t size, size_t period,
 615         enum dma_transfer_direction dir, unsigned long flags)
 616 {
 617         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 618         struct sa11x0_dma_desc *txd;
 619         unsigned i, j, k, sglen, sgperiod;
 620
 621         /* SA11x0 channels can only operate in their native direction */
 622         if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {
 623                 dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",
 624                         &c->vc, c->ddar, dir);
 625                 return NULL;
 626         }
 627
 628         sgperiod = DIV_ROUND_UP(period, DMA_MAX_SIZE & ~DMA_ALIGN);
 629         sglen = size * sgperiod / period;
 630
 631         /* Do not allow zero-sized txds */
 632         if (sglen == 0)
 633                 return NULL;
 634
 635         txd = kzalloc(sizeof(*txd) + sglen * sizeof(txd->sg[0]), GFP_ATOMIC);
 636         if (!txd) {
 637                 dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc);
 638                 return NULL;
 639         }
 640
 641         for (i = k = 0; i < size / period; i++) {
 642                 size_t tlen, len = period;
 643
 644                 for (j = 0; j < sgperiod; j++, k++) {
 645                         tlen = len;
 646
 647                         if (tlen > DMA_MAX_SIZE) {
 648                                 unsigned mult = DIV_ROUND_UP(tlen, DMA_MAX_SIZE & ~DMA_ALIGN);
 649                                 tlen = (tlen / mult) & ~DMA_ALIGN;
 650                         }
 651
 652                         txd->sg[k].addr = addr;
 653                         txd->sg[k].len = tlen;
 654                         addr += tlen;
 655                         len -= tlen;
 656                 }
 657
 658                 WARN_ON(len != 0);
 659         }
 660
 661         WARN_ON(k != sglen);
 662
 663         txd->ddar = c->ddar;
 664         txd->size = size;
 665         txd->sglen = sglen;
 666         txd->cyclic = 1;
 667         txd->period = sgperiod;
 668
 669         return vchan_tx_prep(&c->vc, &txd->vd, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 670 }
 671
 672 static int sa11x0_dma_slave_config(struct sa11x0_dma_chan *c, struct dma_slave_config *cfg)
 673 {
 674         u32 ddar = c->ddar & ((0xf << 4) | DDAR_RW);
 675         dma_addr_t addr;
 676         enum dma_slave_buswidth width;
 677         u32 maxburst;
 678
 679         if (ddar & DDAR_RW) {
 680                 addr = cfg->src_addr;
 681                 width = cfg->src_addr_width;
 682                 maxburst = cfg->src_maxburst;
 683         } else {
 684                 addr = cfg->dst_addr;
 685                 width = cfg->dst_addr_width;
 686                 maxburst = cfg->dst_maxburst;
 687         }
 688
 689         if ((width != DMA_SLAVE_BUSWIDTH_1_BYTE &&
 690              width != DMA_SLAVE_BUSWIDTH_2_BYTES) ||
 691             (maxburst != 4 && maxburst != 8))
 692                 return -EINVAL;
 693
 694         if (width == DMA_SLAVE_BUSWIDTH_2_BYTES)
 695                 ddar |= DDAR_DW;
 696         if (maxburst == 8)
 697                 ddar |= DDAR_BS;
 698
 699         dev_dbg(c->vc.chan.device->dev, "vchan %p: dma_slave_config addr %x width %u burst %u\n",
 700                 &c->vc, addr, width, maxburst);
 701
 702         c->ddar = ddar | (addr & 0xf0000000) | (addr & 0x003ffffc) << 6;
 703
 704         return 0;
 705 }
 706
 707 static int sa11x0_dma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
 708         unsigned long arg)
 709 {
 710         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 711         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 712         struct sa11x0_dma_phy *p;
 713         LIST_HEAD(head);
 714         unsigned long flags;
 715         int ret;
 716
 717         switch (cmd) {
 718         case DMA_SLAVE_CONFIG:
 719                 return sa11x0_dma_slave_config(c, (struct dma_slave_config *)arg);
 720
 721         case DMA_TERMINATE_ALL:
 722                 dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc);
 723                 /* Clear the tx descriptor lists */
 724                 spin_lock_irqsave(&c->vc.lock, flags);
 725                 vchan_get_all_descriptors(&c->vc, &head);
 726
 727                 p = c->phy;
 728                 if (p) {
 729                         dev_dbg(d->slave.dev, "pchan %u: terminating\n", p->num);
 730                         /* vchan is assigned to a pchan - stop the channel */
 731                         writel(DCSR_RUN | DCSR_IE |
 732                                 DCSR_STRTA | DCSR_DONEA |
 733                                 DCSR_STRTB | DCSR_DONEB,
 734                                 p->base + DMA_DCSR_C);
 735
 736                         if (p->txd_load) {
 737                                 if (p->txd_load != p->txd_done)
 738                                         list_add_tail(&p->txd_load->vd.node, &head);
 739                                 p->txd_load = NULL;
 740                         }
 741                         if (p->txd_done) {
 742                                 list_add_tail(&p->txd_done->vd.node, &head);
 743                                 p->txd_done = NULL;
 744                         }
 745                         c->phy = NULL;
 746                         spin_lock(&d->lock);
 747                         p->vchan = NULL;
 748                         spin_unlock(&d->lock);
 749                         tasklet_schedule(&d->task);
 750                 }
 751                 spin_unlock_irqrestore(&c->vc.lock, flags);
 752                 vchan_dma_desc_free_list(&c->vc, &head);
 753                 ret = 0;
 754                 break;
 755
 756         case DMA_PAUSE:
 757                 dev_dbg(d->slave.dev, "vchan %p: pause\n", &c->vc);
 758                 spin_lock_irqsave(&c->vc.lock, flags);
 759                 if (c->status == DMA_IN_PROGRESS) {
 760                         c->status = DMA_PAUSED;
 761
 762                         p = c->phy;
 763                         if (p) {
 764                                 writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
 765                         } else {
 766                                 spin_lock(&d->lock);
 767                                 list_del_init(&c->node);
 768                                 spin_unlock(&d->lock);
 769                         }
 770                 }
 771                 spin_unlock_irqrestore(&c->vc.lock, flags);
 772                 ret = 0;
 773                 break;
 774
 775         case DMA_RESUME:
 776                 dev_dbg(d->slave.dev, "vchan %p: resume\n", &c->vc);
 777                 spin_lock_irqsave(&c->vc.lock, flags);
 778                 if (c->status == DMA_PAUSED) {
 779                         c->status = DMA_IN_PROGRESS;
 780
 781                         p = c->phy;
 782                         if (p) {
 783                                 writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_S);
 784                         } else if (!list_empty(&c->vc.desc_issued)) {
 785                                 spin_lock(&d->lock);
 786                                 list_add_tail(&c->node, &d->chan_pending);
 787                                 spin_unlock(&d->lock);
 788                         }
 789                 }
 790                 spin_unlock_irqrestore(&c->vc.lock, flags);
 791                 ret = 0;
 792                 break;
 793
 794         default:
 795                 ret = -ENXIO;
 796                 break;
 797         }
 798
 799         return ret;
 800 }
 801
 802 struct sa11x0_dma_channel_desc {
 803         u32 ddar;
 804         const char *name;
 805 };
 806
 807 #define CD(d1, d2) { .ddar = DDAR_##d1 | d2, .name = #d1 }
 808 static const struct sa11x0_dma_channel_desc chan_desc[] = {
 809         CD(Ser0UDCTr, 0),
 810         CD(Ser0UDCRc, DDAR_RW),
 811         CD(Ser1SDLCTr, 0),
 812         CD(Ser1SDLCRc, DDAR_RW),
 813         CD(Ser1UARTTr, 0),
 814         CD(Ser1UARTRc, DDAR_RW),
 815         CD(Ser2ICPTr, 0),
 816         CD(Ser2ICPRc, DDAR_RW),
 817         CD(Ser3UARTTr, 0),
 818         CD(Ser3UARTRc, DDAR_RW),
 819         CD(Ser4MCP0Tr, 0),
 820         CD(Ser4MCP0Rc, DDAR_RW),
 821         CD(Ser4MCP1Tr, 0),
 822         CD(Ser4MCP1Rc, DDAR_RW),
 823         CD(Ser4SSPTr, 0),
 824         CD(Ser4SSPRc, DDAR_RW),
 825 };
 826
 827 static int sa11x0_dma_init_dmadev(struct dma_device *dmadev,
 828         struct device *dev)
 829 {
 830         unsigned i;
 831
 832         dmadev->chancnt = ARRAY_SIZE(chan_desc);
 833         INIT_LIST_HEAD(&dmadev->channels);
 834         dmadev->dev = dev;
 835         dmadev->device_alloc_chan_resources = sa11x0_dma_alloc_chan_resources;
 836         dmadev->device_free_chan_resources = sa11x0_dma_free_chan_resources;
 837         dmadev->device_control = sa11x0_dma_control;
 838         dmadev->device_tx_status = sa11x0_dma_tx_status;
 839         dmadev->device_issue_pending = sa11x0_dma_issue_pending;
 840
 841         for (i = 0; i < dmadev->chancnt; i++) {
 842                 struct sa11x0_dma_chan *c;
 843
 844                 c = kzalloc(sizeof(*c), GFP_KERNEL);
 845                 if (!c) {
 846                         dev_err(dev, "no memory for channel %u\n", i);
 847                         return -ENOMEM;
 848                 }
 849
 850                 c->status = DMA_IN_PROGRESS;
 851                 c->ddar = chan_desc[i].ddar;
 852                 c->name = chan_desc[i].name;
 853                 INIT_LIST_HEAD(&c->node);
 854
 855                 c->vc.desc_free = sa11x0_dma_free_desc;
 856                 vchan_init(&c->vc, dmadev);
 857         }
 858
 859         return dma_async_device_register(dmadev);
 860 }
 861
 862 static int sa11x0_dma_request_irq(struct platform_device *pdev, int nr,
 863         void *data)
 864 {
 865         int irq = platform_get_irq(pdev, nr);
 866
 867         if (irq <= 0)
 868                 return -ENXIO;
 869
 870         return request_irq(irq, sa11x0_dma_irq, 0, dev_name(&pdev->dev), data);
 871 }
 872
 873 static void sa11x0_dma_free_irq(struct platform_device *pdev, int nr,
 874         void *data)
 875 {
 876         int irq = platform_get_irq(pdev, nr);
 877         if (irq > 0)
 878                 free_irq(irq, data);
 879 }
 880
 881 static void sa11x0_dma_free_channels(struct dma_device *dmadev)
 882 {
 883         struct sa11x0_dma_chan *c, *cn;
 884
 885         list_for_each_entry_safe(c, cn, &dmadev->channels, vc.chan.device_node) {
 886                 list_del(&c->vc.chan.device_node);
 887                 tasklet_kill(&c->vc.task);
 888                 kfree(c);
 889         }
 890 }
 891
 892 static int sa11x0_dma_probe(struct platform_device *pdev)
 893 {
 894         struct sa11x0_dma_dev *d;
 895         struct resource *res;
 896         unsigned i;
 897         int ret;
 898
 899         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 900         if (!res)
 901                 return -ENXIO;
 902
 903         d = kzalloc(sizeof(*d), GFP_KERNEL);
 904         if (!d) {
 905                 ret = -ENOMEM;
 906                 goto err_alloc;
 907         }
 908
 909         spin_lock_init(&d->lock);
 910         INIT_LIST_HEAD(&d->chan_pending);
 911
 912         d->base = ioremap(res->start, resource_size(res));
 913         if (!d->base) {
 914                 ret = -ENOMEM;
 915                 goto err_ioremap;
 916         }
 917
 918         tasklet_init(&d->task, sa11x0_dma_tasklet, (unsigned long)d);
 919
 920         for (i = 0; i < NR_PHY_CHAN; i++) {
 921                 struct sa11x0_dma_phy *p = &d->phy[i];
 922
 923                 p->dev = d;
 924                 p->num = i;
 925                 p->base = d->base + i * DMA_SIZE;
 926                 writel_relaxed(DCSR_RUN | DCSR_IE | DCSR_ERROR |
 927                         DCSR_DONEA | DCSR_STRTA | DCSR_DONEB | DCSR_STRTB,
 928                         p->base + DMA_DCSR_C);
 929                 writel_relaxed(0, p->base + DMA_DDAR);
 930
 931                 ret = sa11x0_dma_request_irq(pdev, i, p);
 932                 if (ret) {
 933                         while (i) {
 934                                 i--;
 935                                 sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
 936                         }
 937                         goto err_irq;
 938                 }
 939         }
 940
 941         dma_cap_set(DMA_SLAVE, d->slave.cap_mask);
 942         dma_cap_set(DMA_CYCLIC, d->slave.cap_mask);
 943         d->slave.device_prep_slave_sg = sa11x0_dma_prep_slave_sg;
 944         d->slave.device_prep_dma_cyclic = sa11x0_dma_prep_dma_cyclic;
 945         ret = sa11x0_dma_init_dmadev(&d->slave, &pdev->dev);
 946         if (ret) {
 947                 dev_warn(d->slave.dev, "failed to register slave async device: %d\n",
 948                         ret);
 949                 goto err_slave_reg;
 950         }
 951
 952         platform_set_drvdata(pdev, d);
 953         return 0;
 954
 955  err_slave_reg:
 956         sa11x0_dma_free_channels(&d->slave);
 957         for (i = 0; i < NR_PHY_CHAN; i++)
 958                 sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
 959  err_irq:
 960         tasklet_kill(&d->task);
 961         iounmap(d->base);
 962  err_ioremap:
 963         kfree(d);
 964  err_alloc:
 965         return ret;
 966 }
 967
 968 static int sa11x0_dma_remove(struct platform_device *pdev)
 969 {
 970         struct sa11x0_dma_dev *d = platform_get_drvdata(pdev);
 971         unsigned pch;
 972
 973         dma_async_device_unregister(&d->slave);
 974
 975         sa11x0_dma_free_channels(&d->slave);
 976         for (pch = 0; pch < NR_PHY_CHAN; pch++)
 977                 sa11x0_dma_free_irq(pdev, pch, &d->phy[pch]);
 978         tasklet_kill(&d->task);
 979         iounmap(d->base);
 980         kfree(d);
 981
 982         return 0;
 983 }
 984
 985 static int sa11x0_dma_suspend(struct device *dev)
 986 {
 987         struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
 988         unsigned pch;
 989
 990         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
 991                 struct sa11x0_dma_phy *p = &d->phy[pch];
 992                 u32 dcsr, saved_dcsr;
 993
 994                 dcsr = saved_dcsr = readl_relaxed(p->base + DMA_DCSR_R);
 995                 if (dcsr & DCSR_RUN) {
 996                         writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
 997                         dcsr = readl_relaxed(p->base + DMA_DCSR_R);
 998                 }
 999
1000                 saved_dcsr &= DCSR_RUN | DCSR_IE;
1001                 if (dcsr & DCSR_BIU) {
1002                         p->dbs[0] = readl_relaxed(p->base + DMA_DBSB);
1003                         p->dbt[0] = readl_relaxed(p->base + DMA_DBTB);
1004                         p->dbs[1] = readl_relaxed(p->base + DMA_DBSA);
1005                         p->dbt[1] = readl_relaxed(p->base + DMA_DBTA);
1006                         saved_dcsr |= (dcsr & DCSR_STRTA ? DCSR_STRTB : 0) |
1007                                       (dcsr & DCSR_STRTB ? DCSR_STRTA : 0);
1008                 } else {
1009                         p->dbs[0] = readl_relaxed(p->base + DMA_DBSA);
1010                         p->dbt[0] = readl_relaxed(p->base + DMA_DBTA);
1011                         p->dbs[1] = readl_relaxed(p->base + DMA_DBSB);
1012                         p->dbt[1] = readl_relaxed(p->base + DMA_DBTB);
1013                         saved_dcsr |= dcsr & (DCSR_STRTA | DCSR_STRTB);
1014                 }
1015                 p->dcsr = saved_dcsr;
1016
1017                 writel(DCSR_STRTA | DCSR_STRTB, p->base + DMA_DCSR_C);
1018         }
1019
1020         return 0;
1021 }
1022
1023 static int sa11x0_dma_resume(struct device *dev)
1024 {
1025         struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
1026         unsigned pch;
1027
1028         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
1029                 struct sa11x0_dma_phy *p = &d->phy[pch];
1030                 struct sa11x0_dma_desc *txd = NULL;
1031                 u32 dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1032
1033                 WARN_ON(dcsr & (DCSR_BIU | DCSR_STRTA | DCSR_STRTB | DCSR_RUN));
1034
1035                 if (p->txd_done)
1036                         txd = p->txd_done;
1037                 else if (p->txd_load)
1038                         txd = p->txd_load;
1039
1040                 if (!txd)
1041                         continue;
1042
1043                 writel_relaxed(txd->ddar, p->base + DMA_DDAR);
1044
1045                 writel_relaxed(p->dbs[0], p->base + DMA_DBSA);
1046                 writel_relaxed(p->dbt[0], p->base + DMA_DBTA);
1047                 writel_relaxed(p->dbs[1], p->base + DMA_DBSB);
1048                 writel_relaxed(p->dbt[1], p->base + DMA_DBTB);
1049                 writel_relaxed(p->dcsr, p->base + DMA_DCSR_S);
1050         }
1051
1052         return 0;
1053 }
1054
1055 static const struct dev_pm_ops sa11x0_dma_pm_ops = {
1056         .suspend_noirq = sa11x0_dma_suspend,
1057         .resume_noirq = sa11x0_dma_resume,
1058         .freeze_noirq = sa11x0_dma_suspend,
1059         .thaw_noirq = sa11x0_dma_resume,
1060         .poweroff_noirq = sa11x0_dma_suspend,
1061         .restore_noirq = sa11x0_dma_resume,
1062 };
1063
1064 static struct platform_driver sa11x0_dma_driver = {
1065         .driver = {
1066                 .name   = "sa11x0-dma",
1067                 .owner  = THIS_MODULE,
1068                 .pm     = &sa11x0_dma_pm_ops,
1069         },
1070         .probe          = sa11x0_dma_probe,
1071         .remove         = sa11x0_dma_remove,
1072 };
1073
1074 bool sa11x0_dma_filter_fn(struct dma_chan *chan, void *param)
1075 {
1076         if (chan->device->dev->driver == &sa11x0_dma_driver.driver) {
1077                 struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
1078                 const char *p = param;
1079
1080                 return !strcmp(c->name, p);
1081         }
1082         return false;
1083 }
1084 EXPORT_SYMBOL(sa11x0_dma_filter_fn);
1085
1086 static int __init sa11x0_dma_init(void)
1087 {
1088         return platform_driver_register(&sa11x0_dma_driver);
1089 }
1090 subsys_initcall(sa11x0_dma_init);
1091
1092 static void __exit sa11x0_dma_exit(void)
1093 {
1094         platform_driver_unregister(&sa11x0_dma_driver);
1095 }
1096 module_exit(sa11x0_dma_exit);
1097
1098 MODULE_AUTHOR("Russell King");
1099 MODULE_DESCRIPTION("SA-11x0 DMA driver");
1100 MODULE_LICENSE("GPL v2");
1101 MODULE_ALIAS("platform:sa11x0-dma");