drivers/usb/gadget/gr_udc.c

   1 /*
   2  * USB Peripheral Controller driver for Aeroflex Gaisler GRUSBDC.
   3  *
   4  * 2013 (c) Aeroflex Gaisler AB
   5  *
   6  * This driver supports GRUSBDC USB Device Controller cores available in the
   7  * GRLIB VHDL IP core library.
   8  *
   9  * Full documentation of the GRUSBDC core can be found here:
  10  * http://www.gaisler.com/products/grlib/grip.pdf
  11  *
  12  * This program is free software; you can redistribute it and/or modify it
  13  * under the terms of the GNU General Public License as published by the
  14  * Free Software Foundation; either version 2 of the License, or (at your
  15  * option) any later version.
  16  *
  17  * Contributors:
  18  * - Andreas Larsson <andreas@gaisler.com>
  19  * - Marko Isomaki
  20  */
  21
  22 /*
  23  * A GRUSBDC core can have up to 16 IN endpoints and 16 OUT endpoints each
  24  * individually configurable to any of the four USB transfer types. This driver
  25  * only supports cores in DMA mode.
  26  */
  27
  28 #include <linux/kernel.h>
  29 #include <linux/module.h>
  30 #include <linux/slab.h>
  31 #include <linux/spinlock.h>
  32 #include <linux/errno.h>
  33 #include <linux/list.h>
  34 #include <linux/interrupt.h>
  35 #include <linux/device.h>
  36 #include <linux/usb/ch9.h>
  37 #include <linux/usb/gadget.h>
  38 #include <linux/dma-mapping.h>
  39 #include <linux/dmapool.h>
  40 #include <linux/debugfs.h>
  41 #include <linux/seq_file.h>
  42 #include <linux/of_platform.h>
  43 #include <linux/of_irq.h>
  44 #include <linux/of_address.h>
  45
  46 #include <asm/byteorder.h>
  47
  48 #include "gr_udc.h"
  49
  50 #define DRIVER_NAME     "gr_udc"
  51 #define DRIVER_DESC     "Aeroflex Gaisler GRUSBDC USB Peripheral Controller"
  52
  53 static const char driver_name[] = DRIVER_NAME;
  54 static const char driver_desc[] = DRIVER_DESC;
  55
  56 #define gr_read32(x) (ioread32be((x)))
  57 #define gr_write32(x, v) (iowrite32be((v), (x)))
  58
  59 /* USB speed and corresponding string calculated from status register value */
  60 #define GR_SPEED(status) \
  61         ((status & GR_STATUS_SP) ? USB_SPEED_FULL : USB_SPEED_HIGH)
  62 #define GR_SPEED_STR(status) usb_speed_string(GR_SPEED(status))
  63
  64 /* Size of hardware buffer calculated from epctrl register value */
  65 #define GR_BUFFER_SIZE(epctrl)                                        \
  66         ((((epctrl) & GR_EPCTRL_BUFSZ_MASK) >> GR_EPCTRL_BUFSZ_POS) * \
  67          GR_EPCTRL_BUFSZ_SCALER)
  68
  69 /* ---------------------------------------------------------------------- */
  70 /* Debug printout functionality */
  71
  72 static const char * const gr_modestring[] = {"control", "iso", "bulk", "int"};
  73
  74 static const char *gr_ep0state_string(enum gr_ep0state state)
  75 {
  76         static const char *const names[] = {
  77                 [GR_EP0_DISCONNECT] = "disconnect",
  78                 [GR_EP0_SETUP] = "setup",
  79                 [GR_EP0_IDATA] = "idata",
  80                 [GR_EP0_ODATA] = "odata",
  81                 [GR_EP0_ISTATUS] = "istatus",
  82                 [GR_EP0_OSTATUS] = "ostatus",
  83                 [GR_EP0_STALL] = "stall",
  84                 [GR_EP0_SUSPEND] = "suspend",
  85         };
  86
  87         if (state < 0 || state >= ARRAY_SIZE(names))
  88                 return "UNKNOWN";
  89
  90         return names[state];
  91 }
  92
  93 #ifdef VERBOSE_DEBUG
  94
  95 static void gr_dbgprint_request(const char *str, struct gr_ep *ep,
  96                                 struct gr_request *req)
  97 {
  98         int buflen = ep->is_in ? req->req.length : req->req.actual;
  99         int rowlen = 32;
 100         int plen = min(rowlen, buflen);
 101
 102         dev_dbg(ep->dev->dev, "%s: 0x%p, %d bytes data%s:\n", str, req, buflen,
 103                 (buflen > plen ? " (truncated)" : ""));
 104         print_hex_dump_debug("   ", DUMP_PREFIX_NONE,
 105                              rowlen, 4, req->req.buf, plen, false);
 106 }
 107
 108 static void gr_dbgprint_devreq(struct gr_udc *dev, u8 type, u8 request,
 109                                u16 value, u16 index, u16 length)
 110 {
 111         dev_vdbg(dev->dev, "REQ: %02x.%02x v%04x i%04x l%04x\n",
 112                  type, request, value, index, length);
 113 }
 114 #else /* !VERBOSE_DEBUG */
 115
 116 static void gr_dbgprint_request(const char *str, struct gr_ep *ep,
 117                                 struct gr_request *req) {}
 118
 119 static void gr_dbgprint_devreq(struct gr_udc *dev, u8 type, u8 request,
 120                                u16 value, u16 index, u16 length) {}
 121
 122 #endif /* VERBOSE_DEBUG */
 123
 124 /* ---------------------------------------------------------------------- */
 125 /* Debugfs functionality */
 126
 127 #ifdef CONFIG_USB_GADGET_DEBUG_FS
 128
 129 static void gr_seq_ep_show(struct seq_file *seq, struct gr_ep *ep)
 130 {
 131         u32 epctrl = gr_read32(&ep->regs->epctrl);
 132         u32 epstat = gr_read32(&ep->regs->epstat);
 133         int mode = (epctrl & GR_EPCTRL_TT_MASK) >> GR_EPCTRL_TT_POS;
 134         struct gr_request *req;
 135
 136         seq_printf(seq, "%s:\n", ep->ep.name);
 137         seq_printf(seq, "  mode = %s\n", gr_modestring[mode]);
 138         seq_printf(seq, "  halted: %d\n", !!(epctrl & GR_EPCTRL_EH));
 139         seq_printf(seq, "  disabled: %d\n", !!(epctrl & GR_EPCTRL_ED));
 140         seq_printf(seq, "  valid: %d\n", !!(epctrl & GR_EPCTRL_EV));
 141         seq_printf(seq, "  dma_start = %d\n", ep->dma_start);
 142         seq_printf(seq, "  stopped = %d\n", ep->stopped);
 143         seq_printf(seq, "  wedged = %d\n", ep->wedged);
 144         seq_printf(seq, "  callback = %d\n", ep->callback);
 145         seq_printf(seq, "  maxpacket = %d\n", ep->ep.maxpacket);
 146         seq_printf(seq, "  bytes_per_buffer = %d\n", ep->bytes_per_buffer);
 147         if (mode == 1 || mode == 3)
 148                 seq_printf(seq, "  nt = %d\n",
 149                            (epctrl & GR_EPCTRL_NT_MASK) >> GR_EPCTRL_NT_POS);
 150
 151         seq_printf(seq, "  Buffer 0: %s %s%d\n",
 152                    epstat & GR_EPSTAT_B0 ? "valid" : "invalid",
 153                    epstat & GR_EPSTAT_BS ? " " : "selected ",
 154                    (epstat & GR_EPSTAT_B0CNT_MASK) >> GR_EPSTAT_B0CNT_POS);
 155         seq_printf(seq, "  Buffer 1: %s %s%d\n",
 156                    epstat & GR_EPSTAT_B1 ? "valid" : "invalid",
 157                    epstat & GR_EPSTAT_BS ? "selected " : " ",
 158                    (epstat & GR_EPSTAT_B1CNT_MASK) >> GR_EPSTAT_B1CNT_POS);
 159
 160         if (list_empty(&ep->queue)) {
 161                 seq_puts(seq, "  Queue: empty\n\n");
 162                 return;
 163         }
 164
 165         seq_puts(seq, "  Queue:\n");
 166         list_for_each_entry(req, &ep->queue, queue) {
 167                 struct gr_dma_desc *desc;
 168                 struct gr_dma_desc *next;
 169
 170                 seq_printf(seq, "    0x%p: 0x%p %d %d\n", req,
 171                            &req->req.buf, req->req.actual, req->req.length);
 172
 173                 next = req->first_desc;
 174                 do {
 175                         desc = next;
 176                         next = desc->next_desc;
 177                         seq_printf(seq, "    %c 0x%p (0x%08x): 0x%05x 0x%08x\n",
 178                                    desc == req->curr_desc ? 'c' : ' ',
 179                                    desc, desc->paddr, desc->ctrl, desc->data);
 180                 } while (desc != req->last_desc);
 181         }
 182         seq_puts(seq, "\n");
 183 }
 184
 185
 186 static int gr_seq_show(struct seq_file *seq, void *v)
 187 {
 188         struct gr_udc *dev = seq->private;
 189         u32 control = gr_read32(&dev->regs->control);
 190         u32 status = gr_read32(&dev->regs->status);
 191         struct gr_ep *ep;
 192
 193         seq_printf(seq, "usb state = %s\n",
 194                    usb_state_string(dev->gadget.state));
 195         seq_printf(seq, "address = %d\n",
 196                    (control & GR_CONTROL_UA_MASK) >> GR_CONTROL_UA_POS);
 197         seq_printf(seq, "speed = %s\n", GR_SPEED_STR(status));
 198         seq_printf(seq, "ep0state = %s\n", gr_ep0state_string(dev->ep0state));
 199         seq_printf(seq, "irq_enabled = %d\n", dev->irq_enabled);
 200         seq_printf(seq, "remote_wakeup = %d\n", dev->remote_wakeup);
 201         seq_printf(seq, "test_mode = %d\n", dev->test_mode);
 202         seq_puts(seq, "\n");
 203
 204         list_for_each_entry(ep, &dev->ep_list, ep_list)
 205                 gr_seq_ep_show(seq, ep);
 206
 207         return 0;
 208 }
 209
 210 static int gr_dfs_open(struct inode *inode, struct file *file)
 211 {
 212         return single_open(file, gr_seq_show, inode->i_private);
 213 }
 214
 215 static const struct file_operations gr_dfs_fops = {
 216         .owner          = THIS_MODULE,
 217         .open           = gr_dfs_open,
 218         .read           = seq_read,
 219         .llseek         = seq_lseek,
 220         .release        = single_release,
 221 };
 222
 223 static void gr_dfs_create(struct gr_udc *dev)
 224 {
 225         const char *name = "gr_udc_state";
 226
 227         dev->dfs_root = debugfs_create_dir(dev_name(dev->dev), NULL);
 228         dev->dfs_state = debugfs_create_file(name, 0444, dev->dfs_root, dev,
 229                                              &gr_dfs_fops);
 230 }
 231
 232 static void gr_dfs_delete(struct gr_udc *dev)
 233 {
 234         /* Handles NULL and ERR pointers internally */
 235         debugfs_remove(dev->dfs_state);
 236         debugfs_remove(dev->dfs_root);
 237 }
 238
 239 #else /* !CONFIG_USB_GADGET_DEBUG_FS */
 240
 241 static void gr_dfs_create(struct gr_udc *dev) {}
 242 static void gr_dfs_delete(struct gr_udc *dev) {}
 243
 244 #endif /* CONFIG_USB_GADGET_DEBUG_FS */
 245
 246 /* ---------------------------------------------------------------------- */
 247 /* DMA and request handling */
 248
 249 /* Allocates a new struct gr_dma_desc, sets paddr and zeroes the rest */
 250 static struct gr_dma_desc *gr_alloc_dma_desc(struct gr_ep *ep, gfp_t gfp_flags)
 251 {
 252         dma_addr_t paddr;
 253         struct gr_dma_desc *dma_desc;
 254
 255         dma_desc = dma_pool_alloc(ep->dev->desc_pool, gfp_flags, &paddr);
 256         if (!dma_desc) {
 257                 dev_err(ep->dev->dev, "Could not allocate from DMA pool\n");
 258                 return NULL;
 259         }
 260
 261         memset(dma_desc, 0, sizeof(*dma_desc));
 262         dma_desc->paddr = paddr;
 263
 264         return dma_desc;
 265 }
 266
 267 static inline void gr_free_dma_desc(struct gr_udc *dev,
 268                                     struct gr_dma_desc *desc)
 269 {
 270         dma_pool_free(dev->desc_pool, desc, (dma_addr_t)desc->paddr);
 271 }
 272
 273 /* Frees the chain of struct gr_dma_desc for the given request */
 274 static void gr_free_dma_desc_chain(struct gr_udc *dev, struct gr_request *req)
 275 {
 276         struct gr_dma_desc *desc;
 277         struct gr_dma_desc *next;
 278
 279         next = req->first_desc;
 280         if (!next)
 281                 return;
 282
 283         do {
 284                 desc = next;
 285                 next = desc->next_desc;
 286                 gr_free_dma_desc(dev, desc);
 287         } while (desc != req->last_desc);
 288
 289         req->first_desc = NULL;
 290         req->curr_desc = NULL;
 291         req->last_desc = NULL;
 292 }
 293
 294 static void gr_ep0_setup(struct gr_udc *dev, struct gr_request *req);
 295
 296 /*
 297  * Frees allocated resources and calls the appropriate completion function/setup
 298  * package handler for a finished request.
 299  *
 300  * Must be called with dev->lock held and irqs disabled.
 301  */
 302 static void gr_finish_request(struct gr_ep *ep, struct gr_request *req,
 303                               int status)
 304         __releases(&dev->lock)
 305         __acquires(&dev->lock)
 306 {
 307         struct gr_udc *dev;
 308
 309         list_del_init(&req->queue);
 310
 311         if (likely(req->req.status == -EINPROGRESS))
 312                 req->req.status = status;
 313         else
 314                 status = req->req.status;
 315
 316         dev = ep->dev;
 317         usb_gadget_unmap_request(&dev->gadget, &req->req, ep->is_in);
 318         gr_free_dma_desc_chain(dev, req);
 319
 320         if (ep->is_in) /* For OUT, actual gets updated bit by bit */
 321                 req->req.actual = req->req.length;
 322
 323         if (!status) {
 324                 if (ep->is_in)
 325                         gr_dbgprint_request("SENT", ep, req);
 326                 else
 327                         gr_dbgprint_request("RECV", ep, req);
 328         }
 329
 330         /* Prevent changes to ep->queue during callback */
 331         ep->callback = 1;
 332         if (req == dev->ep0reqo && !status) {
 333                 if (req->setup)
 334                         gr_ep0_setup(dev, req);
 335                 else
 336                         dev_err(dev->dev,
 337                                 "Unexpected non setup packet on ep0in\n");
 338         } else if (req->req.complete) {
 339                 spin_unlock(&dev->lock);
 340
 341                 req->req.complete(&ep->ep, &req->req);
 342
 343                 spin_lock(&dev->lock);
 344         }
 345         ep->callback = 0;
 346 }
 347
 348 static struct usb_request *gr_alloc_request(struct usb_ep *_ep, gfp_t gfp_flags)
 349 {
 350         struct gr_request *req;
 351
 352         req = kzalloc(sizeof(*req), gfp_flags);
 353         if (!req)
 354                 return NULL;
 355
 356         INIT_LIST_HEAD(&req->queue);
 357
 358         return &req->req;
 359 }
 360
 361 /*
 362  * Starts DMA for endpoint ep if there are requests in the queue.
 363  *
 364  * Must be called with dev->lock held and with !ep->stopped.
 365  */
 366 static void gr_start_dma(struct gr_ep *ep)
 367 {
 368         struct gr_request *req;
 369         u32 dmactrl;
 370
 371         if (list_empty(&ep->queue)) {
 372                 ep->dma_start = 0;
 373                 return;
 374         }
 375
 376         req = list_first_entry(&ep->queue, struct gr_request, queue);
 377
 378         /* A descriptor should already have been allocated */
 379         BUG_ON(!req->curr_desc);
 380
 381         wmb(); /* Make sure all is settled before handing it over to DMA */
 382
 383         /* Set the descriptor pointer in the hardware */
 384         gr_write32(&ep->regs->dmaaddr, req->curr_desc->paddr);
 385
 386         /* Announce available descriptors */
 387         dmactrl = gr_read32(&ep->regs->dmactrl);
 388         gr_write32(&ep->regs->dmactrl, dmactrl | GR_DMACTRL_DA);
 389
 390         ep->dma_start = 1;
 391 }
 392
 393 /*
 394  * Finishes the first request in the ep's queue and, if available, starts the
 395  * next request in queue.
 396  *
 397  * Must be called with dev->lock held, irqs disabled and with !ep->stopped.
 398  */
 399 static void gr_dma_advance(struct gr_ep *ep, int status)
 400 {
 401         struct gr_request *req;
 402
 403         req = list_first_entry(&ep->queue, struct gr_request, queue);
 404         gr_finish_request(ep, req, status);
 405         gr_start_dma(ep); /* Regardless of ep->dma_start */
 406 }
 407
 408 /*
 409  * Abort DMA for an endpoint. Sets the abort DMA bit which causes an ongoing DMA
 410  * transfer to be canceled and clears GR_DMACTRL_DA.
 411  *
 412  * Must be called with dev->lock held.
 413  */
 414 static void gr_abort_dma(struct gr_ep *ep)
 415 {
 416         u32 dmactrl;
 417
 418         dmactrl = gr_read32(&ep->regs->dmactrl);
 419         gr_write32(&ep->regs->dmactrl, dmactrl | GR_DMACTRL_AD);
 420 }
 421
 422 /*
 423  * Allocates and sets up a struct gr_dma_desc and putting it on the descriptor
 424  * chain.
 425  *
 426  * Size is not used for OUT endpoints. Hardware can not be instructed to handle
 427  * smaller buffer than MAXPL in the OUT direction.
 428  */
 429 static int gr_add_dma_desc(struct gr_ep *ep, struct gr_request *req,
 430                            dma_addr_t data, unsigned size, gfp_t gfp_flags)
 431 {
 432         struct gr_dma_desc *desc;
 433
 434         desc = gr_alloc_dma_desc(ep, gfp_flags);
 435         if (!desc)
 436                 return -ENOMEM;
 437
 438         desc->data = data;
 439         if (ep->is_in)
 440                 desc->ctrl =
 441                         (GR_DESC_IN_CTRL_LEN_MASK & size) | GR_DESC_IN_CTRL_EN;
 442         else
 443                 desc->ctrl = GR_DESC_OUT_CTRL_IE;
 444
 445         if (!req->first_desc) {
 446                 req->first_desc = desc;
 447                 req->curr_desc = desc;
 448         } else {
 449                 req->last_desc->next_desc = desc;
 450                 req->last_desc->next = desc->paddr;
 451                 req->last_desc->ctrl |= GR_DESC_OUT_CTRL_NX;
 452         }
 453         req->last_desc = desc;
 454
 455         return 0;
 456 }
 457
 458 /*
 459  * Sets up a chain of struct gr_dma_descriptors pointing to buffers that
 460  * together covers req->req.length bytes of the buffer at DMA address
 461  * req->req.dma for the OUT direction.
 462  *
 463  * The first descriptor in the chain is enabled, the rest disabled. The
 464  * interrupt handler will later enable them one by one when needed so we can
 465  * find out when the transfer is finished. For OUT endpoints, all descriptors
 466  * therefore generate interrutps.
 467  */
 468 static int gr_setup_out_desc_list(struct gr_ep *ep, struct gr_request *req,
 469                                   gfp_t gfp_flags)
 470 {
 471         u16 bytes_left; /* Bytes left to provide descriptors for */
 472         u16 bytes_used; /* Bytes accommodated for */
 473         int ret = 0;
 474
 475         req->first_desc = NULL; /* Signals that no allocation is done yet */
 476         bytes_left = req->req.length;
 477         bytes_used = 0;
 478         while (bytes_left > 0) {
 479                 dma_addr_t start = req->req.dma + bytes_used;
 480                 u16 size = min(bytes_left, ep->bytes_per_buffer);
 481
 482                 /* Should not happen however - gr_queue stops such lengths */
 483                 if (size < ep->bytes_per_buffer)
 484                         dev_warn(ep->dev->dev,
 485                                  "Buffer overrun risk: %u < %u bytes/buffer\n",
 486                                  size, ep->bytes_per_buffer);
 487
 488                 ret = gr_add_dma_desc(ep, req, start, size, gfp_flags);
 489                 if (ret)
 490                         goto alloc_err;
 491
 492                 bytes_left -= size;
 493                 bytes_used += size;
 494         }
 495
 496         req->first_desc->ctrl |= GR_DESC_OUT_CTRL_EN;
 497
 498         return 0;
 499
 500 alloc_err:
 501         gr_free_dma_desc_chain(ep->dev, req);
 502
 503         return ret;
 504 }
 505
 506 /*
 507  * Sets up a chain of struct gr_dma_descriptors pointing to buffers that
 508  * together covers req->req.length bytes of the buffer at DMA address
 509  * req->req.dma for the IN direction.
 510  *
 511  * When more data is provided than the maximum payload size, the hardware splits
 512  * this up into several payloads automatically. Moreover, ep->bytes_per_buffer
 513  * is always set to a multiple of the maximum payload (restricted to the valid
 514  * number of maximum payloads during high bandwidth isochronous or interrupt
 515  * transfers)
 516  *
 517  * All descriptors are enabled from the beginning and we only generate an
 518  * interrupt for the last one indicating that the entire request has been pushed
 519  * to hardware.
 520  */
 521 static int gr_setup_in_desc_list(struct gr_ep *ep, struct gr_request *req,
 522                                  gfp_t gfp_flags)
 523 {
 524         u16 bytes_left; /* Bytes left in req to provide descriptors for */
 525         u16 bytes_used; /* Bytes in req accommodated for */
 526         int ret = 0;
 527
 528         req->first_desc = NULL; /* Signals that no allocation is done yet */
 529         bytes_left = req->req.length;
 530         bytes_used = 0;
 531         do { /* Allow for zero length packets */
 532                 dma_addr_t start = req->req.dma + bytes_used;
 533                 u16 size = min(bytes_left, ep->bytes_per_buffer);
 534
 535                 ret = gr_add_dma_desc(ep, req, start, size, gfp_flags);
 536                 if (ret)
 537                         goto alloc_err;
 538
 539                 bytes_left -= size;
 540                 bytes_used += size;
 541         } while (bytes_left > 0);
 542
 543         /*
 544          * Send an extra zero length packet to indicate that no more data is
 545          * available when req->req.zero is set and the data length is even
 546          * multiples of ep->ep.maxpacket.
 547          */
 548         if (req->req.zero && (req->req.length % ep->ep.maxpacket == 0)) {
 549                 ret = gr_add_dma_desc(ep, req, 0, 0, gfp_flags);
 550                 if (ret)
 551                         goto alloc_err;
 552         }
 553
 554         /*
 555          * For IN packets we only want to know when the last packet has been
 556          * transmitted (not just put into internal buffers).
 557          */
 558         req->last_desc->ctrl |= GR_DESC_IN_CTRL_PI;
 559
 560         return 0;
 561
 562 alloc_err:
 563         gr_free_dma_desc_chain(ep->dev, req);
 564
 565         return ret;
 566 }
 567
 568 /* Must be called with dev->lock held */
 569 static int gr_queue(struct gr_ep *ep, struct gr_request *req, gfp_t gfp_flags)
 570 {
 571         struct gr_udc *dev = ep->dev;
 572         int ret;
 573
 574         if (unlikely(!ep->ep.desc && ep->num != 0)) {
 575                 dev_err(dev->dev, "No ep descriptor for %s\n", ep->ep.name);
 576                 return -EINVAL;
 577         }
 578
 579         if (unlikely(!req->req.buf || !list_empty(&req->queue))) {
 580                 dev_err(dev->dev,
 581                         "Invalid request for %s: buf=%p list_empty=%d\n",
 582                         ep->ep.name, req->req.buf, list_empty(&req->queue));
 583                 return -EINVAL;
 584         }
 585
 586         /*
 587          * The DMA controller can not handle smaller OUT buffers than
 588          * maxpacket. It could lead to buffer overruns if unexpectedly long
 589          * packet are received.
 590          */
 591         if (!ep->is_in && (req->req.length % ep->ep.maxpacket) != 0) {
 592                 dev_err(dev->dev,
 593                         "OUT request length %d is not multiple of maxpacket\n",
 594                         req->req.length);
 595                 return -EMSGSIZE;
 596         }
 597
 598         if (unlikely(!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN)) {
 599                 dev_err(dev->dev, "-ESHUTDOWN");
 600                 return -ESHUTDOWN;
 601         }
 602
 603         /* Can't touch registers when suspended */
 604         if (dev->ep0state == GR_EP0_SUSPEND) {
 605                 dev_err(dev->dev, "-EBUSY");
 606                 return -EBUSY;
 607         }
 608
 609         /* Set up DMA mapping in case the caller didn't */
 610         ret = usb_gadget_map_request(&dev->gadget, &req->req, ep->is_in);
 611         if (ret) {
 612                 dev_err(dev->dev, "usb_gadget_map_request");
 613                 return ret;
 614         }
 615
 616         if (ep->is_in)
 617                 ret = gr_setup_in_desc_list(ep, req, gfp_flags);
 618         else
 619                 ret = gr_setup_out_desc_list(ep, req, gfp_flags);
 620         if (ret)
 621                 return ret;
 622
 623         req->req.status = -EINPROGRESS;
 624         req->req.actual = 0;
 625         list_add_tail(&req->queue, &ep->queue);
 626
 627         /* Start DMA if not started, otherwise interrupt handler handles it */
 628         if (!ep->dma_start && likely(!ep->stopped))
 629                 gr_start_dma(ep);
 630
 631         return 0;
 632 }
 633
 634 /*
 635  * Queue a request from within the driver.
 636  *
 637  * Must be called with dev->lock held.
 638  */
 639 static inline int gr_queue_int(struct gr_ep *ep, struct gr_request *req,
 640                                gfp_t gfp_flags)
 641 {
 642         if (ep->is_in)
 643                 gr_dbgprint_request("RESP", ep, req);
 644
 645         return gr_queue(ep, req, gfp_flags);
 646 }
 647
 648 /* ---------------------------------------------------------------------- */
 649 /* General helper functions */
 650
 651 /*
 652  * Dequeue ALL requests.
 653  *
 654  * Must be called with dev->lock held and irqs disabled.
 655  */
 656 static void gr_ep_nuke(struct gr_ep *ep)
 657 {
 658         struct gr_request *req;
 659
 660         ep->stopped = 1;
 661         ep->dma_start = 0;
 662         gr_abort_dma(ep);
 663
 664         while (!list_empty(&ep->queue)) {
 665                 req = list_first_entry(&ep->queue, struct gr_request, queue);
 666                 gr_finish_request(ep, req, -ESHUTDOWN);
 667         }
 668 }
 669
 670 /*
 671  * Reset the hardware state of this endpoint.
 672  *
 673  * Must be called with dev->lock held.
 674  */
 675 static void gr_ep_reset(struct gr_ep *ep)
 676 {
 677         gr_write32(&ep->regs->epctrl, 0);
 678         gr_write32(&ep->regs->dmactrl, 0);
 679
 680         ep->ep.maxpacket = MAX_CTRL_PL_SIZE;
 681         ep->ep.desc = NULL;
 682         ep->stopped = 1;
 683         ep->dma_start = 0;
 684 }
 685
 686 /*
 687  * Generate STALL on ep0in/out.
 688  *
 689  * Must be called with dev->lock held.
 690  */
 691 static void gr_control_stall(struct gr_udc *dev)
 692 {
 693         u32 epctrl;
 694
 695         epctrl = gr_read32(&dev->epo[0].regs->epctrl);
 696         gr_write32(&dev->epo[0].regs->epctrl, epctrl | GR_EPCTRL_CS);
 697         epctrl = gr_read32(&dev->epi[0].regs->epctrl);
 698         gr_write32(&dev->epi[0].regs->epctrl, epctrl | GR_EPCTRL_CS);
 699
 700         dev->ep0state = GR_EP0_STALL;
 701 }
 702
 703 /*
 704  * Halts, halts and wedges, or clears halt for an endpoint.
 705  *
 706  * Must be called with dev->lock held.
 707  */
 708 static int gr_ep_halt_wedge(struct gr_ep *ep, int halt, int wedge, int fromhost)
 709 {
 710         u32 epctrl;
 711         int retval = 0;
 712
 713         if (ep->num && !ep->ep.desc)
 714                 return -EINVAL;
 715
 716         if (ep->num && ep->ep.desc->bmAttributes == USB_ENDPOINT_XFER_ISOC)
 717                 return -EOPNOTSUPP;
 718
 719         /* Never actually halt ep0, and therefore never clear halt for ep0 */
 720         if (!ep->num) {
 721                 if (halt && !fromhost) {
 722                         /* ep0 halt from gadget - generate protocol stall */
 723                         gr_control_stall(ep->dev);
 724                         dev_dbg(ep->dev->dev, "EP: stall ep0\n");
 725                         return 0;
 726                 }
 727                 return -EINVAL;
 728         }
 729
 730         dev_dbg(ep->dev->dev, "EP: %s halt %s\n",
 731                 (halt ? (wedge ? "wedge" : "set") : "clear"), ep->ep.name);
 732
 733         epctrl = gr_read32(&ep->regs->epctrl);
 734         if (halt) {
 735                 /* Set HALT */
 736                 gr_write32(&ep->regs->epctrl, epctrl | GR_EPCTRL_EH);
 737                 ep->stopped = 1;
 738                 if (wedge)
 739                         ep->wedged = 1;
 740         } else {
 741                 gr_write32(&ep->regs->epctrl, epctrl & ~GR_EPCTRL_EH);
 742                 ep->stopped = 0;
 743                 ep->wedged = 0;
 744
 745                 /* Things might have been queued up in the meantime */
 746                 if (!ep->dma_start)
 747                         gr_start_dma(ep);
 748         }
 749
 750         return retval;
 751 }
 752
 753 /* Must be called with dev->lock held */
 754 static inline void gr_set_ep0state(struct gr_udc *dev, enum gr_ep0state value)
 755 {
 756         if (dev->ep0state != value)
 757                 dev_vdbg(dev->dev, "STATE:  ep0state=%s\n",
 758                          gr_ep0state_string(value));
 759         dev->ep0state = value;
 760 }
 761
 762 /*
 763  * Should only be called when endpoints can not generate interrupts.
 764  *
 765  * Must be called with dev->lock held.
 766  */
 767 static void gr_disable_interrupts_and_pullup(struct gr_udc *dev)
 768 {
 769         gr_write32(&dev->regs->control, 0);
 770         wmb(); /* Make sure that we do not deny one of our interrupts */
 771         dev->irq_enabled = 0;
 772 }
 773
 774 /*
 775  * Stop all device activity and disable data line pullup.
 776  *
 777  * Must be called with dev->lock held and irqs disabled.
 778  */
 779 static void gr_stop_activity(struct gr_udc *dev)
 780 {
 781         struct gr_ep *ep;
 782
 783         list_for_each_entry(ep, &dev->ep_list, ep_list)
 784                 gr_ep_nuke(ep);
 785
 786         gr_disable_interrupts_and_pullup(dev);
 787
 788         gr_set_ep0state(dev, GR_EP0_DISCONNECT);
 789         usb_gadget_set_state(&dev->gadget, USB_STATE_NOTATTACHED);
 790 }
 791
 792 /* ---------------------------------------------------------------------- */
 793 /* ep0 setup packet handling */
 794
 795 static void gr_ep0_testmode_complete(struct usb_ep *_ep,
 796                                      struct usb_request *_req)
 797 {
 798         struct gr_ep *ep;
 799         struct gr_udc *dev;
 800         u32 control;
 801
 802         ep = container_of(_ep, struct gr_ep, ep);
 803         dev = ep->dev;
 804
 805         spin_lock(&dev->lock);
 806
 807         control = gr_read32(&dev->regs->control);
 808         control |= GR_CONTROL_TM | (dev->test_mode << GR_CONTROL_TS_POS);
 809         gr_write32(&dev->regs->control, control);
 810
 811         spin_unlock(&dev->lock);
 812 }
 813
 814 static void gr_ep0_dummy_complete(struct usb_ep *_ep, struct usb_request *_req)
 815 {
 816         /* Nothing needs to be done here */
 817 }
 818
 819 /*
 820  * Queue a response on ep0in.
 821  *
 822  * Must be called with dev->lock held.
 823  */
 824 static int gr_ep0_respond(struct gr_udc *dev, u8 *buf, int length,
 825                           void (*complete)(struct usb_ep *ep,
 826                                            struct usb_request *req))
 827 {
 828         u8 *reqbuf = dev->ep0reqi->req.buf;
 829         int status;
 830         int i;
 831
 832         for (i = 0; i < length; i++)
 833                 reqbuf[i] = buf[i];
 834         dev->ep0reqi->req.length = length;
 835         dev->ep0reqi->req.complete = complete;
 836
 837         status = gr_queue_int(&dev->epi[0], dev->ep0reqi, GFP_ATOMIC);
 838         if (status < 0)
 839                 dev_err(dev->dev,
 840                         "Could not queue ep0in setup response: %d\n", status);
 841
 842         return status;
 843 }
 844
 845 /*
 846  * Queue a 2 byte response on ep0in.
 847  *
 848  * Must be called with dev->lock held.
 849  */
 850 static inline int gr_ep0_respond_u16(struct gr_udc *dev, u16 response)
 851 {
 852         __le16 le_response = cpu_to_le16(response);
 853
 854         return gr_ep0_respond(dev, (u8 *)&le_response, 2,
 855                               gr_ep0_dummy_complete);
 856 }
 857
 858 /*
 859  * Queue a ZLP response on ep0in.
 860  *
 861  * Must be called with dev->lock held.
 862  */
 863 static inline int gr_ep0_respond_empty(struct gr_udc *dev)
 864 {
 865         return gr_ep0_respond(dev, NULL, 0, gr_ep0_dummy_complete);
 866 }
 867
 868 /*
 869  * This is run when a SET_ADDRESS request is received. First writes
 870  * the new address to the control register which is updated internally
 871  * when the next IN packet is ACKED.
 872  *
 873  * Must be called with dev->lock held.
 874  */
 875 static void gr_set_address(struct gr_udc *dev, u8 address)
 876 {
 877         u32 control;
 878
 879         control = gr_read32(&dev->regs->control) & ~GR_CONTROL_UA_MASK;
 880         control |= (address << GR_CONTROL_UA_POS) & GR_CONTROL_UA_MASK;
 881         control |= GR_CONTROL_SU;
 882         gr_write32(&dev->regs->control, control);
 883 }
 884
 885 /*
 886  * Returns negative for STALL, 0 for successful handling and positive for
 887  * delegation.
 888  *
 889  * Must be called with dev->lock held.
 890  */
 891 static int gr_device_request(struct gr_udc *dev, u8 type, u8 request,
 892                              u16 value, u16 index)
 893 {
 894         u16 response;
 895         u8 test;
 896
 897         switch (request) {
 898         case USB_REQ_SET_ADDRESS:
 899                 dev_dbg(dev->dev, "STATUS: address %d\n", value & 0xff);
 900                 gr_set_address(dev, value & 0xff);
 901                 if (value)
 902                         usb_gadget_set_state(&dev->gadget, USB_STATE_ADDRESS);
 903                 else
 904                         usb_gadget_set_state(&dev->gadget, USB_STATE_DEFAULT);
 905                 return gr_ep0_respond_empty(dev);
 906
 907         case USB_REQ_GET_STATUS:
 908                 /* Self powered | remote wakeup */
 909                 response = 0x0001 | (dev->remote_wakeup ? 0x0002 : 0);
 910                 return gr_ep0_respond_u16(dev, response);
 911
 912         case USB_REQ_SET_FEATURE:
 913                 switch (value) {
 914                 case USB_DEVICE_REMOTE_WAKEUP:
 915                         /* Allow remote wakeup */
 916                         dev->remote_wakeup = 1;
 917                         return gr_ep0_respond_empty(dev);
 918
 919                 case USB_DEVICE_TEST_MODE:
 920                         /* The hardware does not support TEST_FORCE_EN */
 921                         test = index >> 8;
 922                         if (test >= TEST_J && test <= TEST_PACKET) {
 923                                 dev->test_mode = test;
 924                                 return gr_ep0_respond(dev, NULL, 0,
 925                                                       gr_ep0_testmode_complete);
 926                         }
 927                 }
 928                 break;
 929
 930         case USB_REQ_CLEAR_FEATURE:
 931                 switch (value) {
 932                 case USB_DEVICE_REMOTE_WAKEUP:
 933                         /* Disallow remote wakeup */
 934                         dev->remote_wakeup = 0;
 935                         return gr_ep0_respond_empty(dev);
 936                 }
 937                 break;
 938         }
 939
 940         return 1; /* Delegate the rest */
 941 }
 942
 943 /*
 944  * Returns negative for STALL, 0 for successful handling and positive for
 945  * delegation.
 946  *
 947  * Must be called with dev->lock held.
 948  */
 949 static int gr_interface_request(struct gr_udc *dev, u8 type, u8 request,
 950                                 u16 value, u16 index)
 951 {
 952         if (dev->gadget.state != USB_STATE_CONFIGURED)
 953                 return -1;
 954
 955         /*
 956          * Should return STALL for invalid interfaces, but udc driver does not
 957          * know anything about that. However, many gadget drivers do not handle
 958          * GET_STATUS so we need to take care of that.
 959          */
 960
 961         switch (request) {
 962         case USB_REQ_GET_STATUS:
 963                 return gr_ep0_respond_u16(dev, 0x0000);
 964
 965         case USB_REQ_SET_FEATURE:
 966         case USB_REQ_CLEAR_FEATURE:
 967                 /*
 968                  * No possible valid standard requests. Still let gadget drivers
 969                  * have a go at it.
 970                  */
 971                 break;
 972         }
 973
 974         return 1; /* Delegate the rest */
 975 }
 976
 977 /*
 978  * Returns negative for STALL, 0 for successful handling and positive for
 979  * delegation.
 980  *
 981  * Must be called with dev->lock held.
 982  */
 983 static int gr_endpoint_request(struct gr_udc *dev, u8 type, u8 request,
 984                                u16 value, u16 index)
 985 {
 986         struct gr_ep *ep;
 987         int status;
 988         int halted;
 989         u8 epnum = index & USB_ENDPOINT_NUMBER_MASK;
 990         u8 is_in = index & USB_ENDPOINT_DIR_MASK;
 991
 992         if ((is_in && epnum >= dev->nepi) || (!is_in && epnum >= dev->nepo))
 993                 return -1;
 994
 995         if (dev->gadget.state != USB_STATE_CONFIGURED && epnum != 0)
 996                 return -1;
 997
 998         ep = (is_in ? &dev->epi[epnum] : &dev->epo[epnum]);
 999
1000         switch (request) {
1001         case USB_REQ_GET_STATUS:
1002                 halted = gr_read32(&ep->regs->epctrl) & GR_EPCTRL_EH;
1003                 return gr_ep0_respond_u16(dev, halted ? 0x0001 : 0);
1004
1005         case USB_REQ_SET_FEATURE:
1006                 switch (value) {
1007                 case USB_ENDPOINT_HALT:
1008                         status = gr_ep_halt_wedge(ep, 1, 0, 1);
1009                         if (status >= 0)
1010                                 status = gr_ep0_respond_empty(dev);
1011                         return status;
1012                 }
1013                 break;
1014
1015         case USB_REQ_CLEAR_FEATURE:
1016                 switch (value) {
1017                 case USB_ENDPOINT_HALT:
1018                         if (ep->wedged)
1019                                 return -1;
1020                         status = gr_ep_halt_wedge(ep, 0, 0, 1);
1021                         if (status >= 0)
1022                                 status = gr_ep0_respond_empty(dev);
1023                         return status;
1024                 }
1025                 break;
1026         }
1027
1028         return 1; /* Delegate the rest */
1029 }
1030
1031 /* Must be called with dev->lock held */
1032 static void gr_ep0out_requeue(struct gr_udc *dev)
1033 {
1034         int ret = gr_queue_int(&dev->epo[0], dev->ep0reqo, GFP_ATOMIC);
1035
1036         if (ret)
1037                 dev_err(dev->dev, "Could not queue ep0out setup request: %d\n",
1038                         ret);
1039 }
1040
1041 /*
1042  * The main function dealing with setup requests on ep0.
1043  *
1044  * Must be called with dev->lock held and irqs disabled
1045  */
1046 static void gr_ep0_setup(struct gr_udc *dev, struct gr_request *req)
1047         __releases(&dev->lock)
1048         __acquires(&dev->lock)
1049 {
1050         union {
1051                 struct usb_ctrlrequest ctrl;
1052                 u8 raw[8];
1053                 u32 word[2];
1054         } u;
1055         u8 type;
1056         u8 request;
1057         u16 value;
1058         u16 index;
1059         u16 length;
1060         int i;
1061         int status;
1062
1063         /* Restore from ep0 halt */
1064         if (dev->ep0state == GR_EP0_STALL) {
1065                 gr_set_ep0state(dev, GR_EP0_SETUP);
1066                 if (!req->req.actual)
1067                         goto out;
1068         }
1069
1070         if (dev->ep0state == GR_EP0_ISTATUS) {
1071                 gr_set_ep0state(dev, GR_EP0_SETUP);
1072                 if (req->req.actual > 0)
1073                         dev_dbg(dev->dev,
1074                                 "Unexpected setup packet at state %s\n",
1075                                 gr_ep0state_string(GR_EP0_ISTATUS));
1076                 else
1077                         goto out; /* Got expected ZLP */
1078         } else if (dev->ep0state != GR_EP0_SETUP) {
1079                 dev_info(dev->dev,
1080                          "Unexpected ep0out request at state %s - stalling\n",
1081                          gr_ep0state_string(dev->ep0state));
1082                 gr_control_stall(dev);
1083                 gr_set_ep0state(dev, GR_EP0_SETUP);
1084                 goto out;
1085         } else if (!req->req.actual) {
1086                 dev_dbg(dev->dev, "Unexpected ZLP at state %s\n",
1087                         gr_ep0state_string(dev->ep0state));
1088                 goto out;
1089         }
1090
1091         /* Handle SETUP packet */
1092         for (i = 0; i < req->req.actual; i++)
1093                 u.raw[i] = ((u8 *)req->req.buf)[i];
1094
1095         type = u.ctrl.bRequestType;
1096         request = u.ctrl.bRequest;
1097         value = le16_to_cpu(u.ctrl.wValue);
1098         index = le16_to_cpu(u.ctrl.wIndex);
1099         length = le16_to_cpu(u.ctrl.wLength);
1100
1101         gr_dbgprint_devreq(dev, type, request, value, index, length);
1102
1103         /* Check for data stage */
1104         if (length) {
1105                 if (type & USB_DIR_IN)
1106                         gr_set_ep0state(dev, GR_EP0_IDATA);
1107                 else
1108                         gr_set_ep0state(dev, GR_EP0_ODATA);
1109         }
1110
1111         status = 1; /* Positive status flags delegation */
1112         if ((type & USB_TYPE_MASK) == USB_TYPE_STANDARD) {
1113                 switch (type & USB_RECIP_MASK) {
1114                 case USB_RECIP_DEVICE:
1115                         status = gr_device_request(dev, type, request,
1116                                                    value, index);
1117                         break;
1118                 case USB_RECIP_ENDPOINT:
1119                         status =  gr_endpoint_request(dev, type, request,
1120                                                       value, index);
1121                         break;
1122                 case USB_RECIP_INTERFACE:
1123                         status = gr_interface_request(dev, type, request,
1124                                                       value, index);
1125                         break;
1126                 }
1127         }
1128
1129         if (status > 0) {
1130                 spin_unlock(&dev->lock);
1131
1132                 dev_vdbg(dev->dev, "DELEGATE\n");
1133                 status = dev->driver->setup(&dev->gadget, &u.ctrl);
1134
1135                 spin_lock(&dev->lock);
1136         }
1137
1138         /* Generate STALL on both ep0out and ep0in if requested */
1139         if (unlikely(status < 0)) {
1140                 dev_vdbg(dev->dev, "STALL\n");
1141                 gr_control_stall(dev);
1142         }
1143
1144         if ((type & USB_TYPE_MASK) == USB_TYPE_STANDARD &&
1145             request == USB_REQ_SET_CONFIGURATION) {
1146                 if (!value) {
1147                         dev_dbg(dev->dev, "STATUS: deconfigured\n");
1148                         usb_gadget_set_state(&dev->gadget, USB_STATE_ADDRESS);
1149                 } else if (status >= 0) {
1150                         /* Not configured unless gadget OK:s it */
1151                         dev_dbg(dev->dev, "STATUS: configured: %d\n", value);
1152                         usb_gadget_set_state(&dev->gadget,
1153                                              USB_STATE_CONFIGURED);
1154                 }
1155         }
1156
1157         /* Get ready for next stage */
1158         if (dev->ep0state == GR_EP0_ODATA)
1159                 gr_set_ep0state(dev, GR_EP0_OSTATUS);
1160         else if (dev->ep0state == GR_EP0_IDATA)
1161                 gr_set_ep0state(dev, GR_EP0_ISTATUS);
1162         else
1163                 gr_set_ep0state(dev, GR_EP0_SETUP);
1164
1165 out:
1166         gr_ep0out_requeue(dev);
1167 }
1168
1169 /* ---------------------------------------------------------------------- */
1170 /* VBUS and USB reset handling */
1171
1172 /* Must be called with dev->lock held and irqs disabled  */
1173 static void gr_vbus_connected(struct gr_udc *dev, u32 status)
1174 {
1175         u32 control;
1176
1177         dev->gadget.speed = GR_SPEED(status);
1178         usb_gadget_set_state(&dev->gadget, USB_STATE_POWERED);
1179
1180         /* Turn on full interrupts and pullup */
1181         control = (GR_CONTROL_SI | GR_CONTROL_UI | GR_CONTROL_VI |
1182                    GR_CONTROL_SP | GR_CONTROL_EP);
1183         gr_write32(&dev->regs->control, control);
1184 }
1185
1186 /* Must be called with dev->lock held */
1187 static void gr_enable_vbus_detect(struct gr_udc *dev)
1188 {
1189         u32 status;
1190
1191         dev->irq_enabled = 1;
1192         wmb(); /* Make sure we do not ignore an interrupt */
1193         gr_write32(&dev->regs->control, GR_CONTROL_VI);
1194
1195         /* Take care of the case we are already plugged in at this point */
1196         status = gr_read32(&dev->regs->status);
1197         if (status & GR_STATUS_VB)
1198                 gr_vbus_connected(dev, status);
1199 }
1200
1201 /* Must be called with dev->lock held and irqs disabled */
1202 static void gr_vbus_disconnected(struct gr_udc *dev)
1203 {
1204         gr_stop_activity(dev);
1205
1206         /* Report disconnect */
1207         if (dev->driver && dev->driver->disconnect) {
1208                 spin_unlock(&dev->lock);
1209
1210                 dev->driver->disconnect(&dev->gadget);
1211
1212                 spin_lock(&dev->lock);
1213         }
1214
1215         gr_enable_vbus_detect(dev);
1216 }
1217
1218 /* Must be called with dev->lock held and irqs disabled */
1219 static void gr_udc_usbreset(struct gr_udc *dev, u32 status)
1220 {
1221         gr_set_address(dev, 0);
1222         gr_set_ep0state(dev, GR_EP0_SETUP);
1223         usb_gadget_set_state(&dev->gadget, USB_STATE_DEFAULT);
1224         dev->gadget.speed = GR_SPEED(status);
1225
1226         gr_ep_nuke(&dev->epo[0]);
1227         gr_ep_nuke(&dev->epi[0]);
1228         dev->epo[0].stopped = 0;
1229         dev->epi[0].stopped = 0;
1230         gr_ep0out_requeue(dev);
1231 }
1232
1233 /* ---------------------------------------------------------------------- */
1234 /* Irq handling */
1235
1236 /*
1237  * Handles interrupts from in endpoints. Returns whether something was handled.
1238  *
1239  * Must be called with dev->lock held, irqs disabled and with !ep->stopped.
1240  */
1241 static int gr_handle_in_ep(struct gr_ep *ep)
1242 {
1243         struct gr_request *req;
1244
1245         req = list_first_entry(&ep->queue, struct gr_request, queue);
1246         if (!req->last_desc)
1247                 return 0;
1248
1249         if (ACCESS_ONCE(req->last_desc->ctrl) & GR_DESC_IN_CTRL_EN)
1250                 return 0; /* Not put in hardware buffers yet */
1251
1252         if (gr_read32(&ep->regs->epstat) & (GR_EPSTAT_B1 | GR_EPSTAT_B0))
1253                 return 0; /* Not transmitted yet, still in hardware buffers */
1254
1255         /* Write complete */
1256         gr_dma_advance(ep, 0);
1257
1258         return 1;
1259 }
1260
1261 /*
1262  * Handles interrupts from out endpoints. Returns whether something was handled.
1263  *
1264  * Must be called with dev->lock held, irqs disabled and with !ep->stopped.
1265  */
1266 static int gr_handle_out_ep(struct gr_ep *ep)
1267 {
1268         u32 ep_dmactrl;
1269         u32 ctrl;
1270         u16 len;
1271         struct gr_request *req;
1272         struct gr_udc *dev = ep->dev;
1273
1274         req = list_first_entry(&ep->queue, struct gr_request, queue);
1275         if (!req->curr_desc)
1276                 return 0;
1277
1278         ctrl = ACCESS_ONCE(req->curr_desc->ctrl);
1279         if (ctrl & GR_DESC_OUT_CTRL_EN)
1280                 return 0; /* Not received yet */
1281
1282         /* Read complete */
1283         len = ctrl & GR_DESC_OUT_CTRL_LEN_MASK;
1284         req->req.actual += len;
1285         if (ctrl & GR_DESC_OUT_CTRL_SE)
1286                 req->setup = 1;
1287
1288         if (len < ep->ep.maxpacket || req->req.actual == req->req.length) {
1289                 /* Short packet or the expected size - we are done */
1290
1291                 if ((ep == &dev->epo[0]) && (dev->ep0state == GR_EP0_OSTATUS)) {
1292                         /*
1293                          * Send a status stage ZLP to ack the DATA stage in the
1294                          * OUT direction. This needs to be done before
1295                          * gr_dma_advance as that can lead to a call to
1296                          * ep0_setup that can change dev->ep0state.
1297                          */
1298                         gr_ep0_respond_empty(dev);
1299                         gr_set_ep0state(dev, GR_EP0_SETUP);
1300                 }
1301
1302                 gr_dma_advance(ep, 0);
1303         } else {
1304                 /* Not done yet. Enable the next descriptor to receive more. */
1305                 req->curr_desc = req->curr_desc->next_desc;
1306                 req->curr_desc->ctrl |= GR_DESC_OUT_CTRL_EN;
1307
1308                 ep_dmactrl = gr_read32(&ep->regs->dmactrl);
1309                 gr_write32(&ep->regs->dmactrl, ep_dmactrl | GR_DMACTRL_DA);
1310         }
1311
1312         return 1;
1313 }
1314
1315 /*
1316  * Handle state changes. Returns whether something was handled.
1317  *
1318  * Must be called with dev->lock held and irqs disabled.
1319  */
1320 static int gr_handle_state_changes(struct gr_udc *dev)
1321 {
1322         u32 status = gr_read32(&dev->regs->status);
1323         int handled = 0;
1324         int powstate = !(dev->gadget.state == USB_STATE_NOTATTACHED ||
1325                          dev->gadget.state == USB_STATE_ATTACHED);
1326
1327         /* VBUS valid detected */
1328         if (!powstate && (status & GR_STATUS_VB)) {
1329                 dev_dbg(dev->dev, "STATUS: vbus valid detected\n");
1330                 gr_vbus_connected(dev, status);
1331                 handled = 1;
1332         }
1333
1334         /* Disconnect */
1335         if (powstate && !(status & GR_STATUS_VB)) {
1336                 dev_dbg(dev->dev, "STATUS: vbus invalid detected\n");
1337                 gr_vbus_disconnected(dev);
1338                 handled = 1;
1339         }
1340
1341         /* USB reset detected */
1342         if (status & GR_STATUS_UR) {
1343                 dev_dbg(dev->dev, "STATUS: USB reset - speed is %s\n",
1344                         GR_SPEED_STR(status));
1345                 gr_write32(&dev->regs->status, GR_STATUS_UR);
1346                 gr_udc_usbreset(dev, status);
1347                 handled = 1;
1348         }
1349
1350         /* Speed change */
1351         if (dev->gadget.speed != GR_SPEED(status)) {
1352                 dev_dbg(dev->dev, "STATUS: USB Speed change to %s\n",
1353                         GR_SPEED_STR(status));
1354                 dev->gadget.speed = GR_SPEED(status);
1355                 handled = 1;
1356         }
1357
1358         /* Going into suspend */
1359         if ((dev->ep0state != GR_EP0_SUSPEND) && !(status & GR_STATUS_SU)) {
1360                 dev_dbg(dev->dev, "STATUS: USB suspend\n");
1361                 gr_set_ep0state(dev, GR_EP0_SUSPEND);
1362                 dev->suspended_from = dev->gadget.state;
1363                 usb_gadget_set_state(&dev->gadget, USB_STATE_SUSPENDED);
1364
1365                 if ((dev->gadget.speed != USB_SPEED_UNKNOWN) &&
1366                     dev->driver && dev->driver->suspend) {
1367                         spin_unlock(&dev->lock);
1368
1369                         dev->driver->suspend(&dev->gadget);
1370
1371                         spin_lock(&dev->lock);
1372                 }
1373                 handled = 1;
1374         }
1375
1376         /* Coming out of suspend */
1377         if ((dev->ep0state == GR_EP0_SUSPEND) && (status & GR_STATUS_SU)) {
1378                 dev_dbg(dev->dev, "STATUS: USB resume\n");
1379                 if (dev->suspended_from == USB_STATE_POWERED)
1380                         gr_set_ep0state(dev, GR_EP0_DISCONNECT);
1381                 else
1382                         gr_set_ep0state(dev, GR_EP0_SETUP);
1383                 usb_gadget_set_state(&dev->gadget, dev->suspended_from);
1384
1385                 if ((dev->gadget.speed != USB_SPEED_UNKNOWN) &&
1386                     dev->driver && dev->driver->resume) {
1387                         spin_unlock(&dev->lock);
1388
1389                         dev->driver->resume(&dev->gadget);
1390
1391                         spin_lock(&dev->lock);
1392                 }
1393                 handled = 1;
1394         }
1395
1396         return handled;
1397 }
1398
1399 /* Non-interrupt context irq handler */
1400 static irqreturn_t gr_irq_handler(int irq, void *_dev)
1401 {
1402         struct gr_udc *dev = _dev;
1403         struct gr_ep *ep;
1404         int handled = 0;
1405         int i;
1406         unsigned long flags;
1407
1408         spin_lock_irqsave(&dev->lock, flags);
1409
1410         if (!dev->irq_enabled)
1411                 goto out;
1412
1413         /*
1414          * Check IN ep interrupts. We check these before the OUT eps because
1415          * some gadgets reuse the request that might already be currently
1416          * outstanding and needs to be completed (mainly setup requests).
1417          */
1418         for (i = 0; i < dev->nepi; i++) {
1419                 ep = &dev->epi[i];
1420                 if (!ep->stopped && !ep->callback && !list_empty(&ep->queue))
1421                         handled = gr_handle_in_ep(ep) || handled;
1422         }
1423
1424         /* Check OUT ep interrupts */
1425         for (i = 0; i < dev->nepo; i++) {
1426                 ep = &dev->epo[i];
1427                 if (!ep->stopped && !ep->callback && !list_empty(&ep->queue))
1428                         handled = gr_handle_out_ep(ep) || handled;
1429         }
1430
1431         /* Check status interrupts */
1432         handled = gr_handle_state_changes(dev) || handled;
1433
1434         /*
1435          * Check AMBA DMA errors. Only check if we didn't find anything else to
1436          * handle because this shouldn't happen if we did everything right.
1437          */
1438         if (!handled) {
1439                 list_for_each_entry(ep, &dev->ep_list, ep_list) {
1440                         if (gr_read32(&ep->regs->dmactrl) & GR_DMACTRL_AE) {
1441                                 dev_err(dev->dev,
1442                                         "AMBA Error occurred for %s\n",
1443                                         ep->ep.name);
1444                                 handled = 1;
1445                         }
1446                 }
1447         }
1448
1449 out:
1450         spin_unlock_irqrestore(&dev->lock, flags);
1451
1452         return handled ? IRQ_HANDLED : IRQ_NONE;
1453 }
1454
1455 /* Interrupt context irq handler */
1456 static irqreturn_t gr_irq(int irq, void *_dev)
1457 {
1458         struct gr_udc *dev = _dev;
1459
1460         if (!dev->irq_enabled)
1461                 return IRQ_NONE;
1462
1463         return IRQ_WAKE_THREAD;
1464 }
1465
1466 /* ---------------------------------------------------------------------- */
1467 /* USB ep ops */
1468
1469 /* Enable endpoint. Not for ep0in and ep0out that are handled separately. */
1470 static int gr_ep_enable(struct usb_ep *_ep,
1471                         const struct usb_endpoint_descriptor *desc)
1472 {
1473         struct gr_udc *dev;
1474         struct gr_ep *ep;
1475         u8 mode;
1476         u8 nt;
1477         u16 max;
1478         u16 buffer_size = 0;
1479         u32 epctrl;
1480
1481         ep = container_of(_ep, struct gr_ep, ep);
1482         if (!_ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT)
1483                 return -EINVAL;
1484
1485         dev = ep->dev;
1486
1487         /* 'ep0' IN and OUT are reserved */
1488         if (ep == &dev->epo[0] || ep == &dev->epi[0])
1489                 return -EINVAL;
1490
1491         if (!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN)
1492                 return -ESHUTDOWN;
1493
1494         /* Make sure we are clear for enabling */
1495         epctrl = gr_read32(&ep->regs->epctrl);
1496         if (epctrl & GR_EPCTRL_EV)
1497                 return -EBUSY;
1498
1499         /* Check that directions match */
1500         if (!ep->is_in != !usb_endpoint_dir_in(desc))
1501                 return -EINVAL;
1502
1503         /* Check ep num */
1504         if ((!ep->is_in && ep->num >= dev->nepo) ||
1505             (ep->is_in && ep->num >= dev->nepi))
1506                 return -EINVAL;
1507
1508         if (usb_endpoint_xfer_control(desc)) {
1509                 mode = 0;
1510         } else if (usb_endpoint_xfer_isoc(desc)) {
1511                 mode = 1;
1512         } else if (usb_endpoint_xfer_bulk(desc)) {
1513                 mode = 2;
1514         } else if (usb_endpoint_xfer_int(desc)) {
1515                 mode = 3;
1516         } else {
1517                 dev_err(dev->dev, "Unknown transfer type for %s\n",
1518                         ep->ep.name);
1519                 return -EINVAL;
1520         }
1521
1522         /*
1523          * Bits 10-0 set the max payload. 12-11 set the number of
1524          * additional transactions.
1525          */
1526         max = 0x7ff & usb_endpoint_maxp(desc);
1527         nt = 0x3 & (usb_endpoint_maxp(desc) >> 11);
1528         buffer_size = GR_BUFFER_SIZE(epctrl);
1529         if (nt && (mode == 0 || mode == 2)) {
1530                 dev_err(dev->dev,
1531                         "%s mode: multiple trans./microframe not valid\n",
1532                         (mode == 2 ? "Bulk" : "Control"));
1533                 return -EINVAL;
1534         } else if (nt == 0x11) {
1535                 dev_err(dev->dev, "Invalid value for trans./microframe\n");
1536                 return -EINVAL;
1537         } else if ((nt + 1) * max > buffer_size) {
1538                 dev_err(dev->dev, "Hw buffer size %d < max payload %d * %d\n",
1539                         buffer_size, (nt + 1), max);
1540                 return -EINVAL;
1541         } else if (max == 0) {
1542                 dev_err(dev->dev, "Max payload cannot be set to 0\n");
1543                 return -EINVAL;
1544         }
1545
1546         spin_lock(&ep->dev->lock);
1547
1548         if (!ep->stopped) {
1549                 spin_unlock(&ep->dev->lock);
1550                 return -EBUSY;
1551         }
1552
1553         ep->stopped = 0;
1554         ep->wedged = 0;
1555         ep->ep.desc = desc;
1556         ep->ep.maxpacket = max;
1557         ep->dma_start = 0;
1558
1559
1560         if (nt) {
1561                 /*
1562                  * Maximum possible size of all payloads in one microframe
1563                  * regardless of direction when using high-bandwidth mode.
1564                  */
1565                 ep->bytes_per_buffer = (nt + 1) * max;
1566         } else if (ep->is_in) {
1567                 /*
1568                  * The biggest multiple of maximum packet size that fits into
1569                  * the buffer. The hardware will split up into many packets in
1570                  * the IN direction.
1571                  */
1572                 ep->bytes_per_buffer = (buffer_size / max) * max;
1573         } else {
1574                 /*
1575                  * Only single packets will be placed the buffers in the OUT
1576                  * direction.
1577                  */
1578                 ep->bytes_per_buffer = max;
1579         }
1580
1581         epctrl = (max << GR_EPCTRL_MAXPL_POS)
1582                 | (nt << GR_EPCTRL_NT_POS)
1583                 | (mode << GR_EPCTRL_TT_POS)
1584                 | GR_EPCTRL_EV;
1585         if (ep->is_in)
1586                 epctrl |= GR_EPCTRL_PI;
1587         gr_write32(&ep->regs->epctrl, epctrl);
1588
1589         gr_write32(&ep->regs->dmactrl, GR_DMACTRL_IE | GR_DMACTRL_AI);
1590
1591         spin_unlock(&ep->dev->lock);
1592
1593         dev_dbg(ep->dev->dev, "EP: %s enabled - %s with %d bytes/buffer\n",
1594                 ep->ep.name, gr_modestring[mode], ep->bytes_per_buffer);
1595         return 0;
1596 }
1597
1598 /* Disable endpoint. Not for ep0in and ep0out that are handled separately. */
1599 static int gr_ep_disable(struct usb_ep *_ep)
1600 {
1601         struct gr_ep *ep;
1602         struct gr_udc *dev;
1603         unsigned long flags;
1604
1605         ep = container_of(_ep, struct gr_ep, ep);
1606         if (!_ep || !ep->ep.desc)
1607                 return -ENODEV;
1608
1609         dev = ep->dev;
1610
1611         /* 'ep0' IN and OUT are reserved */
1612         if (ep == &dev->epo[0] || ep == &dev->epi[0])
1613                 return -EINVAL;
1614
1615         if (dev->ep0state == GR_EP0_SUSPEND)
1616                 return -EBUSY;
1617
1618         dev_dbg(ep->dev->dev, "EP: disable %s\n", ep->ep.name);
1619
1620         spin_lock_irqsave(&dev->lock, flags);
1621
1622         gr_ep_nuke(ep);
1623         gr_ep_reset(ep);
1624         ep->ep.desc = NULL;
1625
1626         spin_unlock_irqrestore(&dev->lock, flags);
1627
1628         return 0;
1629 }
1630
1631 /*
1632  * Frees a request, but not any DMA buffers associated with it
1633  * (gr_finish_request should already have taken care of that).
1634  */
1635 static void gr_free_request(struct usb_ep *_ep, struct usb_request *_req)
1636 {
1637         struct gr_request *req;
1638
1639         if (!_ep || !_req)
1640                 return;
1641         req = container_of(_req, struct gr_request, req);
1642
1643         /* Leads to memory leak */
1644         WARN(!list_empty(&req->queue),
1645              "request not dequeued properly before freeing\n");
1646
1647         kfree(req);
1648 }
1649
1650 /* Queue a request from the gadget */
1651 static int gr_queue_ext(struct usb_ep *_ep, struct usb_request *_req,
1652                         gfp_t gfp_flags)
1653 {
1654         struct gr_ep *ep;
1655         struct gr_request *req;
1656         struct gr_udc *dev;
1657         int ret;
1658
1659         if (unlikely(!_ep || !_req))
1660                 return -EINVAL;
1661
1662         ep = container_of(_ep, struct gr_ep, ep);
1663         req = container_of(_req, struct gr_request, req);
1664         dev = ep->dev;
1665
1666         spin_lock(&ep->dev->lock);
1667
1668         /*
1669          * The ep0 pointer in the gadget struct is used both for ep0in and
1670          * ep0out. In a data stage in the out direction ep0out needs to be used
1671          * instead of the default ep0in. Completion functions might use
1672          * driver_data, so that needs to be copied as well.
1673          */
1674         if ((ep == &dev->epi[0]) && (dev->ep0state == GR_EP0_ODATA)) {
1675                 ep = &dev->epo[0];
1676                 ep->ep.driver_data = dev->epi[0].ep.driver_data;
1677         }
1678
1679         if (ep->is_in)
1680                 gr_dbgprint_request("EXTERN", ep, req);
1681
1682         ret = gr_queue(ep, req, gfp_flags);
1683
1684         spin_unlock(&ep->dev->lock);
1685
1686         return ret;
1687 }
1688
1689 /* Dequeue JUST ONE request */
1690 static int gr_dequeue(struct usb_ep *_ep, struct usb_request *_req)
1691 {
1692         struct gr_request *req;
1693         struct gr_ep *ep;
1694         struct gr_udc *dev;
1695         int ret = 0;
1696         unsigned long flags;
1697
1698         ep = container_of(_ep, struct gr_ep, ep);
1699         if (!_ep || !_req || (!ep->ep.desc && ep->num != 0))
1700                 return -EINVAL;
1701         dev = ep->dev;
1702         if (!dev->driver)
1703                 return -ESHUTDOWN;
1704
1705         /* We can't touch (DMA) registers when suspended */
1706         if (dev->ep0state == GR_EP0_SUSPEND)
1707                 return -EBUSY;
1708
1709         spin_lock_irqsave(&dev->lock, flags);
1710
1711         /* Make sure it's actually queued on this endpoint */
1712         list_for_each_entry(req, &ep->queue, queue) {
1713                 if (&req->req == _req)
1714                         break;
1715         }
1716         if (&req->req != _req) {
1717                 ret = -EINVAL;
1718                 goto out;
1719         }
1720
1721         if (list_first_entry(&ep->queue, struct gr_request, queue) == req) {
1722                 /* This request is currently being processed */
1723                 gr_abort_dma(ep);
1724                 if (ep->stopped)
1725                         gr_finish_request(ep, req, -ECONNRESET);
1726                 else
1727                         gr_dma_advance(ep, -ECONNRESET);
1728         } else if (!list_empty(&req->queue)) {
1729                 /* Not being processed - gr_finish_request dequeues it */
1730                 gr_finish_request(ep, req, -ECONNRESET);
1731         } else {
1732                 ret = -EOPNOTSUPP;
1733         }
1734
1735 out:
1736         spin_unlock_irqrestore(&dev->lock, flags);
1737
1738         return ret;
1739 }
1740
1741 /* Helper for gr_set_halt and gr_set_wedge */
1742 static int gr_set_halt_wedge(struct usb_ep *_ep, int halt, int wedge)
1743 {
1744         int ret;
1745         struct gr_ep *ep;
1746
1747         if (!_ep)
1748                 return -ENODEV;
1749         ep = container_of(_ep, struct gr_ep, ep);
1750
1751         spin_lock(&ep->dev->lock);
1752
1753         /* Halting an IN endpoint should fail if queue is not empty */
1754         if (halt && ep->is_in && !list_empty(&ep->queue)) {
1755                 ret = -EAGAIN;
1756                 goto out;
1757         }
1758
1759         ret = gr_ep_halt_wedge(ep, halt, wedge, 0);
1760
1761 out:
1762         spin_unlock(&ep->dev->lock);
1763
1764         return ret;
1765 }
1766
1767 /* Halt endpoint */
1768 static int gr_set_halt(struct usb_ep *_ep, int halt)
1769 {
1770         return gr_set_halt_wedge(_ep, halt, 0);
1771 }
1772
1773 /* Halt and wedge endpoint */
1774 static int gr_set_wedge(struct usb_ep *_ep)
1775 {
1776         return gr_set_halt_wedge(_ep, 1, 1);
1777 }
1778
1779 /*
1780  * Return the total number of bytes currently stored in the internal buffers of
1781  * the endpoint.
1782  */
1783 static int gr_fifo_status(struct usb_ep *_ep)
1784 {
1785         struct gr_ep *ep;
1786         u32 epstat;
1787         u32 bytes = 0;
1788
1789         if (!_ep)
1790                 return -ENODEV;
1791         ep = container_of(_ep, struct gr_ep, ep);
1792
1793         epstat = gr_read32(&ep->regs->epstat);
1794
1795         if (epstat & GR_EPSTAT_B0)
1796                 bytes += (epstat & GR_EPSTAT_B0CNT_MASK) >> GR_EPSTAT_B0CNT_POS;
1797         if (epstat & GR_EPSTAT_B1)
1798                 bytes += (epstat & GR_EPSTAT_B1CNT_MASK) >> GR_EPSTAT_B1CNT_POS;
1799
1800         return bytes;
1801 }
1802
1803
1804 /* Empty data from internal buffers of an endpoint. */
1805 static void gr_fifo_flush(struct usb_ep *_ep)
1806 {
1807         struct gr_ep *ep;
1808         u32 epctrl;
1809
1810         if (!_ep)
1811                 return;
1812         ep = container_of(_ep, struct gr_ep, ep);
1813         dev_vdbg(ep->dev->dev, "EP: flush fifo %s\n", ep->ep.name);
1814
1815         spin_lock(&ep->dev->lock);
1816
1817         epctrl = gr_read32(&ep->regs->epctrl);
1818         epctrl |= GR_EPCTRL_CB;
1819         gr_write32(&ep->regs->epctrl, epctrl);
1820
1821         spin_unlock(&ep->dev->lock);
1822 }
1823
1824 static struct usb_ep_ops gr_ep_ops = {
1825         .enable         = gr_ep_enable,
1826         .disable        = gr_ep_disable,
1827
1828         .alloc_request  = gr_alloc_request,
1829         .free_request   = gr_free_request,
1830
1831         .queue          = gr_queue_ext,
1832         .dequeue        = gr_dequeue,
1833
1834         .set_halt       = gr_set_halt,
1835         .set_wedge      = gr_set_wedge,
1836         .fifo_status    = gr_fifo_status,
1837         .fifo_flush     = gr_fifo_flush,
1838 };
1839
1840 /* ---------------------------------------------------------------------- */
1841 /* USB Gadget ops */
1842
1843 static int gr_get_frame(struct usb_gadget *_gadget)
1844 {
1845         struct gr_udc *dev;
1846
1847         if (!_gadget)
1848                 return -ENODEV;
1849         dev = container_of(_gadget, struct gr_udc, gadget);
1850         return gr_read32(&dev->regs->status) & GR_STATUS_FN_MASK;
1851 }
1852
1853 static int gr_wakeup(struct usb_gadget *_gadget)
1854 {
1855         struct gr_udc *dev;
1856
1857         if (!_gadget)
1858                 return -ENODEV;
1859         dev = container_of(_gadget, struct gr_udc, gadget);
1860
1861         /* Remote wakeup feature not enabled by host*/
1862         if (!dev->remote_wakeup)
1863                 return -EINVAL;
1864
1865         spin_lock(&dev->lock);
1866
1867         gr_write32(&dev->regs->control,
1868                    gr_read32(&dev->regs->control) | GR_CONTROL_RW);
1869
1870         spin_unlock(&dev->lock);
1871
1872         return 0;
1873 }
1874
1875 static int gr_pullup(struct usb_gadget *_gadget, int is_on)
1876 {
1877         struct gr_udc *dev;
1878         u32 control;
1879
1880         if (!_gadget)
1881                 return -ENODEV;
1882         dev = container_of(_gadget, struct gr_udc, gadget);
1883
1884         spin_lock(&dev->lock);
1885
1886         control = gr_read32(&dev->regs->control);
1887         if (is_on)
1888                 control |= GR_CONTROL_EP;
1889         else
1890                 control &= ~GR_CONTROL_EP;
1891         gr_write32(&dev->regs->control, control);
1892
1893         spin_unlock(&dev->lock);
1894
1895         return 0;
1896 }
1897
1898 static int gr_udc_start(struct usb_gadget *gadget,
1899                         struct usb_gadget_driver *driver)
1900 {
1901         struct gr_udc *dev = to_gr_udc(gadget);
1902
1903         spin_lock(&dev->lock);
1904
1905         /* Hook up the driver */
1906         driver->driver.bus = NULL;
1907         dev->driver = driver;
1908
1909         /* Get ready for host detection */
1910         gr_enable_vbus_detect(dev);
1911
1912         spin_unlock(&dev->lock);
1913
1914         dev_info(dev->dev, "Started with gadget driver '%s'\n",
1915                  driver->driver.name);
1916
1917         return 0;
1918 }
1919
1920 static int gr_udc_stop(struct usb_gadget *gadget,
1921                        struct usb_gadget_driver *driver)
1922 {
1923         struct gr_udc *dev = to_gr_udc(gadget);
1924         unsigned long flags;
1925
1926         spin_lock_irqsave(&dev->lock, flags);
1927
1928         dev->driver = NULL;
1929         gr_stop_activity(dev);
1930
1931         spin_unlock_irqrestore(&dev->lock, flags);
1932
1933         dev_info(dev->dev, "Stopped\n");
1934
1935         return 0;
1936 }
1937
1938 static const struct usb_gadget_ops gr_ops = {
1939         .get_frame      = gr_get_frame,
1940         .wakeup         = gr_wakeup,
1941         .pullup         = gr_pullup,
1942         .udc_start      = gr_udc_start,
1943         .udc_stop       = gr_udc_stop,
1944         /* Other operations not supported */
1945 };
1946
1947 /* ---------------------------------------------------------------------- */
1948 /* Module probe, removal and of-matching */
1949
1950 static const char * const onames[] = {
1951         "ep0out", "ep1out", "ep2out", "ep3out", "ep4out", "ep5out",
1952         "ep6out", "ep7out", "ep8out", "ep9out", "ep10out", "ep11out",
1953         "ep12out", "ep13out", "ep14out", "ep15out"
1954 };
1955
1956 static const char * const inames[] = {
1957         "ep0in", "ep1in", "ep2in", "ep3in", "ep4in", "ep5in",
1958         "ep6in", "ep7in", "ep8in", "ep9in", "ep10in", "ep11in",
1959         "ep12in", "ep13in", "ep14in", "ep15in"
1960 };
1961
1962 /* Must be called with dev->lock held */
1963 static int gr_ep_init(struct gr_udc *dev, int num, int is_in, u32 maxplimit)
1964 {
1965         struct gr_ep *ep;
1966         struct gr_request *req;
1967         struct usb_request *_req;
1968         void *buf;
1969
1970         if (is_in) {
1971                 ep = &dev->epi[num];
1972                 ep->ep.name = inames[num];
1973                 ep->regs = &dev->regs->epi[num];
1974         } else {
1975                 ep = &dev->epo[num];
1976                 ep->ep.name = onames[num];
1977                 ep->regs = &dev->regs->epo[num];
1978         }
1979
1980         gr_ep_reset(ep);
1981         ep->num = num;
1982         ep->is_in = is_in;
1983         ep->dev = dev;
1984         ep->ep.ops = &gr_ep_ops;
1985         INIT_LIST_HEAD(&ep->queue);
1986
1987         if (num == 0) {
1988                 _req = gr_alloc_request(&ep->ep, GFP_KERNEL);
1989                 buf = devm_kzalloc(dev->dev, PAGE_SIZE, GFP_DMA | GFP_KERNEL);
1990                 if (!_req || !buf) {
1991                         /* possible _req freed by gr_probe via gr_remove */
1992                         return -ENOMEM;
1993                 }
1994
1995                 req = container_of(_req, struct gr_request, req);
1996                 req->req.buf = buf;
1997                 req->req.length = MAX_CTRL_PL_SIZE;
1998
1999                 if (is_in)
2000                         dev->ep0reqi = req; /* Complete gets set as used */
2001                 else
2002                         dev->ep0reqo = req; /* Completion treated separately */
2003
2004                 usb_ep_set_maxpacket_limit(&ep->ep, MAX_CTRL_PL_SIZE);
2005                 ep->bytes_per_buffer = MAX_CTRL_PL_SIZE;
2006         } else {
2007                 usb_ep_set_maxpacket_limit(&ep->ep, (u16)maxplimit);
2008                 list_add_tail(&ep->ep.ep_list, &dev->gadget.ep_list);
2009         }
2010         list_add_tail(&ep->ep_list, &dev->ep_list);
2011
2012         return 0;
2013 }
2014
2015 /* Must be called with dev->lock held */
2016 static int gr_udc_init(struct gr_udc *dev)
2017 {
2018         struct device_node *np = dev->dev->of_node;
2019         u32 epctrl_val;
2020         u32 dmactrl_val;
2021         int i;
2022         int ret = 0;
2023         u32 *bufsizes;
2024         u32 bufsize;
2025         int len;
2026
2027         gr_set_address(dev, 0);
2028
2029         INIT_LIST_HEAD(&dev->gadget.ep_list);
2030         dev->gadget.speed = USB_SPEED_UNKNOWN;
2031         dev->gadget.ep0 = &dev->epi[0].ep;
2032
2033         INIT_LIST_HEAD(&dev->ep_list);
2034         gr_set_ep0state(dev, GR_EP0_DISCONNECT);
2035
2036         bufsizes = (u32 *)of_get_property(np, "epobufsizes", &len);
2037         len /= sizeof(u32);
2038         for (i = 0; i < dev->nepo; i++) {
2039                 bufsize = (bufsizes && i < len) ? bufsizes[i] : 1024;
2040                 ret = gr_ep_init(dev, i, 0, bufsize);
2041                 if (ret)
2042                         return ret;
2043         }
2044
2045         bufsizes = (u32 *)of_get_property(np, "epibufsizes", &len);
2046         len /= sizeof(u32);
2047         for (i = 0; i < dev->nepi; i++) {
2048                 bufsize = (bufsizes && i < len) ? bufsizes[i] : 1024;
2049                 ret = gr_ep_init(dev, i, 1, bufsize);
2050                 if (ret)
2051                         return ret;
2052         }
2053
2054         /* Must be disabled by default */
2055         dev->remote_wakeup = 0;
2056
2057         /* Enable ep0out and ep0in */
2058         epctrl_val = (MAX_CTRL_PL_SIZE << GR_EPCTRL_MAXPL_POS) | GR_EPCTRL_EV;
2059         dmactrl_val = GR_DMACTRL_IE | GR_DMACTRL_AI;
2060         gr_write32(&dev->epo[0].regs->epctrl, epctrl_val);
2061         gr_write32(&dev->epi[0].regs->epctrl, epctrl_val | GR_EPCTRL_PI);
2062         gr_write32(&dev->epo[0].regs->dmactrl, dmactrl_val);
2063         gr_write32(&dev->epi[0].regs->dmactrl, dmactrl_val);
2064
2065         return 0;
2066 }
2067
2068 static int gr_remove(struct platform_device *ofdev)
2069 {
2070         struct gr_udc *dev = dev_get_drvdata(&ofdev->dev);
2071
2072         if (dev->added)
2073                 usb_del_gadget_udc(&dev->gadget); /* Shuts everything down */
2074         if (dev->driver)
2075                 return -EBUSY;
2076
2077         gr_dfs_delete(dev);
2078         if (dev->desc_pool)
2079                 dma_pool_destroy(dev->desc_pool);
2080         dev_set_drvdata(&ofdev->dev, NULL);
2081
2082         gr_free_request(&dev->epi[0].ep, &dev->ep0reqi->req);
2083         gr_free_request(&dev->epo[0].ep, &dev->ep0reqo->req);
2084
2085         return 0;
2086 }
2087 static int gr_request_irq(struct gr_udc *dev, int irq)
2088 {
2089         return devm_request_threaded_irq(dev->dev, irq, gr_irq, gr_irq_handler,
2090                                          IRQF_SHARED, driver_name, dev);
2091 }
2092
2093 static int gr_probe(struct platform_device *ofdev)
2094 {
2095         struct gr_udc *dev;
2096         struct resource *res;
2097         struct gr_regs __iomem *regs;
2098         int retval;
2099         u32 status;
2100
2101         dev = devm_kzalloc(&ofdev->dev, sizeof(*dev), GFP_KERNEL);
2102         if (!dev)
2103                 return -ENOMEM;
2104         dev->dev = &ofdev->dev;
2105
2106         res = platform_get_resource(ofdev, IORESOURCE_MEM, 0);
2107         regs = devm_ioremap_resource(dev->dev, res);
2108         if (IS_ERR(regs))
2109                 return PTR_ERR(regs);
2110
2111         dev->irq = irq_of_parse_and_map(dev->dev->of_node, 0);
2112         if (!dev->irq) {
2113                 dev_err(dev->dev, "No irq found\n");
2114                 return -ENODEV;
2115         }
2116
2117         /* Some core configurations has separate irqs for IN and OUT events */
2118         dev->irqi = irq_of_parse_and_map(dev->dev->of_node, 1);
2119         if (dev->irqi) {
2120                 dev->irqo = irq_of_parse_and_map(dev->dev->of_node, 2);
2121                 if (!dev->irqo) {
2122                         dev_err(dev->dev, "Found irqi but not irqo\n");
2123                         return -ENODEV;
2124                 }
2125         }
2126
2127         dev->gadget.name = driver_name;
2128         dev->gadget.max_speed = USB_SPEED_HIGH;
2129         dev->gadget.ops = &gr_ops;
2130         dev->gadget.quirk_ep_out_aligned_size = true;
2131
2132         spin_lock_init(&dev->lock);
2133         dev->regs = regs;
2134
2135         dev_set_drvdata(&ofdev->dev, dev);
2136
2137         /* Determine number of endpoints and data interface mode */
2138         status = gr_read32(&dev->regs->status);
2139         dev->nepi = ((status & GR_STATUS_NEPI_MASK) >> GR_STATUS_NEPI_POS) + 1;
2140         dev->nepo = ((status & GR_STATUS_NEPO_MASK) >> GR_STATUS_NEPO_POS) + 1;
2141
2142         if (!(status & GR_STATUS_DM)) {
2143                 dev_err(dev->dev, "Slave mode cores are not supported\n");
2144                 return -ENODEV;
2145         }
2146
2147         /* --- Effects of the following calls might need explicit cleanup --- */
2148
2149         /* Create DMA pool for descriptors */
2150         dev->desc_pool = dma_pool_create("desc_pool", dev->dev,
2151                                          sizeof(struct gr_dma_desc), 4, 0);
2152         if (!dev->desc_pool) {
2153                 dev_err(dev->dev, "Could not allocate DMA pool");
2154                 return -ENOMEM;
2155         }
2156
2157         spin_lock(&dev->lock);
2158
2159         /* Inside lock so that no gadget can use this udc until probe is done */
2160         retval = usb_add_gadget_udc(dev->dev, &dev->gadget);
2161         if (retval) {
2162                 dev_err(dev->dev, "Could not add gadget udc");
2163                 goto out;
2164         }
2165         dev->added = 1;
2166
2167         retval = gr_udc_init(dev);
2168         if (retval)
2169                 goto out;
2170
2171         gr_dfs_create(dev);
2172
2173         /* Clear all interrupt enables that might be left on since last boot */
2174         gr_disable_interrupts_and_pullup(dev);
2175
2176         retval = gr_request_irq(dev, dev->irq);
2177         if (retval) {
2178                 dev_err(dev->dev, "Failed to request irq %d\n", dev->irq);
2179                 goto out;
2180         }
2181
2182         if (dev->irqi) {
2183                 retval = gr_request_irq(dev, dev->irqi);
2184                 if (retval) {
2185                         dev_err(dev->dev, "Failed to request irqi %d\n",
2186                                 dev->irqi);
2187                         goto out;
2188                 }
2189                 retval = gr_request_irq(dev, dev->irqo);
2190                 if (retval) {
2191                         dev_err(dev->dev, "Failed to request irqo %d\n",
2192                                 dev->irqo);
2193                         goto out;
2194                 }
2195         }
2196
2197         if (dev->irqi)
2198                 dev_info(dev->dev, "regs: %p, irqs %d, %d, %d\n", dev->regs,
2199                          dev->irq, dev->irqi, dev->irqo);
2200         else
2201                 dev_info(dev->dev, "regs: %p, irq %d\n", dev->regs, dev->irq);
2202
2203 out:
2204         spin_unlock(&dev->lock);
2205
2206         if (retval)
2207                 gr_remove(ofdev);
2208
2209         return retval;
2210 }
2211
2212 static struct of_device_id gr_match[] = {
2213         {.name = "GAISLER_USBDC"},
2214         {.name = "01_021"},
2215         {},
2216 };
2217 MODULE_DEVICE_TABLE(of, gr_match);
2218
2219 static struct platform_driver gr_driver = {
2220         .driver = {
2221                 .name = DRIVER_NAME,
2222                 .owner = THIS_MODULE,
2223                 .of_match_table = gr_match,
2224         },
2225         .probe = gr_probe,
2226         .remove = gr_remove,
2227 };
2228 module_platform_driver(gr_driver);
2229
2230 MODULE_AUTHOR("Aeroflex Gaisler AB.");
2231 MODULE_DESCRIPTION(DRIVER_DESC);
2232 MODULE_LICENSE("GPL");