drivers/usb/gadget/gr_udc.c

   1 /*
   2  * USB Peripheral Controller driver for Aeroflex Gaisler GRUSBDC.
   3  *
   4  * 2013 (c) Aeroflex Gaisler AB
   5  *
   6  * This driver supports GRUSBDC USB Device Controller cores available in the
   7  * GRLIB VHDL IP core library.
   8  *
   9  * Full documentation of the GRUSBDC core can be found here:
  10  * http://www.gaisler.com/products/grlib/grip.pdf
  11  *
  12  * This program is free software; you can redistribute it and/or modify it
  13  * under the terms of the GNU General Public License as published by the
  14  * Free Software Foundation; either version 2 of the License, or (at your
  15  * option) any later version.
  16  *
  17  * Contributors:
  18  * - Andreas Larsson <andreas@gaisler.com>
  19  * - Marko Isomaki
  20  */
  21
  22 /*
  23  * A GRUSBDC core can have up to 16 IN endpoints and 16 OUT endpoints each
  24  * individually configurable to any of the four USB transfer types. This driver
  25  * only supports cores in DMA mode.
  26  */
  27
  28 #include <linux/kernel.h>
  29 #include <linux/module.h>
  30 #include <linux/slab.h>
  31 #include <linux/spinlock.h>
  32 #include <linux/errno.h>
  33 #include <linux/list.h>
  34 #include <linux/interrupt.h>
  35 #include <linux/device.h>
  36 #include <linux/usb/ch9.h>
  37 #include <linux/usb/gadget.h>
  38 #include <linux/dma-mapping.h>
  39 #include <linux/dmapool.h>
  40 #include <linux/debugfs.h>
  41 #include <linux/seq_file.h>
  42 #include <linux/of_platform.h>
  43 #include <linux/of_irq.h>
  44 #include <linux/of_address.h>
  45
  46 #include <asm/byteorder.h>
  47
  48 #include "gr_udc.h"
  49
  50 #define DRIVER_NAME     "gr_udc"
  51 #define DRIVER_DESC     "Aeroflex Gaisler GRUSBDC USB Peripheral Controller"
  52
  53 static const char driver_name[] = DRIVER_NAME;
  54 static const char driver_desc[] = DRIVER_DESC;
  55
  56 #define gr_read32(x) (ioread32be((x)))
  57 #define gr_write32(x, v) (iowrite32be((v), (x)))
  58
  59 /* USB speed and corresponding string calculated from status register value */
  60 #define GR_SPEED(status) \
  61         ((status & GR_STATUS_SP) ? USB_SPEED_FULL : USB_SPEED_HIGH)
  62 #define GR_SPEED_STR(status) usb_speed_string(GR_SPEED(status))
  63
  64 /* Size of hardware buffer calculated from epctrl register value */
  65 #define GR_BUFFER_SIZE(epctrl)                                        \
  66         ((((epctrl) & GR_EPCTRL_BUFSZ_MASK) >> GR_EPCTRL_BUFSZ_POS) * \
  67          GR_EPCTRL_BUFSZ_SCALER)
  68
  69 /* ---------------------------------------------------------------------- */
  70 /* Debug printout functionality */
  71
  72 static const char * const gr_modestring[] = {"control", "iso", "bulk", "int"};
  73
  74 static const char *gr_ep0state_string(enum gr_ep0state state)
  75 {
  76         static const char *const names[] = {
  77                 [GR_EP0_DISCONNECT] = "disconnect",
  78                 [GR_EP0_SETUP] = "setup",
  79                 [GR_EP0_IDATA] = "idata",
  80                 [GR_EP0_ODATA] = "odata",
  81                 [GR_EP0_ISTATUS] = "istatus",
  82                 [GR_EP0_OSTATUS] = "ostatus",
  83                 [GR_EP0_STALL] = "stall",
  84                 [GR_EP0_SUSPEND] = "suspend",
  85         };
  86
  87         if (state < 0 || state >= ARRAY_SIZE(names))
  88                 return "UNKNOWN";
  89
  90         return names[state];
  91 }
  92
  93 #ifdef VERBOSE_DEBUG
  94
  95 static void gr_dbgprint_request(const char *str, struct gr_ep *ep,
  96                                 struct gr_request *req)
  97 {
  98         int buflen = ep->is_in ? req->req.length : req->req.actual;
  99         int rowlen = 32;
 100         int plen = min(rowlen, buflen);
 101
 102         dev_dbg(ep->dev->dev, "%s: 0x%p, %d bytes data%s:\n", str, req, buflen,
 103                 (buflen > plen ? " (truncated)" : ""));
 104         print_hex_dump_debug("   ", DUMP_PREFIX_NONE,
 105                              rowlen, 4, req->req.buf, plen, false);
 106 }
 107
 108 static void gr_dbgprint_devreq(struct gr_udc *dev, u8 type, u8 request,
 109                                u16 value, u16 index, u16 length)
 110 {
 111         dev_vdbg(dev->dev, "REQ: %02x.%02x v%04x i%04x l%04x\n",
 112                  type, request, value, index, length);
 113 }
 114 #else /* !VERBOSE_DEBUG */
 115
 116 static void gr_dbgprint_request(const char *str, struct gr_ep *ep,
 117                                 struct gr_request *req) {}
 118
 119 static void gr_dbgprint_devreq(struct gr_udc *dev, u8 type, u8 request,
 120                                u16 value, u16 index, u16 length) {}
 121
 122 #endif /* VERBOSE_DEBUG */
 123
 124 /* ---------------------------------------------------------------------- */
 125 /* Debugfs functionality */
 126
 127 #ifdef CONFIG_USB_GADGET_DEBUG_FS
 128
 129 static void gr_seq_ep_show(struct seq_file *seq, struct gr_ep *ep)
 130 {
 131         u32 epctrl = gr_read32(&ep->regs->epctrl);
 132         u32 epstat = gr_read32(&ep->regs->epstat);
 133         int mode = (epctrl & GR_EPCTRL_TT_MASK) >> GR_EPCTRL_TT_POS;
 134         struct gr_request *req;
 135
 136         seq_printf(seq, "%s:\n", ep->ep.name);
 137         seq_printf(seq, "  mode = %s\n", gr_modestring[mode]);
 138         seq_printf(seq, "  halted: %d\n", !!(epctrl & GR_EPCTRL_EH));
 139         seq_printf(seq, "  disabled: %d\n", !!(epctrl & GR_EPCTRL_ED));
 140         seq_printf(seq, "  valid: %d\n", !!(epctrl & GR_EPCTRL_EV));
 141         seq_printf(seq, "  dma_start = %d\n", ep->dma_start);
 142         seq_printf(seq, "  stopped = %d\n", ep->stopped);
 143         seq_printf(seq, "  wedged = %d\n", ep->wedged);
 144         seq_printf(seq, "  callback = %d\n", ep->callback);
 145         seq_printf(seq, "  maxpacket = %d\n", ep->ep.maxpacket);
 146         seq_printf(seq, "  bytes_per_buffer = %d\n", ep->bytes_per_buffer);
 147         if (mode == 1 || mode == 3)
 148                 seq_printf(seq, "  nt = %d\n",
 149                            (epctrl & GR_EPCTRL_NT_MASK) >> GR_EPCTRL_NT_POS);
 150
 151         seq_printf(seq, "  Buffer 0: %s %s%d\n",
 152                    epstat & GR_EPSTAT_B0 ? "valid" : "invalid",
 153                    epstat & GR_EPSTAT_BS ? " " : "selected ",
 154                    (epstat & GR_EPSTAT_B0CNT_MASK) >> GR_EPSTAT_B0CNT_POS);
 155         seq_printf(seq, "  Buffer 1: %s %s%d\n",
 156                    epstat & GR_EPSTAT_B1 ? "valid" : "invalid",
 157                    epstat & GR_EPSTAT_BS ? "selected " : " ",
 158                    (epstat & GR_EPSTAT_B1CNT_MASK) >> GR_EPSTAT_B1CNT_POS);
 159
 160         if (list_empty(&ep->queue)) {
 161                 seq_puts(seq, "  Queue: empty\n\n");
 162                 return;
 163         }
 164
 165         seq_puts(seq, "  Queue:\n");
 166         list_for_each_entry(req, &ep->queue, queue) {
 167                 struct gr_dma_desc *desc;
 168                 struct gr_dma_desc *next;
 169
 170                 seq_printf(seq, "    0x%p: 0x%p %d %d\n", req,
 171                            &req->req.buf, req->req.actual, req->req.length);
 172
 173                 next = req->first_desc;
 174                 do {
 175                         desc = next;
 176                         next = desc->next_desc;
 177                         seq_printf(seq, "    %c 0x%p (0x%08x): 0x%05x 0x%08x\n",
 178                                    desc == req->curr_desc ? 'c' : ' ',
 179                                    desc, desc->paddr, desc->ctrl, desc->data);
 180                 } while (desc != req->last_desc);
 181         }
 182         seq_puts(seq, "\n");
 183 }
 184
 185
 186 static int gr_seq_show(struct seq_file *seq, void *v)
 187 {
 188         struct gr_udc *dev = seq->private;
 189         u32 control = gr_read32(&dev->regs->control);
 190         u32 status = gr_read32(&dev->regs->status);
 191         struct gr_ep *ep;
 192
 193         seq_printf(seq, "usb state = %s\n",
 194                    usb_state_string(dev->gadget.state));
 195         seq_printf(seq, "address = %d\n",
 196                    (control & GR_CONTROL_UA_MASK) >> GR_CONTROL_UA_POS);
 197         seq_printf(seq, "speed = %s\n", GR_SPEED_STR(status));
 198         seq_printf(seq, "ep0state = %s\n", gr_ep0state_string(dev->ep0state));
 199         seq_printf(seq, "irq_enabled = %d\n", dev->irq_enabled);
 200         seq_printf(seq, "remote_wakeup = %d\n", dev->remote_wakeup);
 201         seq_printf(seq, "test_mode = %d\n", dev->test_mode);
 202         seq_puts(seq, "\n");
 203
 204         list_for_each_entry(ep, &dev->ep_list, ep_list)
 205                 gr_seq_ep_show(seq, ep);
 206
 207         return 0;
 208 }
 209
 210 static int gr_dfs_open(struct inode *inode, struct file *file)
 211 {
 212         return single_open(file, gr_seq_show, inode->i_private);
 213 }
 214
 215 static const struct file_operations gr_dfs_fops = {
 216         .owner          = THIS_MODULE,
 217         .open           = gr_dfs_open,
 218         .read           = seq_read,
 219         .llseek         = seq_lseek,
 220         .release        = single_release,
 221 };
 222
 223 static void gr_dfs_create(struct gr_udc *dev)
 224 {
 225         const char *name = "gr_udc_state";
 226
 227         dev->dfs_root = debugfs_create_dir(dev_name(dev->dev), NULL);
 228         if (IS_ERR(dev->dfs_root)) {
 229                 dev_err(dev->dev, "Failed to create debugfs directory\n");
 230                 return;
 231         }
 232         dev->dfs_state = debugfs_create_file(name, 0444, dev->dfs_root,
 233                                              dev, &gr_dfs_fops);
 234         if (IS_ERR(dev->dfs_state))
 235                 dev_err(dev->dev, "Failed to create debugfs file %s\n", name);
 236 }
 237
 238 static void gr_dfs_delete(struct gr_udc *dev)
 239 {
 240         /* Handles NULL and ERR pointers internally */
 241         debugfs_remove(dev->dfs_state);
 242         debugfs_remove(dev->dfs_root);
 243 }
 244
 245 #else /* !CONFIG_USB_GADGET_DEBUG_FS */
 246
 247 static void gr_dfs_create(struct gr_udc *dev) {}
 248 static void gr_dfs_delete(struct gr_udc *dev) {}
 249
 250 #endif /* CONFIG_USB_GADGET_DEBUG_FS */
 251
 252 /* ---------------------------------------------------------------------- */
 253 /* DMA and request handling */
 254
 255 /* Allocates a new struct gr_dma_desc, sets paddr and zeroes the rest */
 256 static struct gr_dma_desc *gr_alloc_dma_desc(struct gr_ep *ep, gfp_t gfp_flags)
 257 {
 258         dma_addr_t paddr;
 259         struct gr_dma_desc *dma_desc;
 260
 261         dma_desc = dma_pool_alloc(ep->dev->desc_pool, gfp_flags, &paddr);
 262         if (!dma_desc) {
 263                 dev_err(ep->dev->dev, "Could not allocate from DMA pool\n");
 264                 return NULL;
 265         }
 266
 267         memset(dma_desc, 0, sizeof(*dma_desc));
 268         dma_desc->paddr = paddr;
 269
 270         return dma_desc;
 271 }
 272
 273 static inline void gr_free_dma_desc(struct gr_udc *dev,
 274                                     struct gr_dma_desc *desc)
 275 {
 276         dma_pool_free(dev->desc_pool, desc, (dma_addr_t)desc->paddr);
 277 }
 278
 279 /* Frees the chain of struct gr_dma_desc for the given request */
 280 static void gr_free_dma_desc_chain(struct gr_udc *dev, struct gr_request *req)
 281 {
 282         struct gr_dma_desc *desc;
 283         struct gr_dma_desc *next;
 284
 285         next = req->first_desc;
 286         if (!next)
 287                 return;
 288
 289         do {
 290                 desc = next;
 291                 next = desc->next_desc;
 292                 gr_free_dma_desc(dev, desc);
 293         } while (desc != req->last_desc);
 294
 295         req->first_desc = NULL;
 296         req->curr_desc = NULL;
 297         req->last_desc = NULL;
 298 }
 299
 300 static void gr_ep0_setup(struct gr_udc *dev, struct gr_request *req);
 301
 302 /*
 303  * Frees allocated resources and calls the appropriate completion function/setup
 304  * package handler for a finished request.
 305  *
 306  * Must be called with dev->lock held and irqs disabled.
 307  */
 308 static void gr_finish_request(struct gr_ep *ep, struct gr_request *req,
 309                               int status)
 310         __releases(&dev->lock)
 311         __acquires(&dev->lock)
 312 {
 313         struct gr_udc *dev;
 314
 315         list_del_init(&req->queue);
 316
 317         if (likely(req->req.status == -EINPROGRESS))
 318                 req->req.status = status;
 319         else
 320                 status = req->req.status;
 321
 322         dev = ep->dev;
 323         usb_gadget_unmap_request(&dev->gadget, &req->req, ep->is_in);
 324         gr_free_dma_desc_chain(dev, req);
 325
 326         if (ep->is_in) /* For OUT, actual gets updated bit by bit */
 327                 req->req.actual = req->req.length;
 328
 329         if (!status) {
 330                 if (ep->is_in)
 331                         gr_dbgprint_request("SENT", ep, req);
 332                 else
 333                         gr_dbgprint_request("RECV", ep, req);
 334         }
 335
 336         /* Prevent changes to ep->queue during callback */
 337         ep->callback = 1;
 338         if (req == dev->ep0reqo && !status) {
 339                 if (req->setup)
 340                         gr_ep0_setup(dev, req);
 341                 else
 342                         dev_err(dev->dev,
 343                                 "Unexpected non setup packet on ep0in\n");
 344         } else if (req->req.complete) {
 345                 spin_unlock(&dev->lock);
 346
 347                 req->req.complete(&ep->ep, &req->req);
 348
 349                 spin_lock(&dev->lock);
 350         }
 351         ep->callback = 0;
 352 }
 353
 354 static struct usb_request *gr_alloc_request(struct usb_ep *_ep, gfp_t gfp_flags)
 355 {
 356         struct gr_request *req;
 357
 358         req = kzalloc(sizeof(*req), gfp_flags);
 359         if (!req)
 360                 return NULL;
 361
 362         INIT_LIST_HEAD(&req->queue);
 363
 364         return &req->req;
 365 }
 366
 367 /*
 368  * Starts DMA for endpoint ep if there are requests in the queue.
 369  *
 370  * Must be called with dev->lock held and with !ep->stopped.
 371  */
 372 static void gr_start_dma(struct gr_ep *ep)
 373 {
 374         struct gr_request *req;
 375         u32 dmactrl;
 376
 377         if (list_empty(&ep->queue)) {
 378                 ep->dma_start = 0;
 379                 return;
 380         }
 381
 382         req = list_first_entry(&ep->queue, struct gr_request, queue);
 383
 384         /* A descriptor should already have been allocated */
 385         BUG_ON(!req->curr_desc);
 386
 387         wmb(); /* Make sure all is settled before handing it over to DMA */
 388
 389         /* Set the descriptor pointer in the hardware */
 390         gr_write32(&ep->regs->dmaaddr, req->curr_desc->paddr);
 391
 392         /* Announce available descriptors */
 393         dmactrl = gr_read32(&ep->regs->dmactrl);
 394         gr_write32(&ep->regs->dmactrl, dmactrl | GR_DMACTRL_DA);
 395
 396         ep->dma_start = 1;
 397 }
 398
 399 /*
 400  * Finishes the first request in the ep's queue and, if available, starts the
 401  * next request in queue.
 402  *
 403  * Must be called with dev->lock held, irqs disabled and with !ep->stopped.
 404  */
 405 static void gr_dma_advance(struct gr_ep *ep, int status)
 406 {
 407         struct gr_request *req;
 408
 409         req = list_first_entry(&ep->queue, struct gr_request, queue);
 410         gr_finish_request(ep, req, status);
 411         gr_start_dma(ep); /* Regardless of ep->dma_start */
 412 }
 413
 414 /*
 415  * Abort DMA for an endpoint. Sets the abort DMA bit which causes an ongoing DMA
 416  * transfer to be canceled and clears GR_DMACTRL_DA.
 417  *
 418  * Must be called with dev->lock held.
 419  */
 420 static void gr_abort_dma(struct gr_ep *ep)
 421 {
 422         u32 dmactrl;
 423
 424         dmactrl = gr_read32(&ep->regs->dmactrl);
 425         gr_write32(&ep->regs->dmactrl, dmactrl | GR_DMACTRL_AD);
 426 }
 427
 428 /*
 429  * Allocates and sets up a struct gr_dma_desc and putting it on the descriptor
 430  * chain.
 431  *
 432  * Size is not used for OUT endpoints. Hardware can not be instructed to handle
 433  * smaller buffer than MAXPL in the OUT direction.
 434  */
 435 static int gr_add_dma_desc(struct gr_ep *ep, struct gr_request *req,
 436                            dma_addr_t data, unsigned size, gfp_t gfp_flags)
 437 {
 438         struct gr_dma_desc *desc;
 439
 440         desc = gr_alloc_dma_desc(ep, gfp_flags);
 441         if (!desc)
 442                 return -ENOMEM;
 443
 444         desc->data = data;
 445         if (ep->is_in)
 446                 desc->ctrl =
 447                         (GR_DESC_IN_CTRL_LEN_MASK & size) | GR_DESC_IN_CTRL_EN;
 448         else
 449                 desc->ctrl = GR_DESC_OUT_CTRL_IE;
 450
 451         if (!req->first_desc) {
 452                 req->first_desc = desc;
 453                 req->curr_desc = desc;
 454         } else {
 455                 req->last_desc->next_desc = desc;
 456                 req->last_desc->next = desc->paddr;
 457                 req->last_desc->ctrl |= GR_DESC_OUT_CTRL_NX;
 458         }
 459         req->last_desc = desc;
 460
 461         return 0;
 462 }
 463
 464 /*
 465  * Sets up a chain of struct gr_dma_descriptors pointing to buffers that
 466  * together covers req->req.length bytes of the buffer at DMA address
 467  * req->req.dma for the OUT direction.
 468  *
 469  * The first descriptor in the chain is enabled, the rest disabled. The
 470  * interrupt handler will later enable them one by one when needed so we can
 471  * find out when the transfer is finished. For OUT endpoints, all descriptors
 472  * therefore generate interrutps.
 473  */
 474 static int gr_setup_out_desc_list(struct gr_ep *ep, struct gr_request *req,
 475                                   gfp_t gfp_flags)
 476 {
 477         u16 bytes_left; /* Bytes left to provide descriptors for */
 478         u16 bytes_used; /* Bytes accommodated for */
 479         int ret = 0;
 480
 481         req->first_desc = NULL; /* Signals that no allocation is done yet */
 482         bytes_left = req->req.length;
 483         bytes_used = 0;
 484         while (bytes_left > 0) {
 485                 dma_addr_t start = req->req.dma + bytes_used;
 486                 u16 size = min(bytes_left, ep->bytes_per_buffer);
 487
 488                 /* Should not happen however - gr_queue stops such lengths */
 489                 if (size < ep->bytes_per_buffer)
 490                         dev_warn(ep->dev->dev,
 491                                  "Buffer overrun risk: %u < %u bytes/buffer\n",
 492                                  size, ep->bytes_per_buffer);
 493
 494                 ret = gr_add_dma_desc(ep, req, start, size, gfp_flags);
 495                 if (ret)
 496                         goto alloc_err;
 497
 498                 bytes_left -= size;
 499                 bytes_used += size;
 500         }
 501
 502         req->first_desc->ctrl |= GR_DESC_OUT_CTRL_EN;
 503
 504         return 0;
 505
 506 alloc_err:
 507         gr_free_dma_desc_chain(ep->dev, req);
 508
 509         return ret;
 510 }
 511
 512 /*
 513  * Sets up a chain of struct gr_dma_descriptors pointing to buffers that
 514  * together covers req->req.length bytes of the buffer at DMA address
 515  * req->req.dma for the IN direction.
 516  *
 517  * When more data is provided than the maximum payload size, the hardware splits
 518  * this up into several payloads automatically. Moreover, ep->bytes_per_buffer
 519  * is always set to a multiple of the maximum payload (restricted to the valid
 520  * number of maximum payloads during high bandwidth isochronous or interrupt
 521  * transfers)
 522  *
 523  * All descriptors are enabled from the beginning and we only generate an
 524  * interrupt for the last one indicating that the entire request has been pushed
 525  * to hardware.
 526  */
 527 static int gr_setup_in_desc_list(struct gr_ep *ep, struct gr_request *req,
 528                                  gfp_t gfp_flags)
 529 {
 530         u16 bytes_left; /* Bytes left in req to provide descriptors for */
 531         u16 bytes_used; /* Bytes in req accommodated for */
 532         int ret = 0;
 533
 534         req->first_desc = NULL; /* Signals that no allocation is done yet */
 535         bytes_left = req->req.length;
 536         bytes_used = 0;
 537         do { /* Allow for zero length packets */
 538                 dma_addr_t start = req->req.dma + bytes_used;
 539                 u16 size = min(bytes_left, ep->bytes_per_buffer);
 540
 541                 ret = gr_add_dma_desc(ep, req, start, size, gfp_flags);
 542                 if (ret)
 543                         goto alloc_err;
 544
 545                 bytes_left -= size;
 546                 bytes_used += size;
 547         } while (bytes_left > 0);
 548
 549         /*
 550          * Send an extra zero length packet to indicate that no more data is
 551          * available when req->req.zero is set and the data length is even
 552          * multiples of ep->ep.maxpacket.
 553          */
 554         if (req->req.zero && (req->req.length % ep->ep.maxpacket == 0)) {
 555                 ret = gr_add_dma_desc(ep, req, 0, 0, gfp_flags);
 556                 if (ret)
 557                         goto alloc_err;
 558         }
 559
 560         /*
 561          * For IN packets we only want to know when the last packet has been
 562          * transmitted (not just put into internal buffers).
 563          */
 564         req->last_desc->ctrl |= GR_DESC_IN_CTRL_PI;
 565
 566         return 0;
 567
 568 alloc_err:
 569         gr_free_dma_desc_chain(ep->dev, req);
 570
 571         return ret;
 572 }
 573
 574 /* Must be called with dev->lock held */
 575 static int gr_queue(struct gr_ep *ep, struct gr_request *req, gfp_t gfp_flags)
 576 {
 577         struct gr_udc *dev = ep->dev;
 578         int ret;
 579
 580         if (unlikely(!ep->ep.desc && ep->num != 0)) {
 581                 dev_err(dev->dev, "No ep descriptor for %s\n", ep->ep.name);
 582                 return -EINVAL;
 583         }
 584
 585         if (unlikely(!req->req.buf || !list_empty(&req->queue))) {
 586                 dev_err(dev->dev,
 587                         "Invalid request for %s: buf=%p list_empty=%d\n",
 588                         ep->ep.name, req->req.buf, list_empty(&req->queue));
 589                 return -EINVAL;
 590         }
 591
 592         /*
 593          * The DMA controller can not handle smaller OUT buffers than
 594          * maxpacket. It could lead to buffer overruns if unexpectedly long
 595          * packet are received.
 596          */
 597         if (!ep->is_in && (req->req.length % ep->ep.maxpacket) != 0) {
 598                 dev_err(dev->dev,
 599                         "OUT request length %d is not multiple of maxpacket\n",
 600                         req->req.length);
 601                 return -EMSGSIZE;
 602         }
 603
 604         if (unlikely(!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN)) {
 605                 dev_err(dev->dev, "-ESHUTDOWN");
 606                 return -ESHUTDOWN;
 607         }
 608
 609         /* Can't touch registers when suspended */
 610         if (dev->ep0state == GR_EP0_SUSPEND) {
 611                 dev_err(dev->dev, "-EBUSY");
 612                 return -EBUSY;
 613         }
 614
 615         /* Set up DMA mapping in case the caller didn't */
 616         ret = usb_gadget_map_request(&dev->gadget, &req->req, ep->is_in);
 617         if (ret) {
 618                 dev_err(dev->dev, "usb_gadget_map_request");
 619                 return ret;
 620         }
 621
 622         if (ep->is_in)
 623                 ret = gr_setup_in_desc_list(ep, req, gfp_flags);
 624         else
 625                 ret = gr_setup_out_desc_list(ep, req, gfp_flags);
 626         if (ret)
 627                 return ret;
 628
 629         req->req.status = -EINPROGRESS;
 630         req->req.actual = 0;
 631         list_add_tail(&req->queue, &ep->queue);
 632
 633         /* Start DMA if not started, otherwise interrupt handler handles it */
 634         if (!ep->dma_start && likely(!ep->stopped))
 635                 gr_start_dma(ep);
 636
 637         return 0;
 638 }
 639
 640 /*
 641  * Queue a request from within the driver.
 642  *
 643  * Must be called with dev->lock held.
 644  */
 645 static inline int gr_queue_int(struct gr_ep *ep, struct gr_request *req,
 646                                gfp_t gfp_flags)
 647 {
 648         if (ep->is_in)
 649                 gr_dbgprint_request("RESP", ep, req);
 650
 651         return gr_queue(ep, req, gfp_flags);
 652 }
 653
 654 /* ---------------------------------------------------------------------- */
 655 /* General helper functions */
 656
 657 /*
 658  * Dequeue ALL requests.
 659  *
 660  * Must be called with dev->lock held and irqs disabled.
 661  */
 662 static void gr_ep_nuke(struct gr_ep *ep)
 663 {
 664         struct gr_request *req;
 665
 666         ep->stopped = 1;
 667         ep->dma_start = 0;
 668         gr_abort_dma(ep);
 669
 670         while (!list_empty(&ep->queue)) {
 671                 req = list_first_entry(&ep->queue, struct gr_request, queue);
 672                 gr_finish_request(ep, req, -ESHUTDOWN);
 673         }
 674 }
 675
 676 /*
 677  * Reset the hardware state of this endpoint.
 678  *
 679  * Must be called with dev->lock held.
 680  */
 681 static void gr_ep_reset(struct gr_ep *ep)
 682 {
 683         gr_write32(&ep->regs->epctrl, 0);
 684         gr_write32(&ep->regs->dmactrl, 0);
 685
 686         ep->ep.maxpacket = MAX_CTRL_PL_SIZE;
 687         ep->ep.desc = NULL;
 688         ep->stopped = 1;
 689         ep->dma_start = 0;
 690 }
 691
 692 /*
 693  * Generate STALL on ep0in/out.
 694  *
 695  * Must be called with dev->lock held.
 696  */
 697 static void gr_control_stall(struct gr_udc *dev)
 698 {
 699         u32 epctrl;
 700
 701         epctrl = gr_read32(&dev->epo[0].regs->epctrl);
 702         gr_write32(&dev->epo[0].regs->epctrl, epctrl | GR_EPCTRL_CS);
 703         epctrl = gr_read32(&dev->epi[0].regs->epctrl);
 704         gr_write32(&dev->epi[0].regs->epctrl, epctrl | GR_EPCTRL_CS);
 705
 706         dev->ep0state = GR_EP0_STALL;
 707 }
 708
 709 /*
 710  * Halts, halts and wedges, or clears halt for an endpoint.
 711  *
 712  * Must be called with dev->lock held.
 713  */
 714 static int gr_ep_halt_wedge(struct gr_ep *ep, int halt, int wedge, int fromhost)
 715 {
 716         u32 epctrl;
 717         int retval = 0;
 718
 719         if (ep->num && !ep->ep.desc)
 720                 return -EINVAL;
 721
 722         if (ep->num && ep->ep.desc->bmAttributes == USB_ENDPOINT_XFER_ISOC)
 723                 return -EOPNOTSUPP;
 724
 725         /* Never actually halt ep0, and therefore never clear halt for ep0 */
 726         if (!ep->num) {
 727                 if (halt && !fromhost) {
 728                         /* ep0 halt from gadget - generate protocol stall */
 729                         gr_control_stall(ep->dev);
 730                         dev_dbg(ep->dev->dev, "EP: stall ep0\n");
 731                         return 0;
 732                 }
 733                 return -EINVAL;
 734         }
 735
 736         dev_dbg(ep->dev->dev, "EP: %s halt %s\n",
 737                 (halt ? (wedge ? "wedge" : "set") : "clear"), ep->ep.name);
 738
 739         epctrl = gr_read32(&ep->regs->epctrl);
 740         if (halt) {
 741                 /* Set HALT */
 742                 gr_write32(&ep->regs->epctrl, epctrl | GR_EPCTRL_EH);
 743                 ep->stopped = 1;
 744                 if (wedge)
 745                         ep->wedged = 1;
 746         } else {
 747                 gr_write32(&ep->regs->epctrl, epctrl & ~GR_EPCTRL_EH);
 748                 ep->stopped = 0;
 749                 ep->wedged = 0;
 750
 751                 /* Things might have been queued up in the meantime */
 752                 if (!ep->dma_start)
 753                         gr_start_dma(ep);
 754         }
 755
 756         return retval;
 757 }
 758
 759 /* Must be called with dev->lock held */
 760 static inline void gr_set_ep0state(struct gr_udc *dev, enum gr_ep0state value)
 761 {
 762         if (dev->ep0state != value)
 763                 dev_vdbg(dev->dev, "STATE:  ep0state=%s\n",
 764                          gr_ep0state_string(value));
 765         dev->ep0state = value;
 766 }
 767
 768 /*
 769  * Should only be called when endpoints can not generate interrupts.
 770  *
 771  * Must be called with dev->lock held.
 772  */
 773 static void gr_disable_interrupts_and_pullup(struct gr_udc *dev)
 774 {
 775         gr_write32(&dev->regs->control, 0);
 776         wmb(); /* Make sure that we do not deny one of our interrupts */
 777         dev->irq_enabled = 0;
 778 }
 779
 780 /*
 781  * Stop all device activity and disable data line pullup.
 782  *
 783  * Must be called with dev->lock held and irqs disabled.
 784  */
 785 static void gr_stop_activity(struct gr_udc *dev)
 786 {
 787         struct gr_ep *ep;
 788
 789         list_for_each_entry(ep, &dev->ep_list, ep_list)
 790                 gr_ep_nuke(ep);
 791
 792         gr_disable_interrupts_and_pullup(dev);
 793
 794         gr_set_ep0state(dev, GR_EP0_DISCONNECT);
 795         usb_gadget_set_state(&dev->gadget, USB_STATE_NOTATTACHED);
 796 }
 797
 798 /* ---------------------------------------------------------------------- */
 799 /* ep0 setup packet handling */
 800
 801 static void gr_ep0_testmode_complete(struct usb_ep *_ep,
 802                                      struct usb_request *_req)
 803 {
 804         struct gr_ep *ep;
 805         struct gr_udc *dev;
 806         u32 control;
 807
 808         ep = container_of(_ep, struct gr_ep, ep);
 809         dev = ep->dev;
 810
 811         spin_lock(&dev->lock);
 812
 813         control = gr_read32(&dev->regs->control);
 814         control |= GR_CONTROL_TM | (dev->test_mode << GR_CONTROL_TS_POS);
 815         gr_write32(&dev->regs->control, control);
 816
 817         spin_unlock(&dev->lock);
 818 }
 819
 820 static void gr_ep0_dummy_complete(struct usb_ep *_ep, struct usb_request *_req)
 821 {
 822         /* Nothing needs to be done here */
 823 }
 824
 825 /*
 826  * Queue a response on ep0in.
 827  *
 828  * Must be called with dev->lock held.
 829  */
 830 static int gr_ep0_respond(struct gr_udc *dev, u8 *buf, int length,
 831                           void (*complete)(struct usb_ep *ep,
 832                                            struct usb_request *req))
 833 {
 834         u8 *reqbuf = dev->ep0reqi->req.buf;
 835         int status;
 836         int i;
 837
 838         for (i = 0; i < length; i++)
 839                 reqbuf[i] = buf[i];
 840         dev->ep0reqi->req.length = length;
 841         dev->ep0reqi->req.complete = complete;
 842
 843         status = gr_queue_int(&dev->epi[0], dev->ep0reqi, GFP_ATOMIC);
 844         if (status < 0)
 845                 dev_err(dev->dev,
 846                         "Could not queue ep0in setup response: %d\n", status);
 847
 848         return status;
 849 }
 850
 851 /*
 852  * Queue a 2 byte response on ep0in.
 853  *
 854  * Must be called with dev->lock held.
 855  */
 856 static inline int gr_ep0_respond_u16(struct gr_udc *dev, u16 response)
 857 {
 858         __le16 le_response = cpu_to_le16(response);
 859
 860         return gr_ep0_respond(dev, (u8 *)&le_response, 2,
 861                               gr_ep0_dummy_complete);
 862 }
 863
 864 /*
 865  * Queue a ZLP response on ep0in.
 866  *
 867  * Must be called with dev->lock held.
 868  */
 869 static inline int gr_ep0_respond_empty(struct gr_udc *dev)
 870 {
 871         return gr_ep0_respond(dev, NULL, 0, gr_ep0_dummy_complete);
 872 }
 873
 874 /*
 875  * This is run when a SET_ADDRESS request is received. First writes
 876  * the new address to the control register which is updated internally
 877  * when the next IN packet is ACKED.
 878  *
 879  * Must be called with dev->lock held.
 880  */
 881 static void gr_set_address(struct gr_udc *dev, u8 address)
 882 {
 883         u32 control;
 884
 885         control = gr_read32(&dev->regs->control) & ~GR_CONTROL_UA_MASK;
 886         control |= (address << GR_CONTROL_UA_POS) & GR_CONTROL_UA_MASK;
 887         control |= GR_CONTROL_SU;
 888         gr_write32(&dev->regs->control, control);
 889 }
 890
 891 /*
 892  * Returns negative for STALL, 0 for successful handling and positive for
 893  * delegation.
 894  *
 895  * Must be called with dev->lock held.
 896  */
 897 static int gr_device_request(struct gr_udc *dev, u8 type, u8 request,
 898                              u16 value, u16 index)
 899 {
 900         u16 response;
 901         u8 test;
 902
 903         switch (request) {
 904         case USB_REQ_SET_ADDRESS:
 905                 dev_dbg(dev->dev, "STATUS: address %d\n", value & 0xff);
 906                 gr_set_address(dev, value & 0xff);
 907                 if (value)
 908                         usb_gadget_set_state(&dev->gadget, USB_STATE_ADDRESS);
 909                 else
 910                         usb_gadget_set_state(&dev->gadget, USB_STATE_DEFAULT);
 911                 return gr_ep0_respond_empty(dev);
 912
 913         case USB_REQ_GET_STATUS:
 914                 /* Self powered | remote wakeup */
 915                 response = 0x0001 | (dev->remote_wakeup ? 0x0002 : 0);
 916                 return gr_ep0_respond_u16(dev, response);
 917
 918         case USB_REQ_SET_FEATURE:
 919                 switch (value) {
 920                 case USB_DEVICE_REMOTE_WAKEUP:
 921                         /* Allow remote wakeup */
 922                         dev->remote_wakeup = 1;
 923                         return gr_ep0_respond_empty(dev);
 924
 925                 case USB_DEVICE_TEST_MODE:
 926                         /* The hardware does not support TEST_FORCE_EN */
 927                         test = index >> 8;
 928                         if (test >= TEST_J && test <= TEST_PACKET) {
 929                                 dev->test_mode = test;
 930                                 return gr_ep0_respond(dev, NULL, 0,
 931                                                       gr_ep0_testmode_complete);
 932                         }
 933                 }
 934                 break;
 935
 936         case USB_REQ_CLEAR_FEATURE:
 937                 switch (value) {
 938                 case USB_DEVICE_REMOTE_WAKEUP:
 939                         /* Disallow remote wakeup */
 940                         dev->remote_wakeup = 0;
 941                         return gr_ep0_respond_empty(dev);
 942                 }
 943                 break;
 944         }
 945
 946         return 1; /* Delegate the rest */
 947 }
 948
 949 /*
 950  * Returns negative for STALL, 0 for successful handling and positive for
 951  * delegation.
 952  *
 953  * Must be called with dev->lock held.
 954  */
 955 static int gr_interface_request(struct gr_udc *dev, u8 type, u8 request,
 956                                 u16 value, u16 index)
 957 {
 958         if (dev->gadget.state != USB_STATE_CONFIGURED)
 959                 return -1;
 960
 961         /*
 962          * Should return STALL for invalid interfaces, but udc driver does not
 963          * know anything about that. However, many gadget drivers do not handle
 964          * GET_STATUS so we need to take care of that.
 965          */
 966
 967         switch (request) {
 968         case USB_REQ_GET_STATUS:
 969                 return gr_ep0_respond_u16(dev, 0x0000);
 970
 971         case USB_REQ_SET_FEATURE:
 972         case USB_REQ_CLEAR_FEATURE:
 973                 /*
 974                  * No possible valid standard requests. Still let gadget drivers
 975                  * have a go at it.
 976                  */
 977                 break;
 978         }
 979
 980         return 1; /* Delegate the rest */
 981 }
 982
 983 /*
 984  * Returns negative for STALL, 0 for successful handling and positive for
 985  * delegation.
 986  *
 987  * Must be called with dev->lock held.
 988  */
 989 static int gr_endpoint_request(struct gr_udc *dev, u8 type, u8 request,
 990                                u16 value, u16 index)
 991 {
 992         struct gr_ep *ep;
 993         int status;
 994         int halted;
 995         u8 epnum = index & USB_ENDPOINT_NUMBER_MASK;
 996         u8 is_in = index & USB_ENDPOINT_DIR_MASK;
 997
 998         if ((is_in && epnum >= dev->nepi) || (!is_in && epnum >= dev->nepo))
 999                 return -1;
1000
1001         if (dev->gadget.state != USB_STATE_CONFIGURED && epnum != 0)
1002                 return -1;
1003
1004         ep = (is_in ? &dev->epi[epnum] : &dev->epo[epnum]);
1005
1006         switch (request) {
1007         case USB_REQ_GET_STATUS:
1008                 halted = gr_read32(&ep->regs->epctrl) & GR_EPCTRL_EH;
1009                 return gr_ep0_respond_u16(dev, halted ? 0x0001 : 0);
1010
1011         case USB_REQ_SET_FEATURE:
1012                 switch (value) {
1013                 case USB_ENDPOINT_HALT:
1014                         status = gr_ep_halt_wedge(ep, 1, 0, 1);
1015                         if (status >= 0)
1016                                 status = gr_ep0_respond_empty(dev);
1017                         return status;
1018                 }
1019                 break;
1020
1021         case USB_REQ_CLEAR_FEATURE:
1022                 switch (value) {
1023                 case USB_ENDPOINT_HALT:
1024                         if (ep->wedged)
1025                                 return -1;
1026                         status = gr_ep_halt_wedge(ep, 0, 0, 1);
1027                         if (status >= 0)
1028                                 status = gr_ep0_respond_empty(dev);
1029                         return status;
1030                 }
1031                 break;
1032         }
1033
1034         return 1; /* Delegate the rest */
1035 }
1036
1037 /* Must be called with dev->lock held */
1038 static void gr_ep0out_requeue(struct gr_udc *dev)
1039 {
1040         int ret = gr_queue_int(&dev->epo[0], dev->ep0reqo, GFP_ATOMIC);
1041
1042         if (ret)
1043                 dev_err(dev->dev, "Could not queue ep0out setup request: %d\n",
1044                         ret);
1045 }
1046
1047 /*
1048  * The main function dealing with setup requests on ep0.
1049  *
1050  * Must be called with dev->lock held and irqs disabled
1051  */
1052 static void gr_ep0_setup(struct gr_udc *dev, struct gr_request *req)
1053         __releases(&dev->lock)
1054         __acquires(&dev->lock)
1055 {
1056         union {
1057                 struct usb_ctrlrequest ctrl;
1058                 u8 raw[8];
1059                 u32 word[2];
1060         } u;
1061         u8 type;
1062         u8 request;
1063         u16 value;
1064         u16 index;
1065         u16 length;
1066         int i;
1067         int status;
1068
1069         /* Restore from ep0 halt */
1070         if (dev->ep0state == GR_EP0_STALL) {
1071                 gr_set_ep0state(dev, GR_EP0_SETUP);
1072                 if (!req->req.actual)
1073                         goto out;
1074         }
1075
1076         if (dev->ep0state == GR_EP0_ISTATUS) {
1077                 gr_set_ep0state(dev, GR_EP0_SETUP);
1078                 if (req->req.actual > 0)
1079                         dev_dbg(dev->dev,
1080                                 "Unexpected setup packet at state %s\n",
1081                                 gr_ep0state_string(GR_EP0_ISTATUS));
1082                 else
1083                         goto out; /* Got expected ZLP */
1084         } else if (dev->ep0state != GR_EP0_SETUP) {
1085                 dev_info(dev->dev,
1086                          "Unexpected ep0out request at state %s - stalling\n",
1087                          gr_ep0state_string(dev->ep0state));
1088                 gr_control_stall(dev);
1089                 gr_set_ep0state(dev, GR_EP0_SETUP);
1090                 goto out;
1091         } else if (!req->req.actual) {
1092                 dev_dbg(dev->dev, "Unexpected ZLP at state %s\n",
1093                         gr_ep0state_string(dev->ep0state));
1094                 goto out;
1095         }
1096
1097         /* Handle SETUP packet */
1098         for (i = 0; i < req->req.actual; i++)
1099                 u.raw[i] = ((u8 *)req->req.buf)[i];
1100
1101         type = u.ctrl.bRequestType;
1102         request = u.ctrl.bRequest;
1103         value = le16_to_cpu(u.ctrl.wValue);
1104         index = le16_to_cpu(u.ctrl.wIndex);
1105         length = le16_to_cpu(u.ctrl.wLength);
1106
1107         gr_dbgprint_devreq(dev, type, request, value, index, length);
1108
1109         /* Check for data stage */
1110         if (length) {
1111                 if (type & USB_DIR_IN)
1112                         gr_set_ep0state(dev, GR_EP0_IDATA);
1113                 else
1114                         gr_set_ep0state(dev, GR_EP0_ODATA);
1115         }
1116
1117         status = 1; /* Positive status flags delegation */
1118         if ((type & USB_TYPE_MASK) == USB_TYPE_STANDARD) {
1119                 switch (type & USB_RECIP_MASK) {
1120                 case USB_RECIP_DEVICE:
1121                         status = gr_device_request(dev, type, request,
1122                                                    value, index);
1123                         break;
1124                 case USB_RECIP_ENDPOINT:
1125                         status =  gr_endpoint_request(dev, type, request,
1126                                                       value, index);
1127                         break;
1128                 case USB_RECIP_INTERFACE:
1129                         status = gr_interface_request(dev, type, request,
1130                                                       value, index);
1131                         break;
1132                 }
1133         }
1134
1135         if (status > 0) {
1136                 spin_unlock(&dev->lock);
1137
1138                 dev_vdbg(dev->dev, "DELEGATE\n");
1139                 status = dev->driver->setup(&dev->gadget, &u.ctrl);
1140
1141                 spin_lock(&dev->lock);
1142         }
1143
1144         /* Generate STALL on both ep0out and ep0in if requested */
1145         if (unlikely(status < 0)) {
1146                 dev_vdbg(dev->dev, "STALL\n");
1147                 gr_control_stall(dev);
1148         }
1149
1150         if ((type & USB_TYPE_MASK) == USB_TYPE_STANDARD &&
1151             request == USB_REQ_SET_CONFIGURATION) {
1152                 if (!value) {
1153                         dev_dbg(dev->dev, "STATUS: deconfigured\n");
1154                         usb_gadget_set_state(&dev->gadget, USB_STATE_ADDRESS);
1155                 } else if (status >= 0) {
1156                         /* Not configured unless gadget OK:s it */
1157                         dev_dbg(dev->dev, "STATUS: configured: %d\n", value);
1158                         usb_gadget_set_state(&dev->gadget,
1159                                              USB_STATE_CONFIGURED);
1160                 }
1161         }
1162
1163         /* Get ready for next stage */
1164         if (dev->ep0state == GR_EP0_ODATA)
1165                 gr_set_ep0state(dev, GR_EP0_OSTATUS);
1166         else if (dev->ep0state == GR_EP0_IDATA)
1167                 gr_set_ep0state(dev, GR_EP0_ISTATUS);
1168         else
1169                 gr_set_ep0state(dev, GR_EP0_SETUP);
1170
1171 out:
1172         gr_ep0out_requeue(dev);
1173 }
1174
1175 /* ---------------------------------------------------------------------- */
1176 /* VBUS and USB reset handling */
1177
1178 /* Must be called with dev->lock held and irqs disabled  */
1179 static void gr_vbus_connected(struct gr_udc *dev, u32 status)
1180 {
1181         u32 control;
1182
1183         dev->gadget.speed = GR_SPEED(status);
1184         usb_gadget_set_state(&dev->gadget, USB_STATE_POWERED);
1185
1186         /* Turn on full interrupts and pullup */
1187         control = (GR_CONTROL_SI | GR_CONTROL_UI | GR_CONTROL_VI |
1188                    GR_CONTROL_SP | GR_CONTROL_EP);
1189         gr_write32(&dev->regs->control, control);
1190 }
1191
1192 /* Must be called with dev->lock held */
1193 static void gr_enable_vbus_detect(struct gr_udc *dev)
1194 {
1195         u32 status;
1196
1197         dev->irq_enabled = 1;
1198         wmb(); /* Make sure we do not ignore an interrupt */
1199         gr_write32(&dev->regs->control, GR_CONTROL_VI);
1200
1201         /* Take care of the case we are already plugged in at this point */
1202         status = gr_read32(&dev->regs->status);
1203         if (status & GR_STATUS_VB)
1204                 gr_vbus_connected(dev, status);
1205 }
1206
1207 /* Must be called with dev->lock held and irqs disabled */
1208 static void gr_vbus_disconnected(struct gr_udc *dev)
1209 {
1210         gr_stop_activity(dev);
1211
1212         /* Report disconnect */
1213         if (dev->driver && dev->driver->disconnect) {
1214                 spin_unlock(&dev->lock);
1215
1216                 dev->driver->disconnect(&dev->gadget);
1217
1218                 spin_lock(&dev->lock);
1219         }
1220
1221         gr_enable_vbus_detect(dev);
1222 }
1223
1224 /* Must be called with dev->lock held and irqs disabled */
1225 static void gr_udc_usbreset(struct gr_udc *dev, u32 status)
1226 {
1227         gr_set_address(dev, 0);
1228         gr_set_ep0state(dev, GR_EP0_SETUP);
1229         usb_gadget_set_state(&dev->gadget, USB_STATE_DEFAULT);
1230         dev->gadget.speed = GR_SPEED(status);
1231
1232         gr_ep_nuke(&dev->epo[0]);
1233         gr_ep_nuke(&dev->epi[0]);
1234         dev->epo[0].stopped = 0;
1235         dev->epi[0].stopped = 0;
1236         gr_ep0out_requeue(dev);
1237 }
1238
1239 /* ---------------------------------------------------------------------- */
1240 /* Irq handling */
1241
1242 /*
1243  * Handles interrupts from in endpoints. Returns whether something was handled.
1244  *
1245  * Must be called with dev->lock held, irqs disabled and with !ep->stopped.
1246  */
1247 static int gr_handle_in_ep(struct gr_ep *ep)
1248 {
1249         struct gr_request *req;
1250
1251         req = list_first_entry(&ep->queue, struct gr_request, queue);
1252         if (!req->last_desc)
1253                 return 0;
1254
1255         if (ACCESS_ONCE(req->last_desc->ctrl) & GR_DESC_IN_CTRL_EN)
1256                 return 0; /* Not put in hardware buffers yet */
1257
1258         if (gr_read32(&ep->regs->epstat) & (GR_EPSTAT_B1 | GR_EPSTAT_B0))
1259                 return 0; /* Not transmitted yet, still in hardware buffers */
1260
1261         /* Write complete */
1262         gr_dma_advance(ep, 0);
1263
1264         return 1;
1265 }
1266
1267 /*
1268  * Handles interrupts from out endpoints. Returns whether something was handled.
1269  *
1270  * Must be called with dev->lock held, irqs disabled and with !ep->stopped.
1271  */
1272 static int gr_handle_out_ep(struct gr_ep *ep)
1273 {
1274         u32 ep_dmactrl;
1275         u32 ctrl;
1276         u16 len;
1277         struct gr_request *req;
1278         struct gr_udc *dev = ep->dev;
1279
1280         req = list_first_entry(&ep->queue, struct gr_request, queue);
1281         if (!req->curr_desc)
1282                 return 0;
1283
1284         ctrl = ACCESS_ONCE(req->curr_desc->ctrl);
1285         if (ctrl & GR_DESC_OUT_CTRL_EN)
1286                 return 0; /* Not received yet */
1287
1288         /* Read complete */
1289         len = ctrl & GR_DESC_OUT_CTRL_LEN_MASK;
1290         req->req.actual += len;
1291         if (ctrl & GR_DESC_OUT_CTRL_SE)
1292                 req->setup = 1;
1293
1294         if (len < ep->ep.maxpacket || req->req.actual == req->req.length) {
1295                 /* Short packet or the expected size - we are done */
1296
1297                 if ((ep == &dev->epo[0]) && (dev->ep0state == GR_EP0_OSTATUS)) {
1298                         /*
1299                          * Send a status stage ZLP to ack the DATA stage in the
1300                          * OUT direction. This needs to be done before
1301                          * gr_dma_advance as that can lead to a call to
1302                          * ep0_setup that can change dev->ep0state.
1303                          */
1304                         gr_ep0_respond_empty(dev);
1305                         gr_set_ep0state(dev, GR_EP0_SETUP);
1306                 }
1307
1308                 gr_dma_advance(ep, 0);
1309         } else {
1310                 /* Not done yet. Enable the next descriptor to receive more. */
1311                 req->curr_desc = req->curr_desc->next_desc;
1312                 req->curr_desc->ctrl |= GR_DESC_OUT_CTRL_EN;
1313
1314                 ep_dmactrl = gr_read32(&ep->regs->dmactrl);
1315                 gr_write32(&ep->regs->dmactrl, ep_dmactrl | GR_DMACTRL_DA);
1316         }
1317
1318         return 1;
1319 }
1320
1321 /*
1322  * Handle state changes. Returns whether something was handled.
1323  *
1324  * Must be called with dev->lock held and irqs disabled.
1325  */
1326 static int gr_handle_state_changes(struct gr_udc *dev)
1327 {
1328         u32 status = gr_read32(&dev->regs->status);
1329         int handled = 0;
1330         int powstate = !(dev->gadget.state == USB_STATE_NOTATTACHED ||
1331                          dev->gadget.state == USB_STATE_ATTACHED);
1332
1333         /* VBUS valid detected */
1334         if (!powstate && (status & GR_STATUS_VB)) {
1335                 dev_dbg(dev->dev, "STATUS: vbus valid detected\n");
1336                 gr_vbus_connected(dev, status);
1337                 handled = 1;
1338         }
1339
1340         /* Disconnect */
1341         if (powstate && !(status & GR_STATUS_VB)) {
1342                 dev_dbg(dev->dev, "STATUS: vbus invalid detected\n");
1343                 gr_vbus_disconnected(dev);
1344                 handled = 1;
1345         }
1346
1347         /* USB reset detected */
1348         if (status & GR_STATUS_UR) {
1349                 dev_dbg(dev->dev, "STATUS: USB reset - speed is %s\n",
1350                         GR_SPEED_STR(status));
1351                 gr_write32(&dev->regs->status, GR_STATUS_UR);
1352                 gr_udc_usbreset(dev, status);
1353                 handled = 1;
1354         }
1355
1356         /* Speed change */
1357         if (dev->gadget.speed != GR_SPEED(status)) {
1358                 dev_dbg(dev->dev, "STATUS: USB Speed change to %s\n",
1359                         GR_SPEED_STR(status));
1360                 dev->gadget.speed = GR_SPEED(status);
1361                 handled = 1;
1362         }
1363
1364         /* Going into suspend */
1365         if ((dev->ep0state != GR_EP0_SUSPEND) && !(status & GR_STATUS_SU)) {
1366                 dev_dbg(dev->dev, "STATUS: USB suspend\n");
1367                 gr_set_ep0state(dev, GR_EP0_SUSPEND);
1368                 dev->suspended_from = dev->gadget.state;
1369                 usb_gadget_set_state(&dev->gadget, USB_STATE_SUSPENDED);
1370
1371                 if ((dev->gadget.speed != USB_SPEED_UNKNOWN) &&
1372                     dev->driver && dev->driver->suspend) {
1373                         spin_unlock(&dev->lock);
1374
1375                         dev->driver->suspend(&dev->gadget);
1376
1377                         spin_lock(&dev->lock);
1378                 }
1379                 handled = 1;
1380         }
1381
1382         /* Coming out of suspend */
1383         if ((dev->ep0state == GR_EP0_SUSPEND) && (status & GR_STATUS_SU)) {
1384                 dev_dbg(dev->dev, "STATUS: USB resume\n");
1385                 if (dev->suspended_from == USB_STATE_POWERED)
1386                         gr_set_ep0state(dev, GR_EP0_DISCONNECT);
1387                 else
1388                         gr_set_ep0state(dev, GR_EP0_SETUP);
1389                 usb_gadget_set_state(&dev->gadget, dev->suspended_from);
1390
1391                 if ((dev->gadget.speed != USB_SPEED_UNKNOWN) &&
1392                     dev->driver && dev->driver->resume) {
1393                         spin_unlock(&dev->lock);
1394
1395                         dev->driver->resume(&dev->gadget);
1396
1397                         spin_lock(&dev->lock);
1398                 }
1399                 handled = 1;
1400         }
1401
1402         return handled;
1403 }
1404
1405 /* Non-interrupt context irq handler */
1406 static irqreturn_t gr_irq_handler(int irq, void *_dev)
1407 {
1408         struct gr_udc *dev = _dev;
1409         struct gr_ep *ep;
1410         int handled = 0;
1411         int i;
1412         unsigned long flags;
1413
1414         spin_lock_irqsave(&dev->lock, flags);
1415
1416         if (!dev->irq_enabled)
1417                 goto out;
1418
1419         /*
1420          * Check IN ep interrupts. We check these before the OUT eps because
1421          * some gadgets reuse the request that might already be currently
1422          * outstanding and needs to be completed (mainly setup requests).
1423          */
1424         for (i = 0; i < dev->nepi; i++) {
1425                 ep = &dev->epi[i];
1426                 if (!ep->stopped && !ep->callback && !list_empty(&ep->queue))
1427                         handled = gr_handle_in_ep(ep) || handled;
1428         }
1429
1430         /* Check OUT ep interrupts */
1431         for (i = 0; i < dev->nepo; i++) {
1432                 ep = &dev->epo[i];
1433                 if (!ep->stopped && !ep->callback && !list_empty(&ep->queue))
1434                         handled = gr_handle_out_ep(ep) || handled;
1435         }
1436
1437         /* Check status interrupts */
1438         handled = gr_handle_state_changes(dev) || handled;
1439
1440         /*
1441          * Check AMBA DMA errors. Only check if we didn't find anything else to
1442          * handle because this shouldn't happen if we did everything right.
1443          */
1444         if (!handled) {
1445                 list_for_each_entry(ep, &dev->ep_list, ep_list) {
1446                         if (gr_read32(&ep->regs->dmactrl) & GR_DMACTRL_AE) {
1447                                 dev_err(dev->dev,
1448                                         "AMBA Error occurred for %s\n",
1449                                         ep->ep.name);
1450                                 handled = 1;
1451                         }
1452                 }
1453         }
1454
1455 out:
1456         spin_unlock_irqrestore(&dev->lock, flags);
1457
1458         return handled ? IRQ_HANDLED : IRQ_NONE;
1459 }
1460
1461 /* Interrupt context irq handler */
1462 static irqreturn_t gr_irq(int irq, void *_dev)
1463 {
1464         struct gr_udc *dev = _dev;
1465
1466         if (!dev->irq_enabled)
1467                 return IRQ_NONE;
1468
1469         return IRQ_WAKE_THREAD;
1470 }
1471
1472 /* ---------------------------------------------------------------------- */
1473 /* USB ep ops */
1474
1475 /* Enable endpoint. Not for ep0in and ep0out that are handled separately. */
1476 static int gr_ep_enable(struct usb_ep *_ep,
1477                         const struct usb_endpoint_descriptor *desc)
1478 {
1479         struct gr_udc *dev;
1480         struct gr_ep *ep;
1481         u8 mode;
1482         u8 nt;
1483         u16 max;
1484         u16 buffer_size = 0;
1485         u32 epctrl;
1486
1487         ep = container_of(_ep, struct gr_ep, ep);
1488         if (!_ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT)
1489                 return -EINVAL;
1490
1491         dev = ep->dev;
1492
1493         /* 'ep0' IN and OUT are reserved */
1494         if (ep == &dev->epo[0] || ep == &dev->epi[0])
1495                 return -EINVAL;
1496
1497         if (!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN)
1498                 return -ESHUTDOWN;
1499
1500         /* Make sure we are clear for enabling */
1501         epctrl = gr_read32(&ep->regs->epctrl);
1502         if (epctrl & GR_EPCTRL_EV)
1503                 return -EBUSY;
1504
1505         /* Check that directions match */
1506         if (!ep->is_in != !usb_endpoint_dir_in(desc))
1507                 return -EINVAL;
1508
1509         /* Check ep num */
1510         if ((!ep->is_in && ep->num >= dev->nepo) ||
1511             (ep->is_in && ep->num >= dev->nepi))
1512                 return -EINVAL;
1513
1514         if (usb_endpoint_xfer_control(desc)) {
1515                 mode = 0;
1516         } else if (usb_endpoint_xfer_isoc(desc)) {
1517                 mode = 1;
1518         } else if (usb_endpoint_xfer_bulk(desc)) {
1519                 mode = 2;
1520         } else if (usb_endpoint_xfer_int(desc)) {
1521                 mode = 3;
1522         } else {
1523                 dev_err(dev->dev, "Unknown transfer type for %s\n",
1524                         ep->ep.name);
1525                 return -EINVAL;
1526         }
1527
1528         /*
1529          * Bits 10-0 set the max payload. 12-11 set the number of
1530          * additional transactions.
1531          */
1532         max = 0x7ff & usb_endpoint_maxp(desc);
1533         nt = 0x3 & (usb_endpoint_maxp(desc) >> 11);
1534         buffer_size = GR_BUFFER_SIZE(epctrl);
1535         if (nt && (mode == 0 || mode == 2)) {
1536                 dev_err(dev->dev,
1537                         "%s mode: multiple trans./microframe not valid\n",
1538                         (mode == 2 ? "Bulk" : "Control"));
1539                 return -EINVAL;
1540         } else if (nt == 0x11) {
1541                 dev_err(dev->dev, "Invalid value for trans./microframe\n");
1542                 return -EINVAL;
1543         } else if ((nt + 1) * max > buffer_size) {
1544                 dev_err(dev->dev, "Hw buffer size %d < max payload %d * %d\n",
1545                         buffer_size, (nt + 1), max);
1546                 return -EINVAL;
1547         } else if (max == 0) {
1548                 dev_err(dev->dev, "Max payload cannot be set to 0\n");
1549                 return -EINVAL;
1550         }
1551
1552         spin_lock(&ep->dev->lock);
1553
1554         if (!ep->stopped) {
1555                 spin_unlock(&ep->dev->lock);
1556                 return -EBUSY;
1557         }
1558
1559         ep->stopped = 0;
1560         ep->wedged = 0;
1561         ep->ep.desc = desc;
1562         ep->ep.maxpacket = max;
1563         ep->dma_start = 0;
1564
1565
1566         if (nt) {
1567                 /*
1568                  * Maximum possible size of all payloads in one microframe
1569                  * regardless of direction when using high-bandwidth mode.
1570                  */
1571                 ep->bytes_per_buffer = (nt + 1) * max;
1572         } else if (ep->is_in) {
1573                 /*
1574                  * The biggest multiple of maximum packet size that fits into
1575                  * the buffer. The hardware will split up into many packets in
1576                  * the IN direction.
1577                  */
1578                 ep->bytes_per_buffer = (buffer_size / max) * max;
1579         } else {
1580                 /*
1581                  * Only single packets will be placed the buffers in the OUT
1582                  * direction.
1583                  */
1584                 ep->bytes_per_buffer = max;
1585         }
1586
1587         epctrl = (max << GR_EPCTRL_MAXPL_POS)
1588                 | (nt << GR_EPCTRL_NT_POS)
1589                 | (mode << GR_EPCTRL_TT_POS)
1590                 | GR_EPCTRL_EV;
1591         if (ep->is_in)
1592                 epctrl |= GR_EPCTRL_PI;
1593         gr_write32(&ep->regs->epctrl, epctrl);
1594
1595         gr_write32(&ep->regs->dmactrl, GR_DMACTRL_IE | GR_DMACTRL_AI);
1596
1597         spin_unlock(&ep->dev->lock);
1598
1599         dev_dbg(ep->dev->dev, "EP: %s enabled - %s with %d bytes/buffer\n",
1600                 ep->ep.name, gr_modestring[mode], ep->bytes_per_buffer);
1601         return 0;
1602 }
1603
1604 /* Disable endpoint. Not for ep0in and ep0out that are handled separately. */
1605 static int gr_ep_disable(struct usb_ep *_ep)
1606 {
1607         struct gr_ep *ep;
1608         struct gr_udc *dev;
1609         unsigned long flags;
1610
1611         ep = container_of(_ep, struct gr_ep, ep);
1612         if (!_ep || !ep->ep.desc)
1613                 return -ENODEV;
1614
1615         dev = ep->dev;
1616
1617         /* 'ep0' IN and OUT are reserved */
1618         if (ep == &dev->epo[0] || ep == &dev->epi[0])
1619                 return -EINVAL;
1620
1621         if (dev->ep0state == GR_EP0_SUSPEND)
1622                 return -EBUSY;
1623
1624         dev_dbg(ep->dev->dev, "EP: disable %s\n", ep->ep.name);
1625
1626         spin_lock_irqsave(&dev->lock, flags);
1627
1628         gr_ep_nuke(ep);
1629         gr_ep_reset(ep);
1630         ep->ep.desc = NULL;
1631
1632         spin_unlock_irqrestore(&dev->lock, flags);
1633
1634         return 0;
1635 }
1636
1637 /*
1638  * Frees a request, but not any DMA buffers associated with it
1639  * (gr_finish_request should already have taken care of that).
1640  */
1641 static void gr_free_request(struct usb_ep *_ep, struct usb_request *_req)
1642 {
1643         struct gr_request *req;
1644
1645         if (!_ep || !_req)
1646                 return;
1647         req = container_of(_req, struct gr_request, req);
1648
1649         /* Leads to memory leak */
1650         WARN(!list_empty(&req->queue),
1651              "request not dequeued properly before freeing\n");
1652
1653         kfree(req);
1654 }
1655
1656 /* Queue a request from the gadget */
1657 static int gr_queue_ext(struct usb_ep *_ep, struct usb_request *_req,
1658                         gfp_t gfp_flags)
1659 {
1660         struct gr_ep *ep;
1661         struct gr_request *req;
1662         struct gr_udc *dev;
1663         int ret;
1664
1665         if (unlikely(!_ep || !_req))
1666                 return -EINVAL;
1667
1668         ep = container_of(_ep, struct gr_ep, ep);
1669         req = container_of(_req, struct gr_request, req);
1670         dev = ep->dev;
1671
1672         spin_lock(&ep->dev->lock);
1673
1674         /*
1675          * The ep0 pointer in the gadget struct is used both for ep0in and
1676          * ep0out. In a data stage in the out direction ep0out needs to be used
1677          * instead of the default ep0in. Completion functions might use
1678          * driver_data, so that needs to be copied as well.
1679          */
1680         if ((ep == &dev->epi[0]) && (dev->ep0state == GR_EP0_ODATA)) {
1681                 ep = &dev->epo[0];
1682                 ep->ep.driver_data = dev->epi[0].ep.driver_data;
1683         }
1684
1685         if (ep->is_in)
1686                 gr_dbgprint_request("EXTERN", ep, req);
1687
1688         ret = gr_queue(ep, req, gfp_flags);
1689
1690         spin_unlock(&ep->dev->lock);
1691
1692         return ret;
1693 }
1694
1695 /* Dequeue JUST ONE request */
1696 static int gr_dequeue(struct usb_ep *_ep, struct usb_request *_req)
1697 {
1698         struct gr_request *req;
1699         struct gr_ep *ep;
1700         struct gr_udc *dev;
1701         int ret = 0;
1702         unsigned long flags;
1703
1704         ep = container_of(_ep, struct gr_ep, ep);
1705         if (!_ep || !_req || (!ep->ep.desc && ep->num != 0))
1706                 return -EINVAL;
1707         dev = ep->dev;
1708         if (!dev->driver)
1709                 return -ESHUTDOWN;
1710
1711         /* We can't touch (DMA) registers when suspended */
1712         if (dev->ep0state == GR_EP0_SUSPEND)
1713                 return -EBUSY;
1714
1715         spin_lock_irqsave(&dev->lock, flags);
1716
1717         /* Make sure it's actually queued on this endpoint */
1718         list_for_each_entry(req, &ep->queue, queue) {
1719                 if (&req->req == _req)
1720                         break;
1721         }
1722         if (&req->req != _req) {
1723                 ret = -EINVAL;
1724                 goto out;
1725         }
1726
1727         if (list_first_entry(&ep->queue, struct gr_request, queue) == req) {
1728                 /* This request is currently being processed */
1729                 gr_abort_dma(ep);
1730                 if (ep->stopped)
1731                         gr_finish_request(ep, req, -ECONNRESET);
1732                 else
1733                         gr_dma_advance(ep, -ECONNRESET);
1734         } else if (!list_empty(&req->queue)) {
1735                 /* Not being processed - gr_finish_request dequeues it */
1736                 gr_finish_request(ep, req, -ECONNRESET);
1737         } else {
1738                 ret = -EOPNOTSUPP;
1739         }
1740
1741 out:
1742         spin_unlock_irqrestore(&dev->lock, flags);
1743
1744         return ret;
1745 }
1746
1747 /* Helper for gr_set_halt and gr_set_wedge */
1748 static int gr_set_halt_wedge(struct usb_ep *_ep, int halt, int wedge)
1749 {
1750         int ret;
1751         struct gr_ep *ep;
1752
1753         if (!_ep)
1754                 return -ENODEV;
1755         ep = container_of(_ep, struct gr_ep, ep);
1756
1757         spin_lock(&ep->dev->lock);
1758
1759         /* Halting an IN endpoint should fail if queue is not empty */
1760         if (halt && ep->is_in && !list_empty(&ep->queue)) {
1761                 ret = -EAGAIN;
1762                 goto out;
1763         }
1764
1765         ret = gr_ep_halt_wedge(ep, halt, wedge, 0);
1766
1767 out:
1768         spin_unlock(&ep->dev->lock);
1769
1770         return ret;
1771 }
1772
1773 /* Halt endpoint */
1774 static int gr_set_halt(struct usb_ep *_ep, int halt)
1775 {
1776         return gr_set_halt_wedge(_ep, halt, 0);
1777 }
1778
1779 /* Halt and wedge endpoint */
1780 static int gr_set_wedge(struct usb_ep *_ep)
1781 {
1782         return gr_set_halt_wedge(_ep, 1, 1);
1783 }
1784
1785 /*
1786  * Return the total number of bytes currently stored in the internal buffers of
1787  * the endpoint.
1788  */
1789 static int gr_fifo_status(struct usb_ep *_ep)
1790 {
1791         struct gr_ep *ep;
1792         u32 epstat;
1793         u32 bytes = 0;
1794
1795         if (!_ep)
1796                 return -ENODEV;
1797         ep = container_of(_ep, struct gr_ep, ep);
1798
1799         epstat = gr_read32(&ep->regs->epstat);
1800
1801         if (epstat & GR_EPSTAT_B0)
1802                 bytes += (epstat & GR_EPSTAT_B0CNT_MASK) >> GR_EPSTAT_B0CNT_POS;
1803         if (epstat & GR_EPSTAT_B1)
1804                 bytes += (epstat & GR_EPSTAT_B1CNT_MASK) >> GR_EPSTAT_B1CNT_POS;
1805
1806         return bytes;
1807 }
1808
1809
1810 /* Empty data from internal buffers of an endpoint. */
1811 static void gr_fifo_flush(struct usb_ep *_ep)
1812 {
1813         struct gr_ep *ep;
1814         u32 epctrl;
1815
1816         if (!_ep)
1817                 return;
1818         ep = container_of(_ep, struct gr_ep, ep);
1819         dev_vdbg(ep->dev->dev, "EP: flush fifo %s\n", ep->ep.name);
1820
1821         spin_lock(&ep->dev->lock);
1822
1823         epctrl = gr_read32(&ep->regs->epctrl);
1824         epctrl |= GR_EPCTRL_CB;
1825         gr_write32(&ep->regs->epctrl, epctrl);
1826
1827         spin_unlock(&ep->dev->lock);
1828 }
1829
1830 static struct usb_ep_ops gr_ep_ops = {
1831         .enable         = gr_ep_enable,
1832         .disable        = gr_ep_disable,
1833
1834         .alloc_request  = gr_alloc_request,
1835         .free_request   = gr_free_request,
1836
1837         .queue          = gr_queue_ext,
1838         .dequeue        = gr_dequeue,
1839
1840         .set_halt       = gr_set_halt,
1841         .set_wedge      = gr_set_wedge,
1842         .fifo_status    = gr_fifo_status,
1843         .fifo_flush     = gr_fifo_flush,
1844 };
1845
1846 /* ---------------------------------------------------------------------- */
1847 /* USB Gadget ops */
1848
1849 static int gr_get_frame(struct usb_gadget *_gadget)
1850 {
1851         struct gr_udc *dev;
1852
1853         if (!_gadget)
1854                 return -ENODEV;
1855         dev = container_of(_gadget, struct gr_udc, gadget);
1856         return gr_read32(&dev->regs->status) & GR_STATUS_FN_MASK;
1857 }
1858
1859 static int gr_wakeup(struct usb_gadget *_gadget)
1860 {
1861         struct gr_udc *dev;
1862
1863         if (!_gadget)
1864                 return -ENODEV;
1865         dev = container_of(_gadget, struct gr_udc, gadget);
1866
1867         /* Remote wakeup feature not enabled by host*/
1868         if (!dev->remote_wakeup)
1869                 return -EINVAL;
1870
1871         spin_lock(&dev->lock);
1872
1873         gr_write32(&dev->regs->control,
1874                    gr_read32(&dev->regs->control) | GR_CONTROL_RW);
1875
1876         spin_unlock(&dev->lock);
1877
1878         return 0;
1879 }
1880
1881 static int gr_pullup(struct usb_gadget *_gadget, int is_on)
1882 {
1883         struct gr_udc *dev;
1884         u32 control;
1885
1886         if (!_gadget)
1887                 return -ENODEV;
1888         dev = container_of(_gadget, struct gr_udc, gadget);
1889
1890         spin_lock(&dev->lock);
1891
1892         control = gr_read32(&dev->regs->control);
1893         if (is_on)
1894                 control |= GR_CONTROL_EP;
1895         else
1896                 control &= ~GR_CONTROL_EP;
1897         gr_write32(&dev->regs->control, control);
1898
1899         spin_unlock(&dev->lock);
1900
1901         return 0;
1902 }
1903
1904 static int gr_udc_start(struct usb_gadget *gadget,
1905                         struct usb_gadget_driver *driver)
1906 {
1907         struct gr_udc *dev = to_gr_udc(gadget);
1908
1909         spin_lock(&dev->lock);
1910
1911         /* Hook up the driver */
1912         driver->driver.bus = NULL;
1913         dev->driver = driver;
1914
1915         /* Get ready for host detection */
1916         gr_enable_vbus_detect(dev);
1917
1918         spin_unlock(&dev->lock);
1919
1920         dev_info(dev->dev, "Started with gadget driver '%s'\n",
1921                  driver->driver.name);
1922
1923         return 0;
1924 }
1925
1926 static int gr_udc_stop(struct usb_gadget *gadget,
1927                        struct usb_gadget_driver *driver)
1928 {
1929         struct gr_udc *dev = to_gr_udc(gadget);
1930         unsigned long flags;
1931
1932         spin_lock_irqsave(&dev->lock, flags);
1933
1934         dev->driver = NULL;
1935         gr_stop_activity(dev);
1936
1937         spin_unlock_irqrestore(&dev->lock, flags);
1938
1939         dev_info(dev->dev, "Stopped\n");
1940
1941         return 0;
1942 }
1943
1944 static const struct usb_gadget_ops gr_ops = {
1945         .get_frame      = gr_get_frame,
1946         .wakeup         = gr_wakeup,
1947         .pullup         = gr_pullup,
1948         .udc_start      = gr_udc_start,
1949         .udc_stop       = gr_udc_stop,
1950         /* Other operations not supported */
1951 };
1952
1953 /* ---------------------------------------------------------------------- */
1954 /* Module probe, removal and of-matching */
1955
1956 static const char * const onames[] = {
1957         "ep0out", "ep1out", "ep2out", "ep3out", "ep4out", "ep5out",
1958         "ep6out", "ep7out", "ep8out", "ep9out", "ep10out", "ep11out",
1959         "ep12out", "ep13out", "ep14out", "ep15out"
1960 };
1961
1962 static const char * const inames[] = {
1963         "ep0in", "ep1in", "ep2in", "ep3in", "ep4in", "ep5in",
1964         "ep6in", "ep7in", "ep8in", "ep9in", "ep10in", "ep11in",
1965         "ep12in", "ep13in", "ep14in", "ep15in"
1966 };
1967
1968 /* Must be called with dev->lock held */
1969 static int gr_ep_init(struct gr_udc *dev, int num, int is_in, u32 maxplimit)
1970 {
1971         struct gr_ep *ep;
1972         struct gr_request *req;
1973         struct usb_request *_req;
1974         void *buf;
1975
1976         if (is_in) {
1977                 ep = &dev->epi[num];
1978                 ep->ep.name = inames[num];
1979                 ep->regs = &dev->regs->epi[num];
1980         } else {
1981                 ep = &dev->epo[num];
1982                 ep->ep.name = onames[num];
1983                 ep->regs = &dev->regs->epo[num];
1984         }
1985
1986         gr_ep_reset(ep);
1987         ep->num = num;
1988         ep->is_in = is_in;
1989         ep->dev = dev;
1990         ep->ep.ops = &gr_ep_ops;
1991         INIT_LIST_HEAD(&ep->queue);
1992
1993         if (num == 0) {
1994                 _req = gr_alloc_request(&ep->ep, GFP_KERNEL);
1995                 buf = devm_kzalloc(dev->dev, PAGE_SIZE, GFP_DMA | GFP_KERNEL);
1996                 if (!_req || !buf) {
1997                         /* possible _req freed by gr_probe via gr_remove */
1998                         return -ENOMEM;
1999                 }
2000
2001                 req = container_of(_req, struct gr_request, req);
2002                 req->req.buf = buf;
2003                 req->req.length = MAX_CTRL_PL_SIZE;
2004
2005                 if (is_in)
2006                         dev->ep0reqi = req; /* Complete gets set as used */
2007                 else
2008                         dev->ep0reqo = req; /* Completion treated separately */
2009
2010                 usb_ep_set_maxpacket_limit(&ep->ep, MAX_CTRL_PL_SIZE);
2011                 ep->bytes_per_buffer = MAX_CTRL_PL_SIZE;
2012         } else {
2013                 usb_ep_set_maxpacket_limit(&ep->ep, (u16)maxplimit);
2014                 list_add_tail(&ep->ep.ep_list, &dev->gadget.ep_list);
2015         }
2016         list_add_tail(&ep->ep_list, &dev->ep_list);
2017
2018         return 0;
2019 }
2020
2021 /* Must be called with dev->lock held */
2022 static int gr_udc_init(struct gr_udc *dev)
2023 {
2024         struct device_node *np = dev->dev->of_node;
2025         u32 epctrl_val;
2026         u32 dmactrl_val;
2027         int i;
2028         int ret = 0;
2029         u32 *bufsizes;
2030         u32 bufsize;
2031         int len;
2032
2033         gr_set_address(dev, 0);
2034
2035         INIT_LIST_HEAD(&dev->gadget.ep_list);
2036         dev->gadget.speed = USB_SPEED_UNKNOWN;
2037         dev->gadget.ep0 = &dev->epi[0].ep;
2038
2039         INIT_LIST_HEAD(&dev->ep_list);
2040         gr_set_ep0state(dev, GR_EP0_DISCONNECT);
2041
2042         bufsizes = (u32 *)of_get_property(np, "epobufsizes", &len);
2043         len /= sizeof(u32);
2044         for (i = 0; i < dev->nepo; i++) {
2045                 bufsize = (bufsizes && i < len) ? bufsizes[i] : 1024;
2046                 ret = gr_ep_init(dev, i, 0, bufsize);
2047                 if (ret)
2048                         return ret;
2049         }
2050
2051         bufsizes = (u32 *)of_get_property(np, "epibufsizes", &len);
2052         len /= sizeof(u32);
2053         for (i = 0; i < dev->nepi; i++) {
2054                 bufsize = (bufsizes && i < len) ? bufsizes[i] : 1024;
2055                 ret = gr_ep_init(dev, i, 1, bufsize);
2056                 if (ret)
2057                         return ret;
2058         }
2059
2060         /* Must be disabled by default */
2061         dev->remote_wakeup = 0;
2062
2063         /* Enable ep0out and ep0in */
2064         epctrl_val = (MAX_CTRL_PL_SIZE << GR_EPCTRL_MAXPL_POS) | GR_EPCTRL_EV;
2065         dmactrl_val = GR_DMACTRL_IE | GR_DMACTRL_AI;
2066         gr_write32(&dev->epo[0].regs->epctrl, epctrl_val);
2067         gr_write32(&dev->epi[0].regs->epctrl, epctrl_val | GR_EPCTRL_PI);
2068         gr_write32(&dev->epo[0].regs->dmactrl, dmactrl_val);
2069         gr_write32(&dev->epi[0].regs->dmactrl, dmactrl_val);
2070
2071         return 0;
2072 }
2073
2074 static int gr_remove(struct platform_device *ofdev)
2075 {
2076         struct gr_udc *dev = dev_get_drvdata(&ofdev->dev);
2077
2078         if (dev->added)
2079                 usb_del_gadget_udc(&dev->gadget); /* Shuts everything down */
2080         if (dev->driver)
2081                 return -EBUSY;
2082
2083         gr_dfs_delete(dev);
2084         if (dev->desc_pool)
2085                 dma_pool_destroy(dev->desc_pool);
2086         dev_set_drvdata(&ofdev->dev, NULL);
2087
2088         gr_free_request(&dev->epi[0].ep, &dev->ep0reqi->req);
2089         gr_free_request(&dev->epo[0].ep, &dev->ep0reqo->req);
2090
2091         return 0;
2092 }
2093 static int gr_request_irq(struct gr_udc *dev, int irq)
2094 {
2095         return devm_request_threaded_irq(dev->dev, irq, gr_irq, gr_irq_handler,
2096                                          IRQF_SHARED, driver_name, dev);
2097 }
2098
2099 static int gr_probe(struct platform_device *ofdev)
2100 {
2101         struct gr_udc *dev;
2102         struct resource *res;
2103         struct gr_regs __iomem *regs;
2104         int retval;
2105         u32 status;
2106
2107         dev = devm_kzalloc(&ofdev->dev, sizeof(*dev), GFP_KERNEL);
2108         if (!dev)
2109                 return -ENOMEM;
2110         dev->dev = &ofdev->dev;
2111
2112         res = platform_get_resource(ofdev, IORESOURCE_MEM, 0);
2113         regs = devm_ioremap_resource(dev->dev, res);
2114         if (IS_ERR(regs))
2115                 return PTR_ERR(regs);
2116
2117         dev->irq = irq_of_parse_and_map(dev->dev->of_node, 0);
2118         if (!dev->irq) {
2119                 dev_err(dev->dev, "No irq found\n");
2120                 return -ENODEV;
2121         }
2122
2123         /* Some core configurations has separate irqs for IN and OUT events */
2124         dev->irqi = irq_of_parse_and_map(dev->dev->of_node, 1);
2125         if (dev->irqi) {
2126                 dev->irqo = irq_of_parse_and_map(dev->dev->of_node, 2);
2127                 if (!dev->irqo) {
2128                         dev_err(dev->dev, "Found irqi but not irqo\n");
2129                         return -ENODEV;
2130                 }
2131         }
2132
2133         dev->gadget.name = driver_name;
2134         dev->gadget.max_speed = USB_SPEED_HIGH;
2135         dev->gadget.ops = &gr_ops;
2136         dev->gadget.quirk_ep_out_aligned_size = true;
2137
2138         spin_lock_init(&dev->lock);
2139         dev->regs = regs;
2140
2141         dev_set_drvdata(&ofdev->dev, dev);
2142
2143         /* Determine number of endpoints and data interface mode */
2144         status = gr_read32(&dev->regs->status);
2145         dev->nepi = ((status & GR_STATUS_NEPI_MASK) >> GR_STATUS_NEPI_POS) + 1;
2146         dev->nepo = ((status & GR_STATUS_NEPO_MASK) >> GR_STATUS_NEPO_POS) + 1;
2147
2148         if (!(status & GR_STATUS_DM)) {
2149                 dev_err(dev->dev, "Slave mode cores are not supported\n");
2150                 return -ENODEV;
2151         }
2152
2153         /* --- Effects of the following calls might need explicit cleanup --- */
2154
2155         /* Create DMA pool for descriptors */
2156         dev->desc_pool = dma_pool_create("desc_pool", dev->dev,
2157                                          sizeof(struct gr_dma_desc), 4, 0);
2158         if (!dev->desc_pool) {
2159                 dev_err(dev->dev, "Could not allocate DMA pool");
2160                 return -ENOMEM;
2161         }
2162
2163         spin_lock(&dev->lock);
2164
2165         /* Inside lock so that no gadget can use this udc until probe is done */
2166         retval = usb_add_gadget_udc(dev->dev, &dev->gadget);
2167         if (retval) {
2168                 dev_err(dev->dev, "Could not add gadget udc");
2169                 goto out;
2170         }
2171         dev->added = 1;
2172
2173         retval = gr_udc_init(dev);
2174         if (retval)
2175                 goto out;
2176
2177         gr_dfs_create(dev);
2178
2179         /* Clear all interrupt enables that might be left on since last boot */
2180         gr_disable_interrupts_and_pullup(dev);
2181
2182         retval = gr_request_irq(dev, dev->irq);
2183         if (retval) {
2184                 dev_err(dev->dev, "Failed to request irq %d\n", dev->irq);
2185                 goto out;
2186         }
2187
2188         if (dev->irqi) {
2189                 retval = gr_request_irq(dev, dev->irqi);
2190                 if (retval) {
2191                         dev_err(dev->dev, "Failed to request irqi %d\n",
2192                                 dev->irqi);
2193                         goto out;
2194                 }
2195                 retval = gr_request_irq(dev, dev->irqo);
2196                 if (retval) {
2197                         dev_err(dev->dev, "Failed to request irqo %d\n",
2198                                 dev->irqo);
2199                         goto out;
2200                 }
2201         }
2202
2203         if (dev->irqi)
2204                 dev_info(dev->dev, "regs: %p, irqs %d, %d, %d\n", dev->regs,
2205                          dev->irq, dev->irqi, dev->irqo);
2206         else
2207                 dev_info(dev->dev, "regs: %p, irq %d\n", dev->regs, dev->irq);
2208
2209 out:
2210         spin_unlock(&dev->lock);
2211
2212         if (retval)
2213                 gr_remove(ofdev);
2214
2215         return retval;
2216 }
2217
2218 static struct of_device_id gr_match[] = {
2219         {.name = "GAISLER_USBDC"},
2220         {.name = "01_021"},
2221         {},
2222 };
2223 MODULE_DEVICE_TABLE(of, gr_match);
2224
2225 static struct platform_driver gr_driver = {
2226         .driver = {
2227                 .name = DRIVER_NAME,
2228                 .owner = THIS_MODULE,
2229                 .of_match_table = gr_match,
2230         },
2231         .probe = gr_probe,
2232         .remove = gr_remove,
2233 };
2234 module_platform_driver(gr_driver);
2235
2236 MODULE_AUTHOR("Aeroflex Gaisler AB.");
2237 MODULE_DESCRIPTION(DRIVER_DESC);
2238 MODULE_LICENSE("GPL");