drivers/staging/octeon-usb/octeon-hcd.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 2008 Cavium Networks
   7  *
   8  * Some parts of the code were originally released under BSD license:
   9  *
  10  * Copyright (c) 2003-2010 Cavium Networks (support@cavium.com). All rights
  11  * reserved.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions are
  15  * met:
  16  *
  17  *   * Redistributions of source code must retain the above copyright
  18  *     notice, this list of conditions and the following disclaimer.
  19  *
  20  *   * Redistributions in binary form must reproduce the above
  21  *     copyright notice, this list of conditions and the following
  22  *     disclaimer in the documentation and/or other materials provided
  23  *     with the distribution.
  24  *
  25  *   * Neither the name of Cavium Networks nor the names of
  26  *     its contributors may be used to endorse or promote products
  27  *     derived from this software without specific prior written
  28  *     permission.
  29  *
  30  * This Software, including technical data, may be subject to U.S. export
  31  * control laws, including the U.S. Export Administration Act and its associated
  32  * regulations, and may be subject to export or import regulations in other
  33  * countries.
  34  *
  35  * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
  36  * AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR
  37  * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
  38  * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION
  39  * OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
  40  * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
  41  * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
  42  * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
  43  * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
  44  * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
  45  */
  46 #include <linux/kernel.h>
  47 #include <linux/module.h>
  48 #include <linux/init.h>
  49 #include <linux/pci.h>
  50 #include <linux/interrupt.h>
  51 #include <linux/platform_device.h>
  52 #include <linux/usb.h>
  53
  54 #include <linux/time.h>
  55 #include <linux/delay.h>
  56
  57 #include <asm/octeon/cvmx.h>
  58 #include <asm/octeon/cvmx-iob-defs.h>
  59
  60 #include <linux/usb/hcd.h>
  61
  62 #include <linux/err.h>
  63
  64 #include <asm/octeon/octeon.h>
  65 #include <asm/octeon/cvmx-helper.h>
  66 #include <asm/octeon/cvmx-sysinfo.h>
  67 #include <asm/octeon/cvmx-helper-board.h>
  68
  69 #include "octeon-hcd.h"
  70
  71 /**
  72  * enum cvmx_usb_speed - the possible USB device speeds
  73  *
  74  * @CVMX_USB_SPEED_HIGH: Device is operation at 480Mbps
  75  * @CVMX_USB_SPEED_FULL: Device is operation at 12Mbps
  76  * @CVMX_USB_SPEED_LOW:  Device is operation at 1.5Mbps
  77  */
  78 enum cvmx_usb_speed {
  79         CVMX_USB_SPEED_HIGH = 0,
  80         CVMX_USB_SPEED_FULL = 1,
  81         CVMX_USB_SPEED_LOW = 2,
  82 };
  83
  84 /**
  85  * enum cvmx_usb_transfer - the possible USB transfer types
  86  *
  87  * @CVMX_USB_TRANSFER_CONTROL:     USB transfer type control for hub and status
  88  *                                 transfers
  89  * @CVMX_USB_TRANSFER_ISOCHRONOUS: USB transfer type isochronous for low
  90  *                                 priority periodic transfers
  91  * @CVMX_USB_TRANSFER_BULK:        USB transfer type bulk for large low priority
  92  *                                 transfers
  93  * @CVMX_USB_TRANSFER_INTERRUPT:   USB transfer type interrupt for high priority
  94  *                                 periodic transfers
  95  */
  96 enum cvmx_usb_transfer {
  97         CVMX_USB_TRANSFER_CONTROL = 0,
  98         CVMX_USB_TRANSFER_ISOCHRONOUS = 1,
  99         CVMX_USB_TRANSFER_BULK = 2,
 100         CVMX_USB_TRANSFER_INTERRUPT = 3,
 101 };
 102
 103 /**
 104  * enum cvmx_usb_direction - the transfer directions
 105  *
 106  * @CVMX_USB_DIRECTION_OUT: Data is transferring from Octeon to the device/host
 107  * @CVMX_USB_DIRECTION_IN:  Data is transferring from the device/host to Octeon
 108  */
 109 enum cvmx_usb_direction {
 110         CVMX_USB_DIRECTION_OUT,
 111         CVMX_USB_DIRECTION_IN,
 112 };
 113
 114 /**
 115  * enum cvmx_usb_complete - possible callback function status codes
 116  *
 117  * @CVMX_USB_COMPLETE_SUCCESS:    The transaction / operation finished without
 118  *                                any errors
 119  * @CVMX_USB_COMPLETE_SHORT:      FIXME: This is currently not implemented
 120  * @CVMX_USB_COMPLETE_CANCEL:     The transaction was canceled while in flight
 121  *                                by a user call to cvmx_usb_cancel
 122  * @CVMX_USB_COMPLETE_ERROR:      The transaction aborted with an unexpected
 123  *                                error status
 124  * @CVMX_USB_COMPLETE_STALL:      The transaction received a USB STALL response
 125  *                                from the device
 126  * @CVMX_USB_COMPLETE_XACTERR:    The transaction failed with an error from the
 127  *                                device even after a number of retries
 128  * @CVMX_USB_COMPLETE_DATATGLERR: The transaction failed with a data toggle
 129  *                                error even after a number of retries
 130  * @CVMX_USB_COMPLETE_BABBLEERR:  The transaction failed with a babble error
 131  * @CVMX_USB_COMPLETE_FRAMEERR:   The transaction failed with a frame error
 132  *                                even after a number of retries
 133  */
 134 enum cvmx_usb_complete {
 135         CVMX_USB_COMPLETE_SUCCESS,
 136         CVMX_USB_COMPLETE_SHORT,
 137         CVMX_USB_COMPLETE_CANCEL,
 138         CVMX_USB_COMPLETE_ERROR,
 139         CVMX_USB_COMPLETE_STALL,
 140         CVMX_USB_COMPLETE_XACTERR,
 141         CVMX_USB_COMPLETE_DATATGLERR,
 142         CVMX_USB_COMPLETE_BABBLEERR,
 143         CVMX_USB_COMPLETE_FRAMEERR,
 144 };
 145
 146 /**
 147  * struct cvmx_usb_port_status - the USB port status information
 148  *
 149  * @port_enabled:       1 = Usb port is enabled, 0 = disabled
 150  * @port_over_current:  1 = Over current detected, 0 = Over current not
 151  *                      detected. Octeon doesn't support over current detection.
 152  * @port_powered:       1 = Port power is being supplied to the device, 0 =
 153  *                      power is off. Octeon doesn't support turning port power
 154  *                      off.
 155  * @port_speed:         Current port speed.
 156  * @connected:          1 = A device is connected to the port, 0 = No device is
 157  *                      connected.
 158  * @connect_change:     1 = Device connected state changed since the last set
 159  *                      status call.
 160  */
 161 struct cvmx_usb_port_status {
 162         uint32_t reserved               : 25;
 163         uint32_t port_enabled           : 1;
 164         uint32_t port_over_current      : 1;
 165         uint32_t port_powered           : 1;
 166         enum cvmx_usb_speed port_speed  : 2;
 167         uint32_t connected              : 1;
 168         uint32_t connect_change         : 1;
 169 };
 170
 171 /**
 172  * union cvmx_usb_control_header - the structure of a Control packet header
 173  *
 174  * @s.request_type:     Bit 7 tells the direction: 1=IN, 0=OUT
 175  * @s.request           The standard usb request to make
 176  * @s.value             Value parameter for the request in little endian format
 177  * @s.index             Index for the request in little endian format
 178  * @s.length            Length of the data associated with this request in
 179  *                      little endian format
 180  */
 181 union cvmx_usb_control_header {
 182         uint64_t u64;
 183         struct {
 184                 uint64_t request_type   : 8;
 185                 uint64_t request        : 8;
 186                 uint64_t value          : 16;
 187                 uint64_t index          : 16;
 188                 uint64_t length         : 16;
 189         } s;
 190 };
 191
 192 /**
 193  * struct cvmx_usb_iso_packet - descriptor for Isochronous packets
 194  *
 195  * @offset:     This is the offset in bytes into the main buffer where this data
 196  *              is stored.
 197  * @length:     This is the length in bytes of the data.
 198  * @status:     This is the status of this individual packet transfer.
 199  */
 200 struct cvmx_usb_iso_packet {
 201         int offset;
 202         int length;
 203         enum cvmx_usb_complete status;
 204 };
 205
 206 /**
 207  * enum cvmx_usb_initialize_flags - flags used by the initialization function
 208  *
 209  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI:    The USB port uses a 12MHz crystal
 210  *                                            as clock source at USB_XO and
 211  *                                            USB_XI.
 212  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND:   The USB port uses 12/24/48MHz 2.5V
 213  *                                            board clock source at USB_XO.
 214  *                                            USB_XI should be tied to GND.
 215  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK: Mask for clock speed field
 216  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:    Speed of reference clock or
 217  *                                            crystal
 218  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:    Speed of reference clock
 219  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:    Speed of reference clock
 220  * @CVMX_USB_INITIALIZE_FLAGS_NO_DMA:         Disable DMA and used polled IO for
 221  *                                            data transfer use for the USB
 222  */
 223 enum cvmx_usb_initialize_flags {
 224         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI           = 1 << 0,
 225         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND          = 1 << 1,
 226         CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK        = 3 << 3,
 227         CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ           = 1 << 3,
 228         CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ           = 2 << 3,
 229         CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ           = 3 << 3,
 230         /* Bits 3-4 used to encode the clock frequency */
 231         CVMX_USB_INITIALIZE_FLAGS_NO_DMA                = 1 << 5,
 232 };
 233
 234 /**
 235  * enum cvmx_usb_pipe_flags - internal flags for a pipe.
 236  *
 237  * @__CVMX_USB_PIPE_FLAGS_SCHEDULED: Used internally to determine if a pipe is
 238  *                                   actively using hardware. Do not use.
 239  * @__CVMX_USB_PIPE_FLAGS_NEED_PING: Used internally to determine if a high
 240  *                                   speed pipe is in the ping state. Do not
 241  *                                   use.
 242  */
 243 enum cvmx_usb_pipe_flags {
 244         __CVMX_USB_PIPE_FLAGS_SCHEDULED = 1 << 17,
 245         __CVMX_USB_PIPE_FLAGS_NEED_PING = 1 << 18,
 246 };
 247
 248 /* Normal prefetch that use the pref instruction. */
 249 #define CVMX_PREFETCH(address, offset) asm volatile ("pref %[type], %[off](%[rbase])" : : [rbase] "d" (address), [off] "I" (offset), [type] "n" (0))
 250
 251 /* Maximum number of times to retry failed transactions */
 252 #define MAX_RETRIES             3
 253
 254 /* Maximum number of hardware channels supported by the USB block */
 255 #define MAX_CHANNELS            8
 256
 257 /* The highest valid USB device address */
 258 #define MAX_USB_ADDRESS         127
 259
 260 /* The highest valid USB endpoint number */
 261 #define MAX_USB_ENDPOINT        15
 262
 263 /* The highest valid port number on a hub */
 264 #define MAX_USB_HUB_PORT        15
 265
 266 /*
 267  * The low level hardware can transfer a maximum of this number of bytes in each
 268  * transfer. The field is 19 bits wide
 269  */
 270 #define MAX_TRANSFER_BYTES      ((1<<19)-1)
 271
 272 /*
 273  * The low level hardware can transfer a maximum of this number of packets in
 274  * each transfer. The field is 10 bits wide
 275  */
 276 #define MAX_TRANSFER_PACKETS    ((1<<10)-1)
 277
 278 enum {
 279         USB_CLOCK_TYPE_REF_12,
 280         USB_CLOCK_TYPE_REF_24,
 281         USB_CLOCK_TYPE_REF_48,
 282         USB_CLOCK_TYPE_CRYSTAL_12,
 283 };
 284
 285 /**
 286  * Logical transactions may take numerous low level
 287  * transactions, especially when splits are concerned. This
 288  * enum represents all of the possible stages a transaction can
 289  * be in. Note that split completes are always even. This is so
 290  * the NAK handler can backup to the previous low level
 291  * transaction with a simple clearing of bit 0.
 292  */
 293 enum cvmx_usb_stage {
 294         CVMX_USB_STAGE_NON_CONTROL,
 295         CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE,
 296         CVMX_USB_STAGE_SETUP,
 297         CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE,
 298         CVMX_USB_STAGE_DATA,
 299         CVMX_USB_STAGE_DATA_SPLIT_COMPLETE,
 300         CVMX_USB_STAGE_STATUS,
 301         CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE,
 302 };
 303
 304 /**
 305  * struct cvmx_usb_transaction - describes each pending USB transaction
 306  *                               regardless of type. These are linked together
 307  *                               to form a list of pending requests for a pipe.
 308  *
 309  * @node:               List node for transactions in the pipe.
 310  * @type:               Type of transaction, duplicated of the pipe.
 311  * @flags:              State flags for this transaction.
 312  * @buffer:             User's physical buffer address to read/write.
 313  * @buffer_length:      Size of the user's buffer in bytes.
 314  * @control_header:     For control transactions, physical address of the 8
 315  *                      byte standard header.
 316  * @iso_start_frame:    For ISO transactions, the starting frame number.
 317  * @iso_number_packets: For ISO transactions, the number of packets in the
 318  *                      request.
 319  * @iso_packets:        For ISO transactions, the sub packets in the request.
 320  * @actual_bytes:       Actual bytes transfer for this transaction.
 321  * @stage:              For control transactions, the current stage.
 322  * @urb:                URB.
 323  */
 324 struct cvmx_usb_transaction {
 325         struct list_head node;
 326         enum cvmx_usb_transfer type;
 327         uint64_t buffer;
 328         int buffer_length;
 329         uint64_t control_header;
 330         int iso_start_frame;
 331         int iso_number_packets;
 332         struct cvmx_usb_iso_packet *iso_packets;
 333         int xfersize;
 334         int pktcnt;
 335         int retries;
 336         int actual_bytes;
 337         enum cvmx_usb_stage stage;
 338         struct urb *urb;
 339 };
 340
 341 /**
 342  * struct cvmx_usb_pipe - a pipe represents a virtual connection between Octeon
 343  *                        and some USB device. It contains a list of pending
 344  *                        request to the device.
 345  *
 346  * @node:               List node for pipe list
 347  * @next:               Pipe after this one in the list
 348  * @transactions:       List of pending transactions
 349  * @interval:           For periodic pipes, the interval between packets in
 350  *                      frames
 351  * @next_tx_frame:      The next frame this pipe is allowed to transmit on
 352  * @flags:              State flags for this pipe
 353  * @device_speed:       Speed of device connected to this pipe
 354  * @transfer_type:      Type of transaction supported by this pipe
 355  * @transfer_dir:       IN or OUT. Ignored for Control
 356  * @multi_count:        Max packet in a row for the device
 357  * @max_packet:         The device's maximum packet size in bytes
 358  * @device_addr:        USB device address at other end of pipe
 359  * @endpoint_num:       USB endpoint number at other end of pipe
 360  * @hub_device_addr:    Hub address this device is connected to
 361  * @hub_port:           Hub port this device is connected to
 362  * @pid_toggle:         This toggles between 0/1 on every packet send to track
 363  *                      the data pid needed
 364  * @channel:            Hardware DMA channel for this pipe
 365  * @split_sc_frame:     The low order bits of the frame number the split
 366  *                      complete should be sent on
 367  */
 368 struct cvmx_usb_pipe {
 369         struct list_head node;
 370         struct list_head transactions;
 371         uint64_t interval;
 372         uint64_t next_tx_frame;
 373         enum cvmx_usb_pipe_flags flags;
 374         enum cvmx_usb_speed device_speed;
 375         enum cvmx_usb_transfer transfer_type;
 376         enum cvmx_usb_direction transfer_dir;
 377         int multi_count;
 378         uint16_t max_packet;
 379         uint8_t device_addr;
 380         uint8_t endpoint_num;
 381         uint8_t hub_device_addr;
 382         uint8_t hub_port;
 383         uint8_t pid_toggle;
 384         uint8_t channel;
 385         int8_t split_sc_frame;
 386 };
 387
 388 struct cvmx_usb_tx_fifo {
 389         struct {
 390                 int channel;
 391                 int size;
 392                 uint64_t address;
 393         } entry[MAX_CHANNELS+1];
 394         int head;
 395         int tail;
 396 };
 397
 398 /**
 399  * struct cvmx_usb_state - the state of the USB block
 400  *
 401  * init_flags:             Flags passed to initialize.
 402  * index:                  Which USB block this is for.
 403  * idle_hardware_channels: Bit set for every idle hardware channel.
 404  * usbcx_hprt:             Stored port status so we don't need to read a CSR to
 405  *                         determine splits.
 406  * pipe_for_channel:       Map channels to pipes.
 407  * pipe:                   Storage for pipes.
 408  * indent:                 Used by debug output to indent functions.
 409  * port_status:            Last port status used for change notification.
 410  * idle_pipes:             List of open pipes that have no transactions.
 411  * active_pipes:           Active pipes indexed by transfer type.
 412  * frame_number:           Increments every SOF interrupt for time keeping.
 413  * active_split:           Points to the current active split, or NULL.
 414  */
 415 struct cvmx_usb_state {
 416         int init_flags;
 417         int index;
 418         int idle_hardware_channels;
 419         union cvmx_usbcx_hprt usbcx_hprt;
 420         struct cvmx_usb_pipe *pipe_for_channel[MAX_CHANNELS];
 421         int indent;
 422         struct cvmx_usb_port_status port_status;
 423         struct list_head idle_pipes;
 424         struct list_head active_pipes[4];
 425         uint64_t frame_number;
 426         struct cvmx_usb_transaction *active_split;
 427         struct cvmx_usb_tx_fifo periodic;
 428         struct cvmx_usb_tx_fifo nonperiodic;
 429 };
 430
 431 struct octeon_hcd {
 432         spinlock_t lock;
 433         struct cvmx_usb_state usb;
 434         struct tasklet_struct dequeue_tasklet;
 435         struct list_head dequeue_list;
 436 };
 437
 438 /* This macro spins on a field waiting for it to reach a value */
 439 #define CVMX_WAIT_FOR_FIELD32(address, type, field, op, value, timeout_usec)\
 440         ({int result;                                                       \
 441         do {                                                                \
 442                 uint64_t done = cvmx_get_cycle() + (uint64_t)timeout_usec * \
 443                         octeon_get_clock_rate() / 1000000;                  \
 444                 type c;                                                     \
 445                 while (1) {                                                 \
 446                         c.u32 = __cvmx_usb_read_csr32(usb, address);        \
 447                         if (c.s.field op (value)) {                         \
 448                                 result = 0;                                 \
 449                                 break;                                      \
 450                         } else if (cvmx_get_cycle() > done) {               \
 451                                 result = -1;                                \
 452                                 break;                                      \
 453                         } else                                              \
 454                                 cvmx_wait(100);                             \
 455                 }                                                           \
 456         } while (0);                                                        \
 457         result; })
 458
 459 /*
 460  * This macro logically sets a single field in a CSR. It does the sequence
 461  * read, modify, and write
 462  */
 463 #define USB_SET_FIELD32(address, type, field, value)            \
 464         do {                                                    \
 465                 type c;                                         \
 466                 c.u32 = __cvmx_usb_read_csr32(usb, address);    \
 467                 c.s.field = value;                              \
 468                 __cvmx_usb_write_csr32(usb, address, c.u32);    \
 469         } while (0)
 470
 471 /* Returns the IO address to push/pop stuff data from the FIFOs */
 472 #define USB_FIFO_ADDRESS(channel, usb_index) (CVMX_USBCX_GOTGCTL(usb_index) + ((channel)+1)*0x1000)
 473
 474 static int octeon_usb_get_clock_type(void)
 475 {
 476         switch (cvmx_sysinfo_get()->board_type) {
 477         case CVMX_BOARD_TYPE_BBGW_REF:
 478         case CVMX_BOARD_TYPE_LANAI2_A:
 479         case CVMX_BOARD_TYPE_LANAI2_U:
 480         case CVMX_BOARD_TYPE_LANAI2_G:
 481         case CVMX_BOARD_TYPE_UBNT_E100:
 482                 return USB_CLOCK_TYPE_CRYSTAL_12;
 483         }
 484         return USB_CLOCK_TYPE_REF_48;
 485 }
 486
 487 /**
 488  * Read a USB 32bit CSR. It performs the necessary address swizzle
 489  * for 32bit CSRs and logs the value in a readable format if
 490  * debugging is on.
 491  *
 492  * @usb:     USB block this access is for
 493  * @address: 64bit address to read
 494  *
 495  * Returns: Result of the read
 496  */
 497 static inline uint32_t __cvmx_usb_read_csr32(struct cvmx_usb_state *usb,
 498                                              uint64_t address)
 499 {
 500         uint32_t result = cvmx_read64_uint32(address ^ 4);
 501         return result;
 502 }
 503
 504
 505 /**
 506  * Write a USB 32bit CSR. It performs the necessary address
 507  * swizzle for 32bit CSRs and logs the value in a readable format
 508  * if debugging is on.
 509  *
 510  * @usb:     USB block this access is for
 511  * @address: 64bit address to write
 512  * @value:   Value to write
 513  */
 514 static inline void __cvmx_usb_write_csr32(struct cvmx_usb_state *usb,
 515                                           uint64_t address, uint32_t value)
 516 {
 517         cvmx_write64_uint32(address ^ 4, value);
 518         cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
 519 }
 520
 521
 522 /**
 523  * Read a USB 64bit CSR. It logs the value in a readable format if
 524  * debugging is on.
 525  *
 526  * @usb:     USB block this access is for
 527  * @address: 64bit address to read
 528  *
 529  * Returns: Result of the read
 530  */
 531 static inline uint64_t __cvmx_usb_read_csr64(struct cvmx_usb_state *usb,
 532                                              uint64_t address)
 533 {
 534         uint64_t result = cvmx_read64_uint64(address);
 535         return result;
 536 }
 537
 538
 539 /**
 540  * Write a USB 64bit CSR. It logs the value in a readable format
 541  * if debugging is on.
 542  *
 543  * @usb:     USB block this access is for
 544  * @address: 64bit address to write
 545  * @value:   Value to write
 546  */
 547 static inline void __cvmx_usb_write_csr64(struct cvmx_usb_state *usb,
 548                                           uint64_t address, uint64_t value)
 549 {
 550         cvmx_write64_uint64(address, value);
 551 }
 552
 553 /**
 554  * Return non zero if this pipe connects to a non HIGH speed
 555  * device through a high speed hub.
 556  *
 557  * @usb:    USB block this access is for
 558  * @pipe:   Pipe to check
 559  *
 560  * Returns: Non zero if we need to do split transactions
 561  */
 562 static inline int __cvmx_usb_pipe_needs_split(struct cvmx_usb_state *usb,
 563                                               struct cvmx_usb_pipe *pipe)
 564 {
 565         return pipe->device_speed != CVMX_USB_SPEED_HIGH &&
 566                usb->usbcx_hprt.s.prtspd == CVMX_USB_SPEED_HIGH;
 567 }
 568
 569
 570 /**
 571  * Trivial utility function to return the correct PID for a pipe
 572  *
 573  * @pipe:   pipe to check
 574  *
 575  * Returns: PID for pipe
 576  */
 577 static inline int __cvmx_usb_get_data_pid(struct cvmx_usb_pipe *pipe)
 578 {
 579         if (pipe->pid_toggle)
 580                 return 2; /* Data1 */
 581         else
 582                 return 0; /* Data0 */
 583 }
 584
 585
 586 /**
 587  * Return the number of USB ports supported by this Octeon
 588  * chip. If the chip doesn't support USB, or is not supported
 589  * by this API, a zero will be returned. Most Octeon chips
 590  * support one usb port, but some support two ports.
 591  * cvmx_usb_initialize() must be called on independent
 592  * struct cvmx_usb_state.
 593  *
 594  * Returns: Number of port, zero if usb isn't supported
 595  */
 596 static int cvmx_usb_get_num_ports(void)
 597 {
 598         int arch_ports = 0;
 599
 600         if (OCTEON_IS_MODEL(OCTEON_CN56XX))
 601                 arch_ports = 1;
 602         else if (OCTEON_IS_MODEL(OCTEON_CN52XX))
 603                 arch_ports = 2;
 604         else if (OCTEON_IS_MODEL(OCTEON_CN50XX))
 605                 arch_ports = 1;
 606         else if (OCTEON_IS_MODEL(OCTEON_CN31XX))
 607                 arch_ports = 1;
 608         else if (OCTEON_IS_MODEL(OCTEON_CN30XX))
 609                 arch_ports = 1;
 610         else
 611                 arch_ports = 0;
 612
 613         return arch_ports;
 614 }
 615
 616 /**
 617  * Initialize a USB port for use. This must be called before any
 618  * other access to the Octeon USB port is made. The port starts
 619  * off in the disabled state.
 620  *
 621  * @usb:         Pointer to an empty struct cvmx_usb_state
 622  *               that will be populated by the initialize call.
 623  *               This structure is then passed to all other USB
 624  *               functions.
 625  * @usb_port_number:
 626  *               Which Octeon USB port to initialize.
 627  *
 628  * Returns: 0 or a negative error code.
 629  */
 630 static int cvmx_usb_initialize(struct cvmx_usb_state *usb,
 631                                int usb_port_number)
 632 {
 633         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 634         union cvmx_usbnx_usbp_ctl_status usbn_usbp_ctl_status;
 635         enum cvmx_usb_initialize_flags flags = 0;
 636         int i;
 637
 638         /* At first allow 0-1 for the usb port number */
 639         if ((usb_port_number < 0) || (usb_port_number > 1))
 640                 return -EINVAL;
 641         /* For all chips except 52XX there is only one port */
 642         if (!OCTEON_IS_MODEL(OCTEON_CN52XX) && (usb_port_number > 0))
 643                 return -EINVAL;
 644         /* Try to determine clock type automatically */
 645         if (octeon_usb_get_clock_type() == USB_CLOCK_TYPE_CRYSTAL_12) {
 646                 /* Only 12 MHZ crystals are supported */
 647                 flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI;
 648         } else {
 649                 flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND;
 650
 651                 switch (octeon_usb_get_clock_type()) {
 652                 case USB_CLOCK_TYPE_REF_12:
 653                         flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ;
 654                         break;
 655                 case USB_CLOCK_TYPE_REF_24:
 656                         flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ;
 657                         break;
 658                 case USB_CLOCK_TYPE_REF_48:
 659                         flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ;
 660                         break;
 661                 default:
 662                         return -EINVAL;
 663                         break;
 664                 }
 665         }
 666
 667         memset(usb, 0, sizeof(*usb));
 668         usb->init_flags = flags;
 669
 670         /* Initialize the USB state structure */
 671         usb->index = usb_port_number;
 672         INIT_LIST_HEAD(&usb->idle_pipes);
 673         for (i = 0; i < ARRAY_SIZE(usb->active_pipes); i++)
 674                 INIT_LIST_HEAD(&usb->active_pipes[i]);
 675
 676         /*
 677          * Power On Reset and PHY Initialization
 678          *
 679          * 1. Wait for DCOK to assert (nothing to do)
 680          *
 681          * 2a. Write USBN0/1_CLK_CTL[POR] = 1 and
 682          *     USBN0/1_CLK_CTL[HRST,PRST,HCLK_RST] = 0
 683          */
 684         usbn_clk_ctl.u64 = __cvmx_usb_read_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index));
 685         usbn_clk_ctl.s.por = 1;
 686         usbn_clk_ctl.s.hrst = 0;
 687         usbn_clk_ctl.s.prst = 0;
 688         usbn_clk_ctl.s.hclk_rst = 0;
 689         usbn_clk_ctl.s.enable = 0;
 690         /*
 691          * 2b. Select the USB reference clock/crystal parameters by writing
 692          *     appropriate values to USBN0/1_CLK_CTL[P_C_SEL, P_RTYPE, P_COM_ON]
 693          */
 694         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND) {
 695                 /*
 696                  * The USB port uses 12/24/48MHz 2.5V board clock
 697                  * source at USB_XO. USB_XI should be tied to GND.
 698                  * Most Octeon evaluation boards require this setting
 699                  */
 700                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX) ||
 701                     OCTEON_IS_MODEL(OCTEON_CN56XX) ||
 702                     OCTEON_IS_MODEL(OCTEON_CN50XX))
 703                         /* From CN56XX,CN50XX,CN31XX,CN30XX manuals */
 704                         usbn_clk_ctl.s.p_rtype = 2; /* p_rclk=1 & p_xenbn=0 */
 705                 else
 706                         /* From CN52XX manual */
 707                         usbn_clk_ctl.s.p_rtype = 1;
 708
 709                 switch (flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK) {
 710                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:
 711                         usbn_clk_ctl.s.p_c_sel = 0;
 712                         break;
 713                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:
 714                         usbn_clk_ctl.s.p_c_sel = 1;
 715                         break;
 716                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:
 717                         usbn_clk_ctl.s.p_c_sel = 2;
 718                         break;
 719                 }
 720         } else {
 721                 /*
 722                  * The USB port uses a 12MHz crystal as clock source
 723                  * at USB_XO and USB_XI
 724                  */
 725                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
 726                         /* From CN31XX,CN30XX manual */
 727                         usbn_clk_ctl.s.p_rtype = 3; /* p_rclk=1 & p_xenbn=1 */
 728                 else
 729                         /* From CN56XX,CN52XX,CN50XX manuals. */
 730                         usbn_clk_ctl.s.p_rtype = 0;
 731
 732                 usbn_clk_ctl.s.p_c_sel = 0;
 733         }
 734         /*
 735          * 2c. Select the HCLK via writing USBN0/1_CLK_CTL[DIVIDE, DIVIDE2] and
 736          *     setting USBN0/1_CLK_CTL[ENABLE] = 1. Divide the core clock down
 737          *     such that USB is as close as possible to 125Mhz
 738          */
 739         {
 740                 int divisor = (octeon_get_clock_rate()+125000000-1)/125000000;
 741                 /* Lower than 4 doesn't seem to work properly */
 742                 if (divisor < 4)
 743                         divisor = 4;
 744                 usbn_clk_ctl.s.divide = divisor;
 745                 usbn_clk_ctl.s.divide2 = 0;
 746         }
 747         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 748                                usbn_clk_ctl.u64);
 749         /* 2d. Write USBN0/1_CLK_CTL[HCLK_RST] = 1 */
 750         usbn_clk_ctl.s.hclk_rst = 1;
 751         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 752                                usbn_clk_ctl.u64);
 753         /* 2e.  Wait 64 core-clock cycles for HCLK to stabilize */
 754         cvmx_wait(64);
 755         /*
 756          * 3. Program the power-on reset field in the USBN clock-control
 757          *    register:
 758          *    USBN_CLK_CTL[POR] = 0
 759          */
 760         usbn_clk_ctl.s.por = 0;
 761         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 762                                usbn_clk_ctl.u64);
 763         /* 4. Wait 1 ms for PHY clock to start */
 764         mdelay(1);
 765         /*
 766          * 5. Program the Reset input from automatic test equipment field in the
 767          *    USBP control and status register:
 768          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 1
 769          */
 770         usbn_usbp_ctl_status.u64 = __cvmx_usb_read_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index));
 771         usbn_usbp_ctl_status.s.ate_reset = 1;
 772         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 773                                usbn_usbp_ctl_status.u64);
 774         /* 6. Wait 10 cycles */
 775         cvmx_wait(10);
 776         /*
 777          * 7. Clear ATE_RESET field in the USBN clock-control register:
 778          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 0
 779          */
 780         usbn_usbp_ctl_status.s.ate_reset = 0;
 781         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 782                                usbn_usbp_ctl_status.u64);
 783         /*
 784          * 8. Program the PHY reset field in the USBN clock-control register:
 785          *    USBN_CLK_CTL[PRST] = 1
 786          */
 787         usbn_clk_ctl.s.prst = 1;
 788         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 789                                usbn_clk_ctl.u64);
 790         /*
 791          * 9. Program the USBP control and status register to select host or
 792          *    device mode. USBN_USBP_CTL_STATUS[HST_MODE] = 0 for host, = 1 for
 793          *    device
 794          */
 795         usbn_usbp_ctl_status.s.hst_mode = 0;
 796         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 797                                usbn_usbp_ctl_status.u64);
 798         /* 10. Wait 1 us */
 799         udelay(1);
 800         /*
 801          * 11. Program the hreset_n field in the USBN clock-control register:
 802          *     USBN_CLK_CTL[HRST] = 1
 803          */
 804         usbn_clk_ctl.s.hrst = 1;
 805         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 806                                usbn_clk_ctl.u64);
 807         /* 12. Proceed to USB core initialization */
 808         usbn_clk_ctl.s.enable = 1;
 809         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 810                                usbn_clk_ctl.u64);
 811         udelay(1);
 812
 813         /*
 814          * USB Core Initialization
 815          *
 816          * 1. Read USBC_GHWCFG1, USBC_GHWCFG2, USBC_GHWCFG3, USBC_GHWCFG4 to
 817          *    determine USB core configuration parameters.
 818          *
 819          *    Nothing needed
 820          *
 821          * 2. Program the following fields in the global AHB configuration
 822          *    register (USBC_GAHBCFG)
 823          *    DMA mode, USBC_GAHBCFG[DMAEn]: 1 = DMA mode, 0 = slave mode
 824          *    Burst length, USBC_GAHBCFG[HBSTLEN] = 0
 825          *    Nonperiodic TxFIFO empty level (slave mode only),
 826          *    USBC_GAHBCFG[NPTXFEMPLVL]
 827          *    Periodic TxFIFO empty level (slave mode only),
 828          *    USBC_GAHBCFG[PTXFEMPLVL]
 829          *    Global interrupt mask, USBC_GAHBCFG[GLBLINTRMSK] = 1
 830          */
 831         {
 832                 union cvmx_usbcx_gahbcfg usbcx_gahbcfg;
 833                 /* Due to an errata, CN31XX doesn't support DMA */
 834                 if (OCTEON_IS_MODEL(OCTEON_CN31XX))
 835                         usb->init_flags |= CVMX_USB_INITIALIZE_FLAGS_NO_DMA;
 836                 usbcx_gahbcfg.u32 = 0;
 837                 usbcx_gahbcfg.s.dmaen = !(usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA);
 838                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
 839                         /* Only use one channel with non DMA */
 840                         usb->idle_hardware_channels = 0x1;
 841                 else if (OCTEON_IS_MODEL(OCTEON_CN5XXX))
 842                         /* CN5XXX have an errata with channel 3 */
 843                         usb->idle_hardware_channels = 0xf7;
 844                 else
 845                         usb->idle_hardware_channels = 0xff;
 846                 usbcx_gahbcfg.s.hbstlen = 0;
 847                 usbcx_gahbcfg.s.nptxfemplvl = 1;
 848                 usbcx_gahbcfg.s.ptxfemplvl = 1;
 849                 usbcx_gahbcfg.s.glblintrmsk = 1;
 850                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GAHBCFG(usb->index),
 851                                        usbcx_gahbcfg.u32);
 852         }
 853         /*
 854          * 3. Program the following fields in USBC_GUSBCFG register.
 855          *    HS/FS timeout calibration, USBC_GUSBCFG[TOUTCAL] = 0
 856          *    ULPI DDR select, USBC_GUSBCFG[DDRSEL] = 0
 857          *    USB turnaround time, USBC_GUSBCFG[USBTRDTIM] = 0x5
 858          *    PHY low-power clock select, USBC_GUSBCFG[PHYLPWRCLKSEL] = 0
 859          */
 860         {
 861                 union cvmx_usbcx_gusbcfg usbcx_gusbcfg;
 862                 usbcx_gusbcfg.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index));
 863                 usbcx_gusbcfg.s.toutcal = 0;
 864                 usbcx_gusbcfg.s.ddrsel = 0;
 865                 usbcx_gusbcfg.s.usbtrdtim = 0x5;
 866                 usbcx_gusbcfg.s.phylpwrclksel = 0;
 867                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index),
 868                                        usbcx_gusbcfg.u32);
 869         }
 870         /*
 871          * 4. The software must unmask the following bits in the USBC_GINTMSK
 872          *    register.
 873          *    OTG interrupt mask, USBC_GINTMSK[OTGINTMSK] = 1
 874          *    Mode mismatch interrupt mask, USBC_GINTMSK[MODEMISMSK] = 1
 875          */
 876         {
 877                 union cvmx_usbcx_gintmsk usbcx_gintmsk;
 878                 int channel;
 879
 880                 usbcx_gintmsk.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GINTMSK(usb->index));
 881                 usbcx_gintmsk.s.otgintmsk = 1;
 882                 usbcx_gintmsk.s.modemismsk = 1;
 883                 usbcx_gintmsk.s.hchintmsk = 1;
 884                 usbcx_gintmsk.s.sofmsk = 0;
 885                 /* We need RX FIFO interrupts if we don't have DMA */
 886                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
 887                         usbcx_gintmsk.s.rxflvlmsk = 1;
 888                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTMSK(usb->index),
 889                                        usbcx_gintmsk.u32);
 890
 891                 /*
 892                  * Disable all channel interrupts. We'll enable them per channel
 893                  * later.
 894                  */
 895                 for (channel = 0; channel < 8; channel++)
 896                         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), 0);
 897         }
 898
 899         {
 900                 /*
 901                  * Host Port Initialization
 902                  *
 903                  * 1. Program the host-port interrupt-mask field to unmask,
 904                  *    USBC_GINTMSK[PRTINT] = 1
 905                  */
 906                 USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk,
 907                                 prtintmsk, 1);
 908                 USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk,
 909                                 disconnintmsk, 1);
 910                 /*
 911                  * 2. Program the USBC_HCFG register to select full-speed host
 912                  *    or high-speed host.
 913                  */
 914                 {
 915                         union cvmx_usbcx_hcfg usbcx_hcfg;
 916                         usbcx_hcfg.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCFG(usb->index));
 917                         usbcx_hcfg.s.fslssupp = 0;
 918                         usbcx_hcfg.s.fslspclksel = 0;
 919                         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCFG(usb->index), usbcx_hcfg.u32);
 920                 }
 921                 /*
 922                  * 3. Program the port power bit to drive VBUS on the USB,
 923                  *    USBC_HPRT[PRTPWR] = 1
 924                  */
 925                 USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt, prtpwr, 1);
 926
 927                 /*
 928                  * Steps 4-15 from the manual are done later in the port enable
 929                  */
 930         }
 931
 932         return 0;
 933 }
 934
 935
 936 /**
 937  * Shutdown a USB port after a call to cvmx_usb_initialize().
 938  * The port should be disabled with all pipes closed when this
 939  * function is called.
 940  *
 941  * @usb: USB device state populated by cvmx_usb_initialize().
 942  *
 943  * Returns: 0 or a negative error code.
 944  */
 945 static int cvmx_usb_shutdown(struct cvmx_usb_state *usb)
 946 {
 947         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 948
 949         /* Make sure all pipes are closed */
 950         if (!list_empty(&usb->idle_pipes) ||
 951             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_ISOCHRONOUS]) ||
 952             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_INTERRUPT]) ||
 953             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_CONTROL]) ||
 954             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_BULK]))
 955                 return -EBUSY;
 956
 957         /* Disable the clocks and put them in power on reset */
 958         usbn_clk_ctl.u64 = __cvmx_usb_read_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index));
 959         usbn_clk_ctl.s.enable = 1;
 960         usbn_clk_ctl.s.por = 1;
 961         usbn_clk_ctl.s.hclk_rst = 1;
 962         usbn_clk_ctl.s.prst = 0;
 963         usbn_clk_ctl.s.hrst = 0;
 964         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 965                                usbn_clk_ctl.u64);
 966         return 0;
 967 }
 968
 969
 970 /**
 971  * Enable a USB port. After this call succeeds, the USB port is
 972  * online and servicing requests.
 973  *
 974  * @usb: USB device state populated by cvmx_usb_initialize().
 975  *
 976  * Returns: 0 or a negative error code.
 977  */
 978 static int cvmx_usb_enable(struct cvmx_usb_state *usb)
 979 {
 980         union cvmx_usbcx_ghwcfg3 usbcx_ghwcfg3;
 981
 982         usb->usbcx_hprt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
 983
 984         /*
 985          * If the port is already enabled the just return. We don't need to do
 986          * anything
 987          */
 988         if (usb->usbcx_hprt.s.prtena)
 989                 return 0;
 990
 991         /* If there is nothing plugged into the port then fail immediately */
 992         if (!usb->usbcx_hprt.s.prtconnsts) {
 993                 return -ETIMEDOUT;
 994         }
 995
 996         /* Program the port reset bit to start the reset process */
 997         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt, prtrst, 1);
 998
 999         /*
1000          * Wait at least 50ms (high speed), or 10ms (full speed) for the reset
1001          * process to complete.
1002          */
1003         mdelay(50);
1004
1005         /* Program the port reset bit to 0, USBC_HPRT[PRTRST] = 0 */
1006         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt, prtrst, 0);
1007
1008         /* Wait for the USBC_HPRT[PRTENA]. */
1009         if (CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt,
1010                                   prtena, ==, 1, 100000))
1011                 return -ETIMEDOUT;
1012
1013         /*
1014          * Read the port speed field to get the enumerated speed,
1015          * USBC_HPRT[PRTSPD].
1016          */
1017         usb->usbcx_hprt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
1018         usbcx_ghwcfg3.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GHWCFG3(usb->index));
1019
1020         /*
1021          * 13. Program the USBC_GRXFSIZ register to select the size of the
1022          *     receive FIFO (25%).
1023          */
1024         USB_SET_FIELD32(CVMX_USBCX_GRXFSIZ(usb->index), union cvmx_usbcx_grxfsiz,
1025                         rxfdep, usbcx_ghwcfg3.s.dfifodepth / 4);
1026         /*
1027          * 14. Program the USBC_GNPTXFSIZ register to select the size and the
1028          *     start address of the non- periodic transmit FIFO for nonperiodic
1029          *     transactions (50%).
1030          */
1031         {
1032                 union cvmx_usbcx_gnptxfsiz siz;
1033                 siz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index));
1034                 siz.s.nptxfdep = usbcx_ghwcfg3.s.dfifodepth / 2;
1035                 siz.s.nptxfstaddr = usbcx_ghwcfg3.s.dfifodepth / 4;
1036                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index), siz.u32);
1037         }
1038         /*
1039          * 15. Program the USBC_HPTXFSIZ register to select the size and start
1040          *     address of the periodic transmit FIFO for periodic transactions
1041          *     (25%).
1042          */
1043         {
1044                 union cvmx_usbcx_hptxfsiz siz;
1045                 siz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index));
1046                 siz.s.ptxfsize = usbcx_ghwcfg3.s.dfifodepth / 4;
1047                 siz.s.ptxfstaddr = 3 * usbcx_ghwcfg3.s.dfifodepth / 4;
1048                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index), siz.u32);
1049         }
1050         /* Flush all FIFOs */
1051         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), union cvmx_usbcx_grstctl, txfnum, 0x10);
1052         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), union cvmx_usbcx_grstctl, txfflsh, 1);
1053         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), union cvmx_usbcx_grstctl,
1054                               txfflsh, ==, 0, 100);
1055         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), union cvmx_usbcx_grstctl, rxfflsh, 1);
1056         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), union cvmx_usbcx_grstctl,
1057                               rxfflsh, ==, 0, 100);
1058
1059         return 0;
1060 }
1061
1062
1063 /**
1064  * Disable a USB port. After this call the USB port will not
1065  * generate data transfers and will not generate events.
1066  * Transactions in process will fail and call their
1067  * associated callbacks.
1068  *
1069  * @usb: USB device state populated by cvmx_usb_initialize().
1070  *
1071  * Returns: 0 or a negative error code.
1072  */
1073 static int cvmx_usb_disable(struct cvmx_usb_state *usb)
1074 {
1075         /* Disable the port */
1076         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt, prtena, 1);
1077         return 0;
1078 }
1079
1080
1081 /**
1082  * Get the current state of the USB port. Use this call to
1083  * determine if the usb port has anything connected, is enabled,
1084  * or has some sort of error condition. The return value of this
1085  * call has "changed" bits to signal of the value of some fields
1086  * have changed between calls.
1087  *
1088  * @usb: USB device state populated by cvmx_usb_initialize().
1089  *
1090  * Returns: Port status information
1091  */
1092 static struct cvmx_usb_port_status cvmx_usb_get_status(struct cvmx_usb_state *usb)
1093 {
1094         union cvmx_usbcx_hprt usbc_hprt;
1095         struct cvmx_usb_port_status result;
1096
1097         memset(&result, 0, sizeof(result));
1098
1099         usbc_hprt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
1100         result.port_enabled = usbc_hprt.s.prtena;
1101         result.port_over_current = usbc_hprt.s.prtovrcurract;
1102         result.port_powered = usbc_hprt.s.prtpwr;
1103         result.port_speed = usbc_hprt.s.prtspd;
1104         result.connected = usbc_hprt.s.prtconnsts;
1105         result.connect_change = (result.connected != usb->port_status.connected);
1106
1107         return result;
1108 }
1109
1110 /**
1111  * Open a virtual pipe between the host and a USB device. A pipe
1112  * must be opened before data can be transferred between a device
1113  * and Octeon.
1114  *
1115  * @usb:             USB device state populated by cvmx_usb_initialize().
1116  * @device_addr:
1117  *                   USB device address to open the pipe to
1118  *                   (0-127).
1119  * @endpoint_num:
1120  *                   USB endpoint number to open the pipe to
1121  *                   (0-15).
1122  * @device_speed:
1123  *                   The speed of the device the pipe is going
1124  *                   to. This must match the device's speed,
1125  *                   which may be different than the port speed.
1126  * @max_packet:      The maximum packet length the device can
1127  *                   transmit/receive (low speed=0-8, full
1128  *                   speed=0-1023, high speed=0-1024). This value
1129  *                   comes from the standard endpoint descriptor
1130  *                   field wMaxPacketSize bits <10:0>.
1131  * @transfer_type:
1132  *                   The type of transfer this pipe is for.
1133  * @transfer_dir:
1134  *                   The direction the pipe is in. This is not
1135  *                   used for control pipes.
1136  * @interval:        For ISOCHRONOUS and INTERRUPT transfers,
1137  *                   this is how often the transfer is scheduled
1138  *                   for. All other transfers should specify
1139  *                   zero. The units are in frames (8000/sec at
1140  *                   high speed, 1000/sec for full speed).
1141  * @multi_count:
1142  *                   For high speed devices, this is the maximum
1143  *                   allowed number of packet per microframe.
1144  *                   Specify zero for non high speed devices. This
1145  *                   value comes from the standard endpoint descriptor
1146  *                   field wMaxPacketSize bits <12:11>.
1147  * @hub_device_addr:
1148  *                   Hub device address this device is connected
1149  *                   to. Devices connected directly to Octeon
1150  *                   use zero. This is only used when the device
1151  *                   is full/low speed behind a high speed hub.
1152  *                   The address will be of the high speed hub,
1153  *                   not and full speed hubs after it.
1154  * @hub_port:        Which port on the hub the device is
1155  *                   connected. Use zero for devices connected
1156  *                   directly to Octeon. Like hub_device_addr,
1157  *                   this is only used for full/low speed
1158  *                   devices behind a high speed hub.
1159  *
1160  * Returns: A non-NULL value is a pipe. NULL means an error.
1161  */
1162 static struct cvmx_usb_pipe *cvmx_usb_open_pipe(struct cvmx_usb_state *usb,
1163                                                 int device_addr, int
1164                                                 endpoint_num,
1165                                                 enum cvmx_usb_speed
1166                                                         device_speed,
1167                                                 int max_packet,
1168                                                 enum cvmx_usb_transfer
1169                                                         transfer_type,
1170                                                 enum cvmx_usb_direction
1171                                                         transfer_dir,
1172                                                 int interval, int multi_count,
1173                                                 int hub_device_addr,
1174                                                 int hub_port)
1175 {
1176         struct cvmx_usb_pipe *pipe;
1177
1178         if (unlikely((device_addr < 0) || (device_addr > MAX_USB_ADDRESS)))
1179                 return NULL;
1180         if (unlikely((endpoint_num < 0) || (endpoint_num > MAX_USB_ENDPOINT)))
1181                 return NULL;
1182         if (unlikely(device_speed > CVMX_USB_SPEED_LOW))
1183                 return NULL;
1184         if (unlikely((max_packet <= 0) || (max_packet > 1024)))
1185                 return NULL;
1186         if (unlikely(transfer_type > CVMX_USB_TRANSFER_INTERRUPT))
1187                 return NULL;
1188         if (unlikely((transfer_dir != CVMX_USB_DIRECTION_OUT) &&
1189                 (transfer_dir != CVMX_USB_DIRECTION_IN)))
1190                 return NULL;
1191         if (unlikely(interval < 0))
1192                 return NULL;
1193         if (unlikely((transfer_type == CVMX_USB_TRANSFER_CONTROL) && interval))
1194                 return NULL;
1195         if (unlikely(multi_count < 0))
1196                 return NULL;
1197         if (unlikely((device_speed != CVMX_USB_SPEED_HIGH) &&
1198                 (multi_count != 0)))
1199                 return NULL;
1200         if (unlikely((hub_device_addr < 0) || (hub_device_addr > MAX_USB_ADDRESS)))
1201                 return NULL;
1202         if (unlikely((hub_port < 0) || (hub_port > MAX_USB_HUB_PORT)))
1203                 return NULL;
1204
1205         pipe = kzalloc(sizeof(*pipe), GFP_ATOMIC);
1206         if (!pipe)
1207                 return NULL;
1208         if ((device_speed == CVMX_USB_SPEED_HIGH) &&
1209                 (transfer_dir == CVMX_USB_DIRECTION_OUT) &&
1210                 (transfer_type == CVMX_USB_TRANSFER_BULK))
1211                 pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
1212         pipe->device_addr = device_addr;
1213         pipe->endpoint_num = endpoint_num;
1214         pipe->device_speed = device_speed;
1215         pipe->max_packet = max_packet;
1216         pipe->transfer_type = transfer_type;
1217         pipe->transfer_dir = transfer_dir;
1218         INIT_LIST_HEAD(&pipe->transactions);
1219
1220         /*
1221          * All pipes use interval to rate limit NAK processing. Force an
1222          * interval if one wasn't supplied
1223          */
1224         if (!interval)
1225                 interval = 1;
1226         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1227                 pipe->interval = interval*8;
1228                 /* Force start splits to be schedule on uFrame 0 */
1229                 pipe->next_tx_frame = ((usb->frame_number+7)&~7) + pipe->interval;
1230         } else {
1231                 pipe->interval = interval;
1232                 pipe->next_tx_frame = usb->frame_number + pipe->interval;
1233         }
1234         pipe->multi_count = multi_count;
1235         pipe->hub_device_addr = hub_device_addr;
1236         pipe->hub_port = hub_port;
1237         pipe->pid_toggle = 0;
1238         pipe->split_sc_frame = -1;
1239         list_add_tail(&pipe->node, &usb->idle_pipes);
1240
1241         /*
1242          * We don't need to tell the hardware about this pipe yet since
1243          * it doesn't have any submitted requests
1244          */
1245
1246         return pipe;
1247 }
1248
1249
1250 /**
1251  * Poll the RX FIFOs and remove data as needed. This function is only used
1252  * in non DMA mode. It is very important that this function be called quickly
1253  * enough to prevent FIFO overflow.
1254  *
1255  * @usb:        USB device state populated by cvmx_usb_initialize().
1256  */
1257 static void __cvmx_usb_poll_rx_fifo(struct cvmx_usb_state *usb)
1258 {
1259         union cvmx_usbcx_grxstsph rx_status;
1260         int channel;
1261         int bytes;
1262         uint64_t address;
1263         uint32_t *ptr;
1264
1265         rx_status.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GRXSTSPH(usb->index));
1266         /* Only read data if IN data is there */
1267         if (rx_status.s.pktsts != 2)
1268                 return;
1269         /* Check if no data is available */
1270         if (!rx_status.s.bcnt)
1271                 return;
1272
1273         channel = rx_status.s.chnum;
1274         bytes = rx_status.s.bcnt;
1275         if (!bytes)
1276                 return;
1277
1278         /* Get where the DMA engine would have written this data */
1279         address = __cvmx_usb_read_csr64(usb, CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8);
1280         ptr = cvmx_phys_to_ptr(address);
1281         __cvmx_usb_write_csr64(usb, CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8, address + bytes);
1282
1283         /* Loop writing the FIFO data for this packet into memory */
1284         while (bytes > 0) {
1285                 *ptr++ = __cvmx_usb_read_csr32(usb, USB_FIFO_ADDRESS(channel, usb->index));
1286                 bytes -= 4;
1287         }
1288         CVMX_SYNCW;
1289
1290         return;
1291 }
1292
1293
1294 /**
1295  * Fill the TX hardware fifo with data out of the software
1296  * fifos
1297  *
1298  * @usb:            USB device state populated by cvmx_usb_initialize().
1299  * @fifo:           Software fifo to use
1300  * @available:      Amount of space in the hardware fifo
1301  *
1302  * Returns: Non zero if the hardware fifo was too small and needs
1303  *          to be serviced again.
1304  */
1305 static int __cvmx_usb_fill_tx_hw(struct cvmx_usb_state *usb,
1306                                  struct cvmx_usb_tx_fifo *fifo, int available)
1307 {
1308         /*
1309          * We're done either when there isn't anymore space or the software FIFO
1310          * is empty
1311          */
1312         while (available && (fifo->head != fifo->tail)) {
1313                 int i = fifo->tail;
1314                 const uint32_t *ptr = cvmx_phys_to_ptr(fifo->entry[i].address);
1315                 uint64_t csr_address = USB_FIFO_ADDRESS(fifo->entry[i].channel, usb->index) ^ 4;
1316                 int words = available;
1317
1318                 /* Limit the amount of data to waht the SW fifo has */
1319                 if (fifo->entry[i].size <= available) {
1320                         words = fifo->entry[i].size;
1321                         fifo->tail++;
1322                         if (fifo->tail > MAX_CHANNELS)
1323                                 fifo->tail = 0;
1324                 }
1325
1326                 /* Update the next locations and counts */
1327                 available -= words;
1328                 fifo->entry[i].address += words * 4;
1329                 fifo->entry[i].size -= words;
1330
1331                 /*
1332                  * Write the HW fifo data. The read every three writes is due
1333                  * to an errata on CN3XXX chips
1334                  */
1335                 while (words > 3) {
1336                         cvmx_write64_uint32(csr_address, *ptr++);
1337                         cvmx_write64_uint32(csr_address, *ptr++);
1338                         cvmx_write64_uint32(csr_address, *ptr++);
1339                         cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1340                         words -= 3;
1341                 }
1342                 cvmx_write64_uint32(csr_address, *ptr++);
1343                 if (--words) {
1344                         cvmx_write64_uint32(csr_address, *ptr++);
1345                         if (--words)
1346                                 cvmx_write64_uint32(csr_address, *ptr++);
1347                 }
1348                 cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1349         }
1350         return fifo->head != fifo->tail;
1351 }
1352
1353
1354 /**
1355  * Check the hardware FIFOs and fill them as needed
1356  *
1357  * @usb:        USB device state populated by cvmx_usb_initialize().
1358  */
1359 static void __cvmx_usb_poll_tx_fifo(struct cvmx_usb_state *usb)
1360 {
1361         if (usb->periodic.head != usb->periodic.tail) {
1362                 union cvmx_usbcx_hptxsts tx_status;
1363                 tx_status.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPTXSTS(usb->index));
1364                 if (__cvmx_usb_fill_tx_hw(usb, &usb->periodic, tx_status.s.ptxfspcavail))
1365                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk, ptxfempmsk, 1);
1366                 else
1367                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk, ptxfempmsk, 0);
1368         }
1369
1370         if (usb->nonperiodic.head != usb->nonperiodic.tail) {
1371                 union cvmx_usbcx_gnptxsts tx_status;
1372                 tx_status.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GNPTXSTS(usb->index));
1373                 if (__cvmx_usb_fill_tx_hw(usb, &usb->nonperiodic, tx_status.s.nptxfspcavail))
1374                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk, nptxfempmsk, 1);
1375                 else
1376                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk, nptxfempmsk, 0);
1377         }
1378
1379         return;
1380 }
1381
1382
1383 /**
1384  * Fill the TX FIFO with an outgoing packet
1385  *
1386  * @usb:          USB device state populated by cvmx_usb_initialize().
1387  * @channel:      Channel number to get packet from
1388  */
1389 static void __cvmx_usb_fill_tx_fifo(struct cvmx_usb_state *usb, int channel)
1390 {
1391         union cvmx_usbcx_hccharx hcchar;
1392         union cvmx_usbcx_hcspltx usbc_hcsplt;
1393         union cvmx_usbcx_hctsizx usbc_hctsiz;
1394         struct cvmx_usb_tx_fifo *fifo;
1395
1396         /* We only need to fill data on outbound channels */
1397         hcchar.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index));
1398         if (hcchar.s.epdir != CVMX_USB_DIRECTION_OUT)
1399                 return;
1400
1401         /* OUT Splits only have data on the start and not the complete */
1402         usbc_hcsplt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCSPLTX(channel, usb->index));
1403         if (usbc_hcsplt.s.spltena && usbc_hcsplt.s.compsplt)
1404                 return;
1405
1406         /*
1407          * Find out how many bytes we need to fill and convert it into 32bit
1408          * words.
1409          */
1410         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index));
1411         if (!usbc_hctsiz.s.xfersize)
1412                 return;
1413
1414         if ((hcchar.s.eptype == CVMX_USB_TRANSFER_INTERRUPT) ||
1415                 (hcchar.s.eptype == CVMX_USB_TRANSFER_ISOCHRONOUS))
1416                 fifo = &usb->periodic;
1417         else
1418                 fifo = &usb->nonperiodic;
1419
1420         fifo->entry[fifo->head].channel = channel;
1421         fifo->entry[fifo->head].address = __cvmx_usb_read_csr64(usb, CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8);
1422         fifo->entry[fifo->head].size = (usbc_hctsiz.s.xfersize+3)>>2;
1423         fifo->head++;
1424         if (fifo->head > MAX_CHANNELS)
1425                 fifo->head = 0;
1426
1427         __cvmx_usb_poll_tx_fifo(usb);
1428
1429         return;
1430 }
1431
1432 /**
1433  * Perform channel specific setup for Control transactions. All
1434  * the generic stuff will already have been done in
1435  * __cvmx_usb_start_channel()
1436  *
1437  * @usb:          USB device state populated by cvmx_usb_initialize().
1438  * @channel:      Channel to setup
1439  * @pipe:         Pipe for control transaction
1440  */
1441 static void __cvmx_usb_start_channel_control(struct cvmx_usb_state *usb,
1442                                              int channel,
1443                                              struct cvmx_usb_pipe *pipe)
1444 {
1445         struct cvmx_usb_transaction *transaction =
1446                 list_first_entry(&pipe->transactions, typeof(*transaction),
1447                                  node);
1448         union cvmx_usb_control_header *header =
1449                 cvmx_phys_to_ptr(transaction->control_header);
1450         int bytes_to_transfer = transaction->buffer_length - transaction->actual_bytes;
1451         int packets_to_transfer;
1452         union cvmx_usbcx_hctsizx usbc_hctsiz;
1453
1454         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index));
1455
1456         switch (transaction->stage) {
1457         case CVMX_USB_STAGE_NON_CONTROL:
1458         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
1459                 cvmx_dprintf("%s: ERROR - Non control stage\n", __FUNCTION__);
1460                 break;
1461         case CVMX_USB_STAGE_SETUP:
1462                 usbc_hctsiz.s.pid = 3; /* Setup */
1463                 bytes_to_transfer = sizeof(*header);
1464                 /* All Control operations start with a setup going OUT */
1465                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), union cvmx_usbcx_hccharx, epdir, CVMX_USB_DIRECTION_OUT);
1466                 /*
1467                  * Setup send the control header instead of the buffer data. The
1468                  * buffer data will be used in the next stage
1469                  */
1470                 __cvmx_usb_write_csr64(usb, CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8, transaction->control_header);
1471                 break;
1472         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
1473                 usbc_hctsiz.s.pid = 3; /* Setup */
1474                 bytes_to_transfer = 0;
1475                 /* All Control operations start with a setup going OUT */
1476                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), union cvmx_usbcx_hccharx, epdir, CVMX_USB_DIRECTION_OUT);
1477                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index), union cvmx_usbcx_hcspltx, compsplt, 1);
1478                 break;
1479         case CVMX_USB_STAGE_DATA:
1480                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1481                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1482                         if (header->s.request_type & 0x80)
1483                                 bytes_to_transfer = 0;
1484                         else if (bytes_to_transfer > pipe->max_packet)
1485                                 bytes_to_transfer = pipe->max_packet;
1486                 }
1487                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1488                                 union cvmx_usbcx_hccharx, epdir,
1489                                 ((header->s.request_type & 0x80) ?
1490                                         CVMX_USB_DIRECTION_IN :
1491                                         CVMX_USB_DIRECTION_OUT));
1492                 break;
1493         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
1494                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1495                 if (!(header->s.request_type & 0x80))
1496                         bytes_to_transfer = 0;
1497                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1498                                 union cvmx_usbcx_hccharx, epdir,
1499                                 ((header->s.request_type & 0x80) ?
1500                                         CVMX_USB_DIRECTION_IN :
1501                                         CVMX_USB_DIRECTION_OUT));
1502                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index), union cvmx_usbcx_hcspltx, compsplt, 1);
1503                 break;
1504         case CVMX_USB_STAGE_STATUS:
1505                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1506                 bytes_to_transfer = 0;
1507                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), union cvmx_usbcx_hccharx, epdir,
1508                                 ((header->s.request_type & 0x80) ?
1509                                         CVMX_USB_DIRECTION_OUT :
1510                                         CVMX_USB_DIRECTION_IN));
1511                 break;
1512         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
1513                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1514                 bytes_to_transfer = 0;
1515                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), union cvmx_usbcx_hccharx, epdir,
1516                                 ((header->s.request_type & 0x80) ?
1517                                         CVMX_USB_DIRECTION_OUT :
1518                                         CVMX_USB_DIRECTION_IN));
1519                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index), union cvmx_usbcx_hcspltx, compsplt, 1);
1520                 break;
1521         }
1522
1523         /*
1524          * Make sure the transfer never exceeds the byte limit of the hardware.
1525          * Further bytes will be sent as continued transactions
1526          */
1527         if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1528                 /* Round MAX_TRANSFER_BYTES to a multiple of out packet size */
1529                 bytes_to_transfer = MAX_TRANSFER_BYTES / pipe->max_packet;
1530                 bytes_to_transfer *= pipe->max_packet;
1531         }
1532
1533         /*
1534          * Calculate the number of packets to transfer. If the length is zero
1535          * we still need to transfer one packet
1536          */
1537         packets_to_transfer = (bytes_to_transfer + pipe->max_packet - 1) / pipe->max_packet;
1538         if (packets_to_transfer == 0)
1539                 packets_to_transfer = 1;
1540         else if ((packets_to_transfer > 1) && (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1541                 /*
1542                  * Limit to one packet when not using DMA. Channels must be
1543                  * restarted between every packet for IN transactions, so there
1544                  * is no reason to do multiple packets in a row
1545                  */
1546                 packets_to_transfer = 1;
1547                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1548         } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1549                 /*
1550                  * Limit the number of packet and data transferred to what the
1551                  * hardware can handle
1552                  */
1553                 packets_to_transfer = MAX_TRANSFER_PACKETS;
1554                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1555         }
1556
1557         usbc_hctsiz.s.xfersize = bytes_to_transfer;
1558         usbc_hctsiz.s.pktcnt = packets_to_transfer;
1559
1560         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index), usbc_hctsiz.u32);
1561         return;
1562 }
1563
1564
1565 /**
1566  * Start a channel to perform the pipe's head transaction
1567  *
1568  * @usb:          USB device state populated by cvmx_usb_initialize().
1569  * @channel:      Channel to setup
1570  * @pipe:         Pipe to start
1571  */
1572 static void __cvmx_usb_start_channel(struct cvmx_usb_state *usb,
1573                                      int channel,
1574                                      struct cvmx_usb_pipe *pipe)
1575 {
1576         struct cvmx_usb_transaction *transaction =
1577                 list_first_entry(&pipe->transactions, typeof(*transaction),
1578                                  node);
1579
1580         /* Make sure all writes to the DMA region get flushed */
1581         CVMX_SYNCW;
1582
1583         /* Attach the channel to the pipe */
1584         usb->pipe_for_channel[channel] = pipe;
1585         pipe->channel = channel;
1586         pipe->flags |= __CVMX_USB_PIPE_FLAGS_SCHEDULED;
1587
1588         /* Mark this channel as in use */
1589         usb->idle_hardware_channels &= ~(1<<channel);
1590
1591         /* Enable the channel interrupt bits */
1592         {
1593                 union cvmx_usbcx_hcintx usbc_hcint;
1594                 union cvmx_usbcx_hcintmskx usbc_hcintmsk;
1595                 union cvmx_usbcx_haintmsk usbc_haintmsk;
1596
1597                 /* Clear all channel status bits */
1598                 usbc_hcint.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCINTX(channel, usb->index));
1599                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTX(channel, usb->index), usbc_hcint.u32);
1600
1601                 usbc_hcintmsk.u32 = 0;
1602                 usbc_hcintmsk.s.chhltdmsk = 1;
1603                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1604                         /*
1605                          * Channels need these extra interrupts when we aren't
1606                          * in DMA mode.
1607                          */
1608                         usbc_hcintmsk.s.datatglerrmsk = 1;
1609                         usbc_hcintmsk.s.frmovrunmsk = 1;
1610                         usbc_hcintmsk.s.bblerrmsk = 1;
1611                         usbc_hcintmsk.s.xacterrmsk = 1;
1612                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1613                                 /*
1614                                  * Splits don't generate xfercompl, so we need
1615                                  * ACK and NYET.
1616                                  */
1617                                 usbc_hcintmsk.s.nyetmsk = 1;
1618                                 usbc_hcintmsk.s.ackmsk = 1;
1619                         }
1620                         usbc_hcintmsk.s.nakmsk = 1;
1621                         usbc_hcintmsk.s.stallmsk = 1;
1622                         usbc_hcintmsk.s.xfercomplmsk = 1;
1623                 }
1624                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), usbc_hcintmsk.u32);
1625
1626                 /* Enable the channel interrupt to propagate */
1627                 usbc_haintmsk.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HAINTMSK(usb->index));
1628                 usbc_haintmsk.s.haintmsk |= 1<<channel;
1629                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HAINTMSK(usb->index), usbc_haintmsk.u32);
1630         }
1631
1632         /* Setup the locations the DMA engines use  */
1633         {
1634                 uint64_t dma_address = transaction->buffer + transaction->actual_bytes;
1635                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1636                         dma_address = transaction->buffer + transaction->iso_packets[0].offset + transaction->actual_bytes;
1637                 __cvmx_usb_write_csr64(usb, CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8, dma_address);
1638                 __cvmx_usb_write_csr64(usb, CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8, dma_address);
1639         }
1640
1641         /* Setup both the size of the transfer and the SPLIT characteristics */
1642         {
1643                 union cvmx_usbcx_hcspltx usbc_hcsplt = {.u32 = 0};
1644                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 = 0};
1645                 int packets_to_transfer;
1646                 int bytes_to_transfer = transaction->buffer_length - transaction->actual_bytes;
1647
1648                 /*
1649                  * ISOCHRONOUS transactions store each individual transfer size
1650                  * in the packet structure, not the global buffer_length
1651                  */
1652                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1653                         bytes_to_transfer = transaction->iso_packets[0].length - transaction->actual_bytes;
1654
1655                 /*
1656                  * We need to do split transactions when we are talking to non
1657                  * high speed devices that are behind a high speed hub
1658                  */
1659                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1660                         /*
1661                          * On the start split phase (stage is even) record the
1662                          * frame number we will need to send the split complete.
1663                          * We only store the lower two bits since the time ahead
1664                          * can only be two frames
1665                          */
1666                         if ((transaction->stage&1) == 0) {
1667                                 if (transaction->type == CVMX_USB_TRANSFER_BULK)
1668                                         pipe->split_sc_frame = (usb->frame_number + 1) & 0x7f;
1669                                 else
1670                                         pipe->split_sc_frame = (usb->frame_number + 2) & 0x7f;
1671                         } else
1672                                 pipe->split_sc_frame = -1;
1673
1674                         usbc_hcsplt.s.spltena = 1;
1675                         usbc_hcsplt.s.hubaddr = pipe->hub_device_addr;
1676                         usbc_hcsplt.s.prtaddr = pipe->hub_port;
1677                         usbc_hcsplt.s.compsplt = (transaction->stage == CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE);
1678
1679                         /*
1680                          * SPLIT transactions can only ever transmit one data
1681                          * packet so limit the transfer size to the max packet
1682                          * size
1683                          */
1684                         if (bytes_to_transfer > pipe->max_packet)
1685                                 bytes_to_transfer = pipe->max_packet;
1686
1687                         /*
1688                          * ISOCHRONOUS OUT splits are unique in that they limit
1689                          * data transfers to 188 byte chunks representing the
1690                          * begin/middle/end of the data or all
1691                          */
1692                         if (!usbc_hcsplt.s.compsplt &&
1693                                 (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) &&
1694                                 (pipe->transfer_type == CVMX_USB_TRANSFER_ISOCHRONOUS)) {
1695                                 /*
1696                                  * Clear the split complete frame number as
1697                                  * there isn't going to be a split complete
1698                                  */
1699                                 pipe->split_sc_frame = -1;
1700                                 /*
1701                                  * See if we've started this transfer and sent
1702                                  * data
1703                                  */
1704                                 if (transaction->actual_bytes == 0) {
1705                                         /*
1706                                          * Nothing sent yet, this is either a
1707                                          * begin or the entire payload
1708                                          */
1709                                         if (bytes_to_transfer <= 188)
1710                                                 /* Entire payload in one go */
1711                                                 usbc_hcsplt.s.xactpos = 3;
1712                                         else
1713                                                 /* First part of payload */
1714                                                 usbc_hcsplt.s.xactpos = 2;
1715                                 } else {
1716                                         /*
1717                                          * Continuing the previous data, we must
1718                                          * either be in the middle or at the end
1719                                          */
1720                                         if (bytes_to_transfer <= 188)
1721                                                 /* End of payload */
1722                                                 usbc_hcsplt.s.xactpos = 1;
1723                                         else
1724                                                 /* Middle of payload */
1725                                                 usbc_hcsplt.s.xactpos = 0;
1726                                 }
1727                                 /*
1728                                  * Again, the transfer size is limited to 188
1729                                  * bytes
1730                                  */
1731                                 if (bytes_to_transfer > 188)
1732                                         bytes_to_transfer = 188;
1733                         }
1734                 }
1735
1736                 /*
1737                  * Make sure the transfer never exceeds the byte limit of the
1738                  * hardware. Further bytes will be sent as continued
1739                  * transactions
1740                  */
1741                 if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1742                         /*
1743                          * Round MAX_TRANSFER_BYTES to a multiple of out packet
1744                          * size
1745                          */
1746                         bytes_to_transfer = MAX_TRANSFER_BYTES / pipe->max_packet;
1747                         bytes_to_transfer *= pipe->max_packet;
1748                 }
1749
1750                 /*
1751                  * Calculate the number of packets to transfer. If the length is
1752                  * zero we still need to transfer one packet
1753                  */
1754                 packets_to_transfer = (bytes_to_transfer + pipe->max_packet - 1) / pipe->max_packet;
1755                 if (packets_to_transfer == 0)
1756                         packets_to_transfer = 1;
1757                 else if ((packets_to_transfer > 1) && (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1758                         /*
1759                          * Limit to one packet when not using DMA. Channels must
1760                          * be restarted between every packet for IN
1761                          * transactions, so there is no reason to do multiple
1762                          * packets in a row
1763                          */
1764                         packets_to_transfer = 1;
1765                         bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1766                 } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1767                         /*
1768                          * Limit the number of packet and data transferred to
1769                          * what the hardware can handle
1770                          */
1771                         packets_to_transfer = MAX_TRANSFER_PACKETS;
1772                         bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1773                 }
1774
1775                 usbc_hctsiz.s.xfersize = bytes_to_transfer;
1776                 usbc_hctsiz.s.pktcnt = packets_to_transfer;
1777
1778                 /* Update the DATA0/DATA1 toggle */
1779                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1780                 /*
1781                  * High speed pipes may need a hardware ping before they start
1782                  */
1783                 if (pipe->flags & __CVMX_USB_PIPE_FLAGS_NEED_PING)
1784                         usbc_hctsiz.s.dopng = 1;
1785
1786                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCSPLTX(channel, usb->index), usbc_hcsplt.u32);
1787                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index), usbc_hctsiz.u32);
1788         }
1789
1790         /* Setup the Host Channel Characteristics Register */
1791         {
1792                 union cvmx_usbcx_hccharx usbc_hcchar = {.u32 = 0};
1793
1794                 /*
1795                  * Set the startframe odd/even properly. This is only used for
1796                  * periodic
1797                  */
1798                 usbc_hcchar.s.oddfrm = usb->frame_number&1;
1799
1800                 /*
1801                  * Set the number of back to back packets allowed by this
1802                  * endpoint. Split transactions interpret "ec" as the number of
1803                  * immediate retries of failure. These retries happen too
1804                  * quickly, so we disable these entirely for splits
1805                  */
1806                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
1807                         usbc_hcchar.s.ec = 1;
1808                 else if (pipe->multi_count < 1)
1809                         usbc_hcchar.s.ec = 1;
1810                 else if (pipe->multi_count > 3)
1811                         usbc_hcchar.s.ec = 3;
1812                 else
1813                         usbc_hcchar.s.ec = pipe->multi_count;
1814
1815                 /* Set the rest of the endpoint specific settings */
1816                 usbc_hcchar.s.devaddr = pipe->device_addr;
1817                 usbc_hcchar.s.eptype = transaction->type;
1818                 usbc_hcchar.s.lspddev = (pipe->device_speed == CVMX_USB_SPEED_LOW);
1819                 usbc_hcchar.s.epdir = pipe->transfer_dir;
1820                 usbc_hcchar.s.epnum = pipe->endpoint_num;
1821                 usbc_hcchar.s.mps = pipe->max_packet;
1822                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index), usbc_hcchar.u32);
1823         }
1824
1825         /* Do transaction type specific fixups as needed */
1826         switch (transaction->type) {
1827         case CVMX_USB_TRANSFER_CONTROL:
1828                 __cvmx_usb_start_channel_control(usb, channel, pipe);
1829                 break;
1830         case CVMX_USB_TRANSFER_BULK:
1831         case CVMX_USB_TRANSFER_INTERRUPT:
1832                 break;
1833         case CVMX_USB_TRANSFER_ISOCHRONOUS:
1834                 if (!__cvmx_usb_pipe_needs_split(usb, pipe)) {
1835                         /*
1836                          * ISO transactions require different PIDs depending on
1837                          * direction and how many packets are needed
1838                          */
1839                         if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
1840                                 if (pipe->multi_count < 2) /* Need DATA0 */
1841                                         USB_SET_FIELD32(CVMX_USBCX_HCTSIZX(channel, usb->index), union cvmx_usbcx_hctsizx, pid, 0);
1842                                 else /* Need MDATA */
1843                                         USB_SET_FIELD32(CVMX_USBCX_HCTSIZX(channel, usb->index), union cvmx_usbcx_hctsizx, pid, 3);
1844                         }
1845                 }
1846                 break;
1847         }
1848         {
1849                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index))};
1850                 transaction->xfersize = usbc_hctsiz.s.xfersize;
1851                 transaction->pktcnt = usbc_hctsiz.s.pktcnt;
1852         }
1853         /* Remeber when we start a split transaction */
1854         if (__cvmx_usb_pipe_needs_split(usb, pipe))
1855                 usb->active_split = transaction;
1856         USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), union cvmx_usbcx_hccharx, chena, 1);
1857         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
1858                 __cvmx_usb_fill_tx_fifo(usb, channel);
1859         return;
1860 }
1861
1862
1863 /**
1864  * Find a pipe that is ready to be scheduled to hardware.
1865  * @usb:         USB device state populated by cvmx_usb_initialize().
1866  * @list:        Pipe list to search
1867  * @current_frame:
1868  *               Frame counter to use as a time reference.
1869  *
1870  * Returns: Pipe or NULL if none are ready
1871  */
1872 static struct cvmx_usb_pipe *__cvmx_usb_find_ready_pipe(struct cvmx_usb_state *usb, struct list_head *list, uint64_t current_frame)
1873 {
1874         struct cvmx_usb_pipe *pipe;
1875
1876         list_for_each_entry(pipe, list, node) {
1877                 struct cvmx_usb_transaction *t =
1878                         list_first_entry(&pipe->transactions, typeof(*t), node);
1879                 if (!(pipe->flags & __CVMX_USB_PIPE_FLAGS_SCHEDULED) && t &&
1880                         (pipe->next_tx_frame <= current_frame) &&
1881                         ((pipe->split_sc_frame == -1) || ((((int)current_frame - (int)pipe->split_sc_frame) & 0x7f) < 0x40)) &&
1882                         (!usb->active_split || (usb->active_split == t))) {
1883                         CVMX_PREFETCH(pipe, 128);
1884                         CVMX_PREFETCH(t, 0);
1885                         return pipe;
1886                 }
1887         }
1888         return NULL;
1889 }
1890
1891
1892 /**
1893  * Called whenever a pipe might need to be scheduled to the
1894  * hardware.
1895  *
1896  * @usb:         USB device state populated by cvmx_usb_initialize().
1897  * @is_sof:      True if this schedule was called on a SOF interrupt.
1898  */
1899 static void __cvmx_usb_schedule(struct cvmx_usb_state *usb, int is_sof)
1900 {
1901         int channel;
1902         struct cvmx_usb_pipe *pipe;
1903         int need_sof;
1904         enum cvmx_usb_transfer ttype;
1905
1906         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1907                 /*
1908                  * Without DMA we need to be careful to not schedule something
1909                  * at the end of a frame and cause an overrun.
1910                  */
1911                 union cvmx_usbcx_hfnum hfnum = {.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index))};
1912                 union cvmx_usbcx_hfir hfir = {.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HFIR(usb->index))};
1913                 if (hfnum.s.frrem < hfir.s.frint/4)
1914                         goto done;
1915         }
1916
1917         while (usb->idle_hardware_channels) {
1918                 /* Find an idle channel */
1919                 channel = __fls(usb->idle_hardware_channels);
1920                 if (unlikely(channel > 7))
1921                         break;
1922
1923                 /* Find a pipe needing service */
1924                 pipe = NULL;
1925                 if (is_sof) {
1926                         /*
1927                          * Only process periodic pipes on SOF interrupts. This
1928                          * way we are sure that the periodic data is sent in the
1929                          * beginning of the frame
1930                          */
1931                         pipe = __cvmx_usb_find_ready_pipe(usb, usb->active_pipes + CVMX_USB_TRANSFER_ISOCHRONOUS, usb->frame_number);
1932                         if (likely(!pipe))
1933                                 pipe = __cvmx_usb_find_ready_pipe(usb, usb->active_pipes + CVMX_USB_TRANSFER_INTERRUPT, usb->frame_number);
1934                 }
1935                 if (likely(!pipe)) {
1936                         pipe = __cvmx_usb_find_ready_pipe(usb, usb->active_pipes + CVMX_USB_TRANSFER_CONTROL, usb->frame_number);
1937                         if (likely(!pipe))
1938                                 pipe = __cvmx_usb_find_ready_pipe(usb, usb->active_pipes + CVMX_USB_TRANSFER_BULK, usb->frame_number);
1939                 }
1940                 if (!pipe)
1941                         break;
1942
1943                 __cvmx_usb_start_channel(usb, channel, pipe);
1944         }
1945
1946 done:
1947         /*
1948          * Only enable SOF interrupts when we have transactions pending in the
1949          * future that might need to be scheduled
1950          */
1951         need_sof = 0;
1952         for (ttype = CVMX_USB_TRANSFER_CONTROL; ttype <= CVMX_USB_TRANSFER_INTERRUPT; ttype++) {
1953                 list_for_each_entry(pipe, &usb->active_pipes[ttype], node) {
1954                         if (pipe->next_tx_frame > usb->frame_number) {
1955                                 need_sof = 1;
1956                                 break;
1957                         }
1958                 }
1959         }
1960         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk, sofmsk, need_sof);
1961         return;
1962 }
1963
1964 static inline struct octeon_hcd *cvmx_usb_to_octeon(struct cvmx_usb_state *p)
1965 {
1966         return container_of(p, struct octeon_hcd, usb);
1967 }
1968
1969 static inline struct usb_hcd *octeon_to_hcd(struct octeon_hcd *p)
1970 {
1971         return container_of((void *)p, struct usb_hcd, hcd_priv);
1972 }
1973
1974 static void octeon_usb_urb_complete_callback(struct cvmx_usb_state *usb,
1975                                              enum cvmx_usb_complete status,
1976                                              struct cvmx_usb_pipe *pipe,
1977                                              struct cvmx_usb_transaction
1978                                                 *transaction,
1979                                              int bytes_transferred,
1980                                              struct urb *urb)
1981 {
1982         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
1983         struct usb_hcd *hcd = octeon_to_hcd(priv);
1984         struct device *dev = hcd->self.controller;
1985
1986         urb->actual_length = bytes_transferred;
1987         urb->hcpriv = NULL;
1988
1989         if (!list_empty(&urb->urb_list))
1990                 /*
1991                  * It is on the dequeue_list, but we are going to call
1992                  * usb_hcd_giveback_urb(), so we must clear it from
1993                  * the list.  We got to it before the
1994                  * octeon_usb_urb_dequeue_work() tasklet did.
1995                  */
1996                 list_del_init(&urb->urb_list);
1997
1998         /* For Isochronous transactions we need to update the URB packet status
1999            list from data in our private copy */
2000         if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
2001                 int i;
2002                 /*
2003                  * The pointer to the private list is stored in the setup_packet
2004                  * field.
2005                  */
2006                 struct cvmx_usb_iso_packet *iso_packet =
2007                         (struct cvmx_usb_iso_packet *) urb->setup_packet;
2008                 /* Recalculate the transfer size by adding up each packet */
2009                 urb->actual_length = 0;
2010                 for (i = 0; i < urb->number_of_packets; i++) {
2011                         if (iso_packet[i].status == CVMX_USB_COMPLETE_SUCCESS) {
2012                                 urb->iso_frame_desc[i].status = 0;
2013                                 urb->iso_frame_desc[i].actual_length = iso_packet[i].length;
2014                                 urb->actual_length += urb->iso_frame_desc[i].actual_length;
2015                         } else {
2016                                 dev_dbg(dev, "ISOCHRONOUS packet=%d of %d status=%d pipe=%p transaction=%p size=%d\n",
2017                                         i, urb->number_of_packets,
2018                                         iso_packet[i].status, pipe,
2019                                         transaction, iso_packet[i].length);
2020                                 urb->iso_frame_desc[i].status = -EREMOTEIO;
2021                         }
2022                 }
2023                 /* Free the private list now that we don't need it anymore */
2024                 kfree(iso_packet);
2025                 urb->setup_packet = NULL;
2026         }
2027
2028         switch (status) {
2029         case CVMX_USB_COMPLETE_SUCCESS:
2030                 urb->status = 0;
2031                 break;
2032         case CVMX_USB_COMPLETE_CANCEL:
2033                 if (urb->status == 0)
2034                         urb->status = -ENOENT;
2035                 break;
2036         case CVMX_USB_COMPLETE_STALL:
2037                 dev_dbg(dev, "status=stall pipe=%p transaction=%p size=%d\n",
2038                         pipe, transaction, bytes_transferred);
2039                 urb->status = -EPIPE;
2040                 break;
2041         case CVMX_USB_COMPLETE_BABBLEERR:
2042                 dev_dbg(dev, "status=babble pipe=%p transaction=%p size=%d\n",
2043                         pipe, transaction, bytes_transferred);
2044                 urb->status = -EPIPE;
2045                 break;
2046         case CVMX_USB_COMPLETE_SHORT:
2047                 dev_dbg(dev, "status=short pipe=%p transaction=%p size=%d\n",
2048                         pipe, transaction, bytes_transferred);
2049                 urb->status = -EREMOTEIO;
2050                 break;
2051         case CVMX_USB_COMPLETE_ERROR:
2052         case CVMX_USB_COMPLETE_XACTERR:
2053         case CVMX_USB_COMPLETE_DATATGLERR:
2054         case CVMX_USB_COMPLETE_FRAMEERR:
2055                 dev_dbg(dev, "status=%d pipe=%p transaction=%p size=%d\n",
2056                         status, pipe, transaction, bytes_transferred);
2057                 urb->status = -EPROTO;
2058                 break;
2059         }
2060         spin_unlock(&priv->lock);
2061         usb_hcd_giveback_urb(octeon_to_hcd(priv), urb, urb->status);
2062         spin_lock(&priv->lock);
2063 }
2064
2065 /**
2066  * Signal the completion of a transaction and free it. The
2067  * transaction will be removed from the pipe transaction list.
2068  *
2069  * @usb:         USB device state populated by cvmx_usb_initialize().
2070  * @pipe:        Pipe the transaction is on
2071  * @transaction:
2072  *               Transaction that completed
2073  * @complete_code:
2074  *               Completion code
2075  */
2076 static void __cvmx_usb_perform_complete(struct cvmx_usb_state *usb,
2077                                         struct cvmx_usb_pipe *pipe,
2078                                         struct cvmx_usb_transaction *transaction,
2079                                         enum cvmx_usb_complete complete_code)
2080 {
2081         /* If this was a split then clear our split in progress marker */
2082         if (usb->active_split == transaction)
2083                 usb->active_split = NULL;
2084
2085         /*
2086          * Isochronous transactions need extra processing as they might not be
2087          * done after a single data transfer
2088          */
2089         if (unlikely(transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)) {
2090                 /* Update the number of bytes transferred in this ISO packet */
2091                 transaction->iso_packets[0].length = transaction->actual_bytes;
2092                 transaction->iso_packets[0].status = complete_code;
2093
2094                 /*
2095                  * If there are more ISOs pending and we succeeded, schedule the
2096                  * next one
2097                  */
2098                 if ((transaction->iso_number_packets > 1) && (complete_code == CVMX_USB_COMPLETE_SUCCESS)) {
2099                         /* No bytes transferred for this packet as of yet */
2100                         transaction->actual_bytes = 0;
2101                         /* One less ISO waiting to transfer */
2102                         transaction->iso_number_packets--;
2103                         /* Increment to the next location in our packet array */
2104                         transaction->iso_packets++;
2105                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2106                         goto done;
2107                 }
2108         }
2109
2110         /* Remove the transaction from the pipe list */
2111         list_del(&transaction->node);
2112         if (list_empty(&pipe->transactions))
2113                 list_move_tail(&pipe->node, &usb->idle_pipes);
2114         octeon_usb_urb_complete_callback(usb, complete_code, pipe,
2115                                          transaction,
2116                                          transaction->actual_bytes,
2117                                          transaction->urb);
2118         kfree(transaction);
2119 done:
2120         return;
2121 }
2122
2123
2124 /**
2125  * Submit a usb transaction to a pipe. Called for all types
2126  * of transactions.
2127  *
2128  * @usb:
2129  * @pipe:           Which pipe to submit to.
2130  * @type:           Transaction type
2131  * @buffer:         User buffer for the transaction
2132  * @buffer_length:
2133  *                  User buffer's length in bytes
2134  * @control_header:
2135  *                  For control transactions, the 8 byte standard header
2136  * @iso_start_frame:
2137  *                  For ISO transactions, the start frame
2138  * @iso_number_packets:
2139  *                  For ISO, the number of packet in the transaction.
2140  * @iso_packets:
2141  *                  A description of each ISO packet
2142  * @urb:            URB for the callback
2143  *
2144  * Returns: Transaction or NULL on failure.
2145  */
2146 static struct cvmx_usb_transaction *__cvmx_usb_submit_transaction(struct cvmx_usb_state *usb,
2147                                                                   struct cvmx_usb_pipe *pipe,
2148                                                                   enum cvmx_usb_transfer type,
2149                                                                   uint64_t buffer,
2150                                                                   int buffer_length,
2151                                                                   uint64_t control_header,
2152                                                                   int iso_start_frame,
2153                                                                   int iso_number_packets,
2154                                                                   struct cvmx_usb_iso_packet *iso_packets,
2155                                                                   struct urb *urb)
2156 {
2157         struct cvmx_usb_transaction *transaction;
2158
2159         if (unlikely(pipe->transfer_type != type))
2160                 return NULL;
2161
2162         transaction = kzalloc(sizeof(*transaction), GFP_ATOMIC);
2163         if (unlikely(!transaction))
2164                 return NULL;
2165
2166         transaction->type = type;
2167         transaction->buffer = buffer;
2168         transaction->buffer_length = buffer_length;
2169         transaction->control_header = control_header;
2170         /* FIXME: This is not used, implement it. */
2171         transaction->iso_start_frame = iso_start_frame;
2172         transaction->iso_number_packets = iso_number_packets;
2173         transaction->iso_packets = iso_packets;
2174         transaction->urb = urb;
2175         if (transaction->type == CVMX_USB_TRANSFER_CONTROL)
2176                 transaction->stage = CVMX_USB_STAGE_SETUP;
2177         else
2178                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2179
2180         if (!list_empty(&pipe->transactions)) {
2181                 list_add_tail(&transaction->node, &pipe->transactions);
2182         } else {
2183                 list_add_tail(&transaction->node, &pipe->transactions);
2184                 list_move_tail(&pipe->node,
2185                                &usb->active_pipes[pipe->transfer_type]);
2186
2187                 /*
2188                  * We may need to schedule the pipe if this was the head of the
2189                  * pipe.
2190                  */
2191                 __cvmx_usb_schedule(usb, 0);
2192         }
2193
2194         return transaction;
2195 }
2196
2197
2198 /**
2199  * Call to submit a USB Bulk transfer to a pipe.
2200  *
2201  * @usb:            USB device state populated by cvmx_usb_initialize().
2202  * @pipe:           Handle to the pipe for the transfer.
2203  * @urb:            URB.
2204  *
2205  * Returns: A submitted transaction or NULL on failure.
2206  */
2207 static struct cvmx_usb_transaction *cvmx_usb_submit_bulk(struct cvmx_usb_state *usb,
2208                                                          struct cvmx_usb_pipe *pipe,
2209                                                          struct urb *urb)
2210 {
2211         return __cvmx_usb_submit_transaction(usb, pipe, CVMX_USB_TRANSFER_BULK,
2212                                              urb->transfer_dma,
2213                                              urb->transfer_buffer_length,
2214                                              0, /* control_header */
2215                                              0, /* iso_start_frame */
2216                                              0, /* iso_number_packets */
2217                                              NULL, /* iso_packets */
2218                                              urb);
2219 }
2220
2221
2222 /**
2223  * Call to submit a USB Interrupt transfer to a pipe.
2224  *
2225  * @usb:            USB device state populated by cvmx_usb_initialize().
2226  * @pipe:           Handle to the pipe for the transfer.
2227  * @urb:            URB returned when the callback is called.
2228  *
2229  * Returns: A submitted transaction or NULL on failure.
2230  */
2231 static struct cvmx_usb_transaction *cvmx_usb_submit_interrupt(struct cvmx_usb_state *usb,
2232                                                               struct cvmx_usb_pipe *pipe,
2233                                                               struct urb *urb)
2234 {
2235         return __cvmx_usb_submit_transaction(usb, pipe,
2236                                              CVMX_USB_TRANSFER_INTERRUPT,
2237                                              urb->transfer_dma,
2238                                              urb->transfer_buffer_length,
2239                                              0, /* control_header */
2240                                              0, /* iso_start_frame */
2241                                              0, /* iso_number_packets */
2242                                              NULL, /* iso_packets */
2243                                              urb);
2244 }
2245
2246
2247 /**
2248  * Call to submit a USB Control transfer to a pipe.
2249  *
2250  * @usb:            USB device state populated by cvmx_usb_initialize().
2251  * @pipe:           Handle to the pipe for the transfer.
2252  * @urb:            URB.
2253  *
2254  * Returns: A submitted transaction or NULL on failure.
2255  */
2256 static struct cvmx_usb_transaction *cvmx_usb_submit_control(struct cvmx_usb_state *usb,
2257                                                             struct cvmx_usb_pipe *pipe,
2258                                                             struct urb *urb)
2259 {
2260         int buffer_length = urb->transfer_buffer_length;
2261         uint64_t control_header = urb->setup_dma;
2262         union cvmx_usb_control_header *header =
2263                 cvmx_phys_to_ptr(control_header);
2264
2265         if ((header->s.request_type & 0x80) == 0)
2266                 buffer_length = le16_to_cpu(header->s.length);
2267
2268         return __cvmx_usb_submit_transaction(usb, pipe,
2269                                              CVMX_USB_TRANSFER_CONTROL,
2270                                              urb->transfer_dma, buffer_length,
2271                                              control_header,
2272                                              0, /* iso_start_frame */
2273                                              0, /* iso_number_packets */
2274                                              NULL, /* iso_packets */
2275                                              urb);
2276 }
2277
2278
2279 /**
2280  * Call to submit a USB Isochronous transfer to a pipe.
2281  *
2282  * @usb:            USB device state populated by cvmx_usb_initialize().
2283  * @pipe:           Handle to the pipe for the transfer.
2284  * @urb:            URB returned when the callback is called.
2285  *
2286  * Returns: A submitted transaction or NULL on failure.
2287  */
2288 static struct cvmx_usb_transaction *cvmx_usb_submit_isochronous(struct cvmx_usb_state *usb,
2289                                                                 struct cvmx_usb_pipe *pipe,
2290                                                                 struct urb *urb)
2291 {
2292         struct cvmx_usb_iso_packet *packets;
2293
2294         packets = (struct cvmx_usb_iso_packet *) urb->setup_packet;
2295         return __cvmx_usb_submit_transaction(usb, pipe,
2296                                              CVMX_USB_TRANSFER_ISOCHRONOUS,
2297                                              urb->transfer_dma,
2298                                              urb->transfer_buffer_length,
2299                                              0, /* control_header */
2300                                              urb->start_frame,
2301                                              urb->number_of_packets,
2302                                              packets, urb);
2303 }
2304
2305
2306 /**
2307  * Cancel one outstanding request in a pipe. Canceling a request
2308  * can fail if the transaction has already completed before cancel
2309  * is called. Even after a successful cancel call, it may take
2310  * a frame or two for the cvmx_usb_poll() function to call the
2311  * associated callback.
2312  *
2313  * @usb:         USB device state populated by cvmx_usb_initialize().
2314  * @pipe:        Pipe to cancel requests in.
2315  * @transaction: Transaction to cancel, returned by the submit function.
2316  *
2317  * Returns: 0 or a negative error code.
2318  */
2319 static int cvmx_usb_cancel(struct cvmx_usb_state *usb,
2320                            struct cvmx_usb_pipe *pipe,
2321                            struct cvmx_usb_transaction *transaction)
2322 {
2323         /*
2324          * If the transaction is the HEAD of the queue and scheduled. We need to
2325          * treat it special
2326          */
2327         if (list_first_entry(&pipe->transactions, typeof(*transaction), node) ==
2328             transaction && (pipe->flags & __CVMX_USB_PIPE_FLAGS_SCHEDULED)) {
2329                 union cvmx_usbcx_hccharx usbc_hcchar;
2330
2331                 usb->pipe_for_channel[pipe->channel] = NULL;
2332                 pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_SCHEDULED;
2333
2334                 CVMX_SYNCW;
2335
2336                 usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCCHARX(pipe->channel, usb->index));
2337                 /*
2338                  * If the channel isn't enabled then the transaction already
2339                  * completed.
2340                  */
2341                 if (usbc_hcchar.s.chena) {
2342                         usbc_hcchar.s.chdis = 1;
2343                         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCCHARX(pipe->channel, usb->index), usbc_hcchar.u32);
2344                 }
2345         }
2346         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_CANCEL);
2347         return 0;
2348 }
2349
2350
2351 /**
2352  * Cancel all outstanding requests in a pipe. Logically all this
2353  * does is call cvmx_usb_cancel() in a loop.
2354  *
2355  * @usb:         USB device state populated by cvmx_usb_initialize().
2356  * @pipe:        Pipe to cancel requests in.
2357  *
2358  * Returns: 0 or a negative error code.
2359  */
2360 static int cvmx_usb_cancel_all(struct cvmx_usb_state *usb,
2361                                struct cvmx_usb_pipe *pipe)
2362 {
2363         struct cvmx_usb_transaction *transaction, *next;
2364
2365         /* Simply loop through and attempt to cancel each transaction */
2366         list_for_each_entry_safe(transaction, next, &pipe->transactions, node) {
2367                 int result = cvmx_usb_cancel(usb, pipe, transaction);
2368                 if (unlikely(result != 0))
2369                         return result;
2370         }
2371         return 0;
2372 }
2373
2374
2375 /**
2376  * Close a pipe created with cvmx_usb_open_pipe().
2377  *
2378  * @usb:         USB device state populated by cvmx_usb_initialize().
2379  * @pipe:        Pipe to close.
2380  *
2381  * Returns: 0 or a negative error code. EBUSY is returned if the pipe has
2382  *          outstanding transfers.
2383  */
2384 static int cvmx_usb_close_pipe(struct cvmx_usb_state *usb,
2385                                struct cvmx_usb_pipe *pipe)
2386 {
2387         /* Fail if the pipe has pending transactions */
2388         if (!list_empty(&pipe->transactions))
2389                 return -EBUSY;
2390
2391         list_del(&pipe->node);
2392         kfree(pipe);
2393
2394         return 0;
2395 }
2396
2397 /**
2398  * Get the current USB protocol level frame number. The frame
2399  * number is always in the range of 0-0x7ff.
2400  *
2401  * @usb: USB device state populated by cvmx_usb_initialize().
2402  *
2403  * Returns: USB frame number
2404  */
2405 static int cvmx_usb_get_frame_number(struct cvmx_usb_state *usb)
2406 {
2407         int frame_number;
2408         union cvmx_usbcx_hfnum usbc_hfnum;
2409
2410         usbc_hfnum.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index));
2411         frame_number = usbc_hfnum.s.frnum;
2412
2413         return frame_number;
2414 }
2415
2416
2417 /**
2418  * Poll a channel for status
2419  *
2420  * @usb:     USB device
2421  * @channel: Channel to poll
2422  *
2423  * Returns: Zero on success
2424  */
2425 static int __cvmx_usb_poll_channel(struct cvmx_usb_state *usb, int channel)
2426 {
2427         union cvmx_usbcx_hcintx usbc_hcint;
2428         union cvmx_usbcx_hctsizx usbc_hctsiz;
2429         union cvmx_usbcx_hccharx usbc_hcchar;
2430         struct cvmx_usb_pipe *pipe;
2431         struct cvmx_usb_transaction *transaction;
2432         int bytes_this_transfer;
2433         int bytes_in_last_packet;
2434         int packets_processed;
2435         int buffer_space_left;
2436
2437         /* Read the interrupt status bits for the channel */
2438         usbc_hcint.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCINTX(channel, usb->index));
2439
2440         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
2441                 usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index));
2442
2443                 if (usbc_hcchar.s.chena && usbc_hcchar.s.chdis) {
2444                         /*
2445                          * There seems to be a bug in CN31XX which can cause
2446                          * interrupt IN transfers to get stuck until we do a
2447                          * write of HCCHARX without changing things
2448                          */
2449                         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index), usbc_hcchar.u32);
2450                         return 0;
2451                 }
2452
2453                 /*
2454                  * In non DMA mode the channels don't halt themselves. We need
2455                  * to manually disable channels that are left running
2456                  */
2457                 if (!usbc_hcint.s.chhltd) {
2458                         if (usbc_hcchar.s.chena) {
2459                                 union cvmx_usbcx_hcintmskx hcintmsk;
2460                                 /* Disable all interrupts except CHHLTD */
2461                                 hcintmsk.u32 = 0;
2462                                 hcintmsk.s.chhltdmsk = 1;
2463                                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), hcintmsk.u32);
2464                                 usbc_hcchar.s.chdis = 1;
2465                                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index), usbc_hcchar.u32);
2466                                 return 0;
2467                         } else if (usbc_hcint.s.xfercompl) {
2468                                 /*
2469                                  * Successful IN/OUT with transfer complete.
2470                                  * Channel halt isn't needed.
2471                                  */
2472                         } else {
2473                                 cvmx_dprintf("USB%d: Channel %d interrupt without halt\n", usb->index, channel);
2474                                 return 0;
2475                         }
2476                 }
2477         } else {
2478                 /*
2479                  * There is are no interrupts that we need to process when the
2480                  * channel is still running
2481                  */
2482                 if (!usbc_hcint.s.chhltd)
2483                         return 0;
2484         }
2485
2486         /* Disable the channel interrupts now that it is done */
2487         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), 0);
2488         usb->idle_hardware_channels |= (1<<channel);
2489
2490         /* Make sure this channel is tied to a valid pipe */
2491         pipe = usb->pipe_for_channel[channel];
2492         CVMX_PREFETCH(pipe, 0);
2493         CVMX_PREFETCH(pipe, 128);
2494         if (!pipe)
2495                 return 0;
2496         transaction = list_first_entry(&pipe->transactions, typeof(*transaction),
2497                                        node);
2498         CVMX_PREFETCH(transaction, 0);
2499
2500         /*
2501          * Disconnect this pipe from the HW channel. Later the schedule
2502          * function will figure out which pipe needs to go
2503          */
2504         usb->pipe_for_channel[channel] = NULL;
2505         pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_SCHEDULED;
2506
2507         /*
2508          * Read the channel config info so we can figure out how much data
2509          * transfered
2510          */
2511         usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index));
2512         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index));
2513
2514         /*
2515          * Calculating the number of bytes successfully transferred is dependent
2516          * on the transfer direction
2517          */
2518         packets_processed = transaction->pktcnt - usbc_hctsiz.s.pktcnt;
2519         if (usbc_hcchar.s.epdir) {
2520                 /*
2521                  * IN transactions are easy. For every byte received the
2522                  * hardware decrements xfersize. All we need to do is subtract
2523                  * the current value of xfersize from its starting value and we
2524                  * know how many bytes were written to the buffer
2525                  */
2526                 bytes_this_transfer = transaction->xfersize - usbc_hctsiz.s.xfersize;
2527         } else {
2528                 /*
2529                  * OUT transaction don't decrement xfersize. Instead pktcnt is
2530                  * decremented on every successful packet send. The hardware
2531                  * does this when it receives an ACK, or NYET. If it doesn't
2532                  * receive one of these responses pktcnt doesn't change
2533                  */
2534                 bytes_this_transfer = packets_processed * usbc_hcchar.s.mps;
2535                 /*
2536                  * The last packet may not be a full transfer if we didn't have
2537                  * enough data
2538                  */
2539                 if (bytes_this_transfer > transaction->xfersize)
2540                         bytes_this_transfer = transaction->xfersize;
2541         }
2542         /* Figure out how many bytes were in the last packet of the transfer */
2543         if (packets_processed)
2544                 bytes_in_last_packet = bytes_this_transfer - (packets_processed-1) * usbc_hcchar.s.mps;
2545         else
2546                 bytes_in_last_packet = bytes_this_transfer;
2547
2548         /*
2549          * As a special case, setup transactions output the setup header, not
2550          * the user's data. For this reason we don't count setup data as bytes
2551          * transferred
2552          */
2553         if ((transaction->stage == CVMX_USB_STAGE_SETUP) ||
2554                 (transaction->stage == CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE))
2555                 bytes_this_transfer = 0;
2556
2557         /*
2558          * Add the bytes transferred to the running total. It is important that
2559          * bytes_this_transfer doesn't count any data that needs to be
2560          * retransmitted
2561          */
2562         transaction->actual_bytes += bytes_this_transfer;
2563         if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
2564                 buffer_space_left = transaction->iso_packets[0].length - transaction->actual_bytes;
2565         else
2566                 buffer_space_left = transaction->buffer_length - transaction->actual_bytes;
2567
2568         /*
2569          * We need to remember the PID toggle state for the next transaction.
2570          * The hardware already updated it for the next transaction
2571          */
2572         pipe->pid_toggle = !(usbc_hctsiz.s.pid == 0);
2573
2574         /*
2575          * For high speed bulk out, assume the next transaction will need to do
2576          * a ping before proceeding. If this isn't true the ACK processing below
2577          * will clear this flag
2578          */
2579         if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
2580                 (pipe->transfer_type == CVMX_USB_TRANSFER_BULK) &&
2581                 (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT))
2582                 pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
2583
2584         if (usbc_hcint.s.stall) {
2585                 /*
2586                  * STALL as a response means this transaction cannot be
2587                  * completed because the device can't process transactions. Tell
2588                  * the user. Any data that was transferred will be counted on
2589                  * the actual bytes transferred
2590                  */
2591                 pipe->pid_toggle = 0;
2592                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_STALL);
2593         } else if (usbc_hcint.s.xacterr) {
2594                 /*
2595                  * We know at least one packet worked if we get a ACK or NAK.
2596                  * Reset the retry counter
2597                  */
2598                 if (usbc_hcint.s.nak || usbc_hcint.s.ack)
2599                         transaction->retries = 0;
2600                 transaction->retries++;
2601                 if (transaction->retries > MAX_RETRIES) {
2602                         /*
2603                          * XactErr as a response means the device signaled
2604                          * something wrong with the transfer. For example, PID
2605                          * toggle errors cause these
2606                          */
2607                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_XACTERR);
2608                 } else {
2609                         /*
2610                          * If this was a split then clear our split in progress
2611                          * marker
2612                          */
2613                         if (usb->active_split == transaction)
2614                                 usb->active_split = NULL;
2615                         /*
2616                          * Rewind to the beginning of the transaction by anding
2617                          * off the split complete bit
2618                          */
2619                         transaction->stage &= ~1;
2620                         pipe->split_sc_frame = -1;
2621                         pipe->next_tx_frame += pipe->interval;
2622                         if (pipe->next_tx_frame < usb->frame_number)
2623                                 pipe->next_tx_frame = usb->frame_number + pipe->interval -
2624                                                       (usb->frame_number - pipe->next_tx_frame) % pipe->interval;
2625                 }
2626         } else if (usbc_hcint.s.bblerr) {
2627                 /* Babble Error (BblErr) */
2628                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_BABBLEERR);
2629         } else if (usbc_hcint.s.datatglerr) {
2630                 /* We'll retry the exact same transaction again */
2631                 transaction->retries++;
2632         } else if (usbc_hcint.s.nyet) {
2633                 /*
2634                  * NYET as a response is only allowed in three cases: as a
2635                  * response to a ping, as a response to a split transaction, and
2636                  * as a response to a bulk out. The ping case is handled by
2637                  * hardware, so we only have splits and bulk out
2638                  */
2639                 if (!__cvmx_usb_pipe_needs_split(usb, pipe)) {
2640                         transaction->retries = 0;
2641                         /*
2642                          * If there is more data to go then we need to try
2643                          * again. Otherwise this transaction is complete
2644                          */
2645                         if ((buffer_space_left == 0) || (bytes_in_last_packet < pipe->max_packet))
2646                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2647                 } else {
2648                         /*
2649                          * Split transactions retry the split complete 4 times
2650                          * then rewind to the start split and do the entire
2651                          * transactions again
2652                          */
2653                         transaction->retries++;
2654                         if ((transaction->retries & 0x3) == 0) {
2655                                 /*
2656                                  * Rewind to the beginning of the transaction by
2657                                  * anding off the split complete bit
2658                                  */
2659                                 transaction->stage &= ~1;
2660                                 pipe->split_sc_frame = -1;
2661                         }
2662                 }
2663         } else if (usbc_hcint.s.ack) {
2664                 transaction->retries = 0;
2665                 /*
2666                  * The ACK bit can only be checked after the other error bits.
2667                  * This is because a multi packet transfer may succeed in a
2668                  * number of packets and then get a different response on the
2669                  * last packet. In this case both ACK and the last response bit
2670                  * will be set. If none of the other response bits is set, then
2671                  * the last packet must have been an ACK
2672                  *
2673                  * Since we got an ACK, we know we don't need to do a ping on
2674                  * this pipe
2675                  */
2676                 pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_NEED_PING;
2677
2678                 switch (transaction->type) {
2679                 case CVMX_USB_TRANSFER_CONTROL:
2680                         switch (transaction->stage) {
2681                         case CVMX_USB_STAGE_NON_CONTROL:
2682                         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
2683                                 /* This should be impossible */
2684                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_ERROR);
2685                                 break;
2686                         case CVMX_USB_STAGE_SETUP:
2687                                 pipe->pid_toggle = 1;
2688                                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
2689                                         transaction->stage = CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE;
2690                                 else {
2691                                         union cvmx_usb_control_header *header =
2692                                                 cvmx_phys_to_ptr(transaction->control_header);
2693                                         if (header->s.length)
2694                                                 transaction->stage = CVMX_USB_STAGE_DATA;
2695                                         else
2696                                                 transaction->stage = CVMX_USB_STAGE_STATUS;
2697                                 }
2698                                 break;
2699                         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
2700                                 {
2701                                         union cvmx_usb_control_header *header =
2702                                                 cvmx_phys_to_ptr(transaction->control_header);
2703                                         if (header->s.length)
2704                                                 transaction->stage = CVMX_USB_STAGE_DATA;
2705                                         else
2706                                                 transaction->stage = CVMX_USB_STAGE_STATUS;
2707                                 }
2708                                 break;
2709                         case CVMX_USB_STAGE_DATA:
2710                                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2711                                         transaction->stage = CVMX_USB_STAGE_DATA_SPLIT_COMPLETE;
2712                                         /*
2713                                          * For setup OUT data that are splits,
2714                                          * the hardware doesn't appear to count
2715                                          * transferred data. Here we manually
2716                                          * update the data transferred
2717                                          */
2718                                         if (!usbc_hcchar.s.epdir) {
2719                                                 if (buffer_space_left < pipe->max_packet)
2720                                                         transaction->actual_bytes += buffer_space_left;
2721                                                 else
2722                                                         transaction->actual_bytes += pipe->max_packet;
2723                                         }
2724                                 } else if ((buffer_space_left == 0) || (bytes_in_last_packet < pipe->max_packet)) {
2725                                         pipe->pid_toggle = 1;
2726                                         transaction->stage = CVMX_USB_STAGE_STATUS;
2727                                 }
2728                                 break;
2729                         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
2730                                 if ((buffer_space_left == 0) || (bytes_in_last_packet < pipe->max_packet)) {
2731                                         pipe->pid_toggle = 1;
2732                                         transaction->stage = CVMX_USB_STAGE_STATUS;
2733                                 } else {
2734                                         transaction->stage = CVMX_USB_STAGE_DATA;
2735                                 }
2736                                 break;
2737                         case CVMX_USB_STAGE_STATUS:
2738                                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
2739                                         transaction->stage = CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE;
2740                                 else
2741                                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2742                                 break;
2743                         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
2744                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2745                                 break;
2746                         }
2747                         break;
2748                 case CVMX_USB_TRANSFER_BULK:
2749                 case CVMX_USB_TRANSFER_INTERRUPT:
2750                         /*
2751                          * The only time a bulk transfer isn't complete when it
2752                          * finishes with an ACK is during a split transaction.
2753                          * For splits we need to continue the transfer if more
2754                          * data is needed
2755                          */
2756                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2757                                 if (transaction->stage == CVMX_USB_STAGE_NON_CONTROL)
2758                                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2759                                 else {
2760                                         if (buffer_space_left && (bytes_in_last_packet == pipe->max_packet))
2761                                                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2762                                         else {
2763                                                 if (transaction->type == CVMX_USB_TRANSFER_INTERRUPT)
2764                                                         pipe->next_tx_frame += pipe->interval;
2765                                                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2766                                         }
2767                                 }
2768                         } else {
2769                                 if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
2770                                     (pipe->transfer_type == CVMX_USB_TRANSFER_BULK) &&
2771                                     (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) &&
2772                                     (usbc_hcint.s.nak))
2773                                         pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
2774                                 if (!buffer_space_left || (bytes_in_last_packet < pipe->max_packet)) {
2775                                         if (transaction->type == CVMX_USB_TRANSFER_INTERRUPT)
2776                                                 pipe->next_tx_frame += pipe->interval;
2777                                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2778                                 }
2779                         }
2780                         break;
2781                 case CVMX_USB_TRANSFER_ISOCHRONOUS:
2782                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2783                                 /*
2784                                  * ISOCHRONOUS OUT splits don't require a
2785                                  * complete split stage. Instead they use a
2786                                  * sequence of begin OUT splits to transfer the
2787                                  * data 188 bytes at a time. Once the transfer
2788                                  * is complete, the pipe sleeps until the next
2789                                  * schedule interval
2790                                  */
2791                                 if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
2792                                         /*
2793                                          * If no space left or this wasn't a max
2794                                          * size packet then this transfer is
2795                                          * complete. Otherwise start it again to
2796                                          * send the next 188 bytes
2797                                          */
2798                                         if (!buffer_space_left || (bytes_this_transfer < 188)) {
2799                                                 pipe->next_tx_frame += pipe->interval;
2800                                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2801                                         }
2802                                 } else {
2803                                         if (transaction->stage == CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE) {
2804                                                 /*
2805                                                  * We are in the incoming data
2806                                                  * phase. Keep getting data
2807                                                  * until we run out of space or
2808                                                  * get a small packet
2809                                                  */
2810                                                 if ((buffer_space_left == 0) || (bytes_in_last_packet < pipe->max_packet)) {
2811                                                         pipe->next_tx_frame += pipe->interval;
2812                                                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2813                                                 }
2814                                         } else
2815                                                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2816                                 }
2817                         } else {
2818                                 pipe->next_tx_frame += pipe->interval;
2819                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2820                         }
2821                         break;
2822                 }
2823         } else if (usbc_hcint.s.nak) {
2824                 /*
2825                  * If this was a split then clear our split in progress marker.
2826                  */
2827                 if (usb->active_split == transaction)
2828                         usb->active_split = NULL;
2829                 /*
2830                  * NAK as a response means the device couldn't accept the
2831                  * transaction, but it should be retried in the future. Rewind
2832                  * to the beginning of the transaction by anding off the split
2833                  * complete bit. Retry in the next interval
2834                  */
2835                 transaction->retries = 0;
2836                 transaction->stage &= ~1;
2837                 pipe->next_tx_frame += pipe->interval;
2838                 if (pipe->next_tx_frame < usb->frame_number)
2839                         pipe->next_tx_frame = usb->frame_number + pipe->interval -
2840                                 (usb->frame_number - pipe->next_tx_frame) % pipe->interval;
2841         } else {
2842                 struct cvmx_usb_port_status port;
2843                 port = cvmx_usb_get_status(usb);
2844                 if (port.port_enabled) {
2845                         /* We'll retry the exact same transaction again */
2846                         transaction->retries++;
2847                 } else {
2848                         /*
2849                          * We get channel halted interrupts with no result bits
2850                          * sets when the cable is unplugged
2851                          */
2852                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_ERROR);
2853                 }
2854         }
2855         return 0;
2856 }
2857
2858 static void octeon_usb_port_callback(struct cvmx_usb_state *usb)
2859 {
2860         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
2861
2862         spin_unlock(&priv->lock);
2863         usb_hcd_poll_rh_status(octeon_to_hcd(priv));
2864         spin_lock(&priv->lock);
2865 }
2866
2867 /**
2868  * Poll the USB block for status and call all needed callback
2869  * handlers. This function is meant to be called in the interrupt
2870  * handler for the USB controller. It can also be called
2871  * periodically in a loop for non-interrupt based operation.
2872  *
2873  * @usb: USB device state populated by cvmx_usb_initialize().
2874  *
2875  * Returns: 0 or a negative error code.
2876  */
2877 static int cvmx_usb_poll(struct cvmx_usb_state *usb)
2878 {
2879         union cvmx_usbcx_hfnum usbc_hfnum;
2880         union cvmx_usbcx_gintsts usbc_gintsts;
2881
2882         CVMX_PREFETCH(usb, 0);
2883         CVMX_PREFETCH(usb, 1*128);
2884         CVMX_PREFETCH(usb, 2*128);
2885         CVMX_PREFETCH(usb, 3*128);
2886         CVMX_PREFETCH(usb, 4*128);
2887
2888         /* Update the frame counter */
2889         usbc_hfnum.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index));
2890         if ((usb->frame_number&0x3fff) > usbc_hfnum.s.frnum)
2891                 usb->frame_number += 0x4000;
2892         usb->frame_number &= ~0x3fffull;
2893         usb->frame_number |= usbc_hfnum.s.frnum;
2894
2895         /* Read the pending interrupts */
2896         usbc_gintsts.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GINTSTS(usb->index));
2897
2898         /* Clear the interrupts now that we know about them */
2899         __cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTSTS(usb->index), usbc_gintsts.u32);
2900
2901         if (usbc_gintsts.s.rxflvl) {
2902                 /*
2903                  * RxFIFO Non-Empty (RxFLvl)
2904                  * Indicates that there is at least one packet pending to be
2905                  * read from the RxFIFO.
2906                  *
2907                  * In DMA mode this is handled by hardware
2908                  */
2909                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
2910                         __cvmx_usb_poll_rx_fifo(usb);
2911         }
2912         if (usbc_gintsts.s.ptxfemp || usbc_gintsts.s.nptxfemp) {
2913                 /* Fill the Tx FIFOs when not in DMA mode */
2914                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
2915                         __cvmx_usb_poll_tx_fifo(usb);
2916         }
2917         if (usbc_gintsts.s.disconnint || usbc_gintsts.s.prtint) {
2918                 union cvmx_usbcx_hprt usbc_hprt;
2919                 /*
2920                  * Disconnect Detected Interrupt (DisconnInt)
2921                  * Asserted when a device disconnect is detected.
2922                  *
2923                  * Host Port Interrupt (PrtInt)
2924                  * The core sets this bit to indicate a change in port status of
2925                  * one of the O2P USB core ports in Host mode. The application
2926                  * must read the Host Port Control and Status (HPRT) register to
2927                  * determine the exact event that caused this interrupt. The
2928                  * application must clear the appropriate status bit in the Host
2929                  * Port Control and Status register to clear this bit.
2930                  *
2931                  * Call the user's port callback
2932                  */
2933                 octeon_usb_port_callback(usb);
2934                 /* Clear the port change bits */
2935                 usbc_hprt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
2936                 usbc_hprt.s.prtena = 0;
2937                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HPRT(usb->index), usbc_hprt.u32);
2938         }
2939         if (usbc_gintsts.s.hchint) {
2940                 /*
2941                  * Host Channels Interrupt (HChInt)
2942                  * The core sets this bit to indicate that an interrupt is
2943                  * pending on one of the channels of the core (in Host mode).
2944                  * The application must read the Host All Channels Interrupt
2945                  * (HAINT) register to determine the exact number of the channel
2946                  * on which the interrupt occurred, and then read the
2947                  * corresponding Host Channel-n Interrupt (HCINTn) register to
2948                  * determine the exact cause of the interrupt. The application
2949                  * must clear the appropriate status bit in the HCINTn register
2950                  * to clear this bit.
2951                  */
2952                 union cvmx_usbcx_haint usbc_haint;
2953                 usbc_haint.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HAINT(usb->index));
2954                 while (usbc_haint.u32) {
2955                         int channel;
2956
2957                         channel = __fls(usbc_haint.u32);
2958                         __cvmx_usb_poll_channel(usb, channel);
2959                         usbc_haint.u32 ^= 1<<channel;
2960                 }
2961         }
2962
2963         __cvmx_usb_schedule(usb, usbc_gintsts.s.sof);
2964
2965         return 0;
2966 }
2967
2968 /* convert between an HCD pointer and the corresponding struct octeon_hcd */
2969 static inline struct octeon_hcd *hcd_to_octeon(struct usb_hcd *hcd)
2970 {
2971         return (struct octeon_hcd *)(hcd->hcd_priv);
2972 }
2973
2974 static irqreturn_t octeon_usb_irq(struct usb_hcd *hcd)
2975 {
2976         struct octeon_hcd *priv = hcd_to_octeon(hcd);
2977         unsigned long flags;
2978
2979         spin_lock_irqsave(&priv->lock, flags);
2980         cvmx_usb_poll(&priv->usb);
2981         spin_unlock_irqrestore(&priv->lock, flags);
2982         return IRQ_HANDLED;
2983 }
2984
2985 static int octeon_usb_start(struct usb_hcd *hcd)
2986 {
2987         hcd->state = HC_STATE_RUNNING;
2988         return 0;
2989 }
2990
2991 static void octeon_usb_stop(struct usb_hcd *hcd)
2992 {
2993         hcd->state = HC_STATE_HALT;
2994 }
2995
2996 static int octeon_usb_get_frame_number(struct usb_hcd *hcd)
2997 {
2998         struct octeon_hcd *priv = hcd_to_octeon(hcd);
2999
3000         return cvmx_usb_get_frame_number(&priv->usb);
3001 }
3002
3003 static int octeon_usb_urb_enqueue(struct usb_hcd *hcd,
3004                                   struct urb *urb,
3005                                   gfp_t mem_flags)
3006 {
3007         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3008         struct device *dev = hcd->self.controller;
3009         struct cvmx_usb_transaction *transaction = NULL;
3010         struct cvmx_usb_pipe *pipe;
3011         unsigned long flags;
3012         struct cvmx_usb_iso_packet *iso_packet;
3013         struct usb_host_endpoint *ep = urb->ep;
3014
3015         urb->status = 0;
3016         INIT_LIST_HEAD(&urb->urb_list); /* not enqueued on dequeue_list */
3017         spin_lock_irqsave(&priv->lock, flags);
3018
3019         if (!ep->hcpriv) {
3020                 enum cvmx_usb_transfer transfer_type;
3021                 enum cvmx_usb_speed speed;
3022                 int split_device = 0;
3023                 int split_port = 0;
3024                 switch (usb_pipetype(urb->pipe)) {
3025                 case PIPE_ISOCHRONOUS:
3026                         transfer_type = CVMX_USB_TRANSFER_ISOCHRONOUS;
3027                         break;
3028                 case PIPE_INTERRUPT:
3029                         transfer_type = CVMX_USB_TRANSFER_INTERRUPT;
3030                         break;
3031                 case PIPE_CONTROL:
3032                         transfer_type = CVMX_USB_TRANSFER_CONTROL;
3033                         break;
3034                 default:
3035                         transfer_type = CVMX_USB_TRANSFER_BULK;
3036                         break;
3037                 }
3038                 switch (urb->dev->speed) {
3039                 case USB_SPEED_LOW:
3040                         speed = CVMX_USB_SPEED_LOW;
3041                         break;
3042                 case USB_SPEED_FULL:
3043                         speed = CVMX_USB_SPEED_FULL;
3044                         break;
3045                 default:
3046                         speed = CVMX_USB_SPEED_HIGH;
3047                         break;
3048                 }
3049                 /*
3050                  * For slow devices on high speed ports we need to find the hub
3051                  * that does the speed translation so we know where to send the
3052                  * split transactions.
3053                  */
3054                 if (speed != CVMX_USB_SPEED_HIGH) {
3055                         /*
3056                          * Start at this device and work our way up the usb
3057                          * tree.
3058                          */
3059                         struct usb_device *dev = urb->dev;
3060                         while (dev->parent) {
3061                                 /*
3062                                  * If our parent is high speed then he'll
3063                                  * receive the splits.
3064                                  */
3065                                 if (dev->parent->speed == USB_SPEED_HIGH) {
3066                                         split_device = dev->parent->devnum;
3067                                         split_port = dev->portnum;
3068                                         break;
3069                                 }
3070                                 /*
3071                                  * Move up the tree one level. If we make it all
3072                                  * the way up the tree, then the port must not
3073                                  * be in high speed mode and we don't need a
3074                                  * split.
3075                                  */
3076                                 dev = dev->parent;
3077                         }
3078                 }
3079                 pipe = cvmx_usb_open_pipe(&priv->usb, usb_pipedevice(urb->pipe),
3080                                           usb_pipeendpoint(urb->pipe), speed,
3081                                           le16_to_cpu(ep->desc.wMaxPacketSize) & 0x7ff,
3082                                           transfer_type,
3083                                           usb_pipein(urb->pipe) ?
3084                                                 CVMX_USB_DIRECTION_IN :
3085                                                 CVMX_USB_DIRECTION_OUT,
3086                                           urb->interval,
3087                                           (le16_to_cpu(ep->desc.wMaxPacketSize) >> 11) & 0x3,
3088                                           split_device, split_port);
3089                 if (!pipe) {
3090                         spin_unlock_irqrestore(&priv->lock, flags);
3091                         dev_dbg(dev, "Failed to create pipe\n");
3092                         return -ENOMEM;
3093                 }
3094                 ep->hcpriv = pipe;
3095         } else {
3096                 pipe = ep->hcpriv;
3097         }
3098
3099         switch (usb_pipetype(urb->pipe)) {
3100         case PIPE_ISOCHRONOUS:
3101                 dev_dbg(dev, "Submit isochronous to %d.%d\n",
3102                         usb_pipedevice(urb->pipe), usb_pipeendpoint(urb->pipe));
3103                 /*
3104                  * Allocate a structure to use for our private list of
3105                  * isochronous packets.
3106                  */
3107                 iso_packet = kmalloc(urb->number_of_packets *
3108                                      sizeof(struct cvmx_usb_iso_packet),
3109                                      GFP_ATOMIC);
3110                 if (iso_packet) {
3111                         int i;
3112                         /* Fill the list with the data from the URB */
3113                         for (i = 0; i < urb->number_of_packets; i++) {
3114                                 iso_packet[i].offset = urb->iso_frame_desc[i].offset;
3115                                 iso_packet[i].length = urb->iso_frame_desc[i].length;
3116                                 iso_packet[i].status = CVMX_USB_COMPLETE_ERROR;
3117                         }
3118                         /*
3119                          * Store a pointer to the list in the URB setup_packet
3120                          * field. We know this currently isn't being used and
3121                          * this saves us a bunch of logic.
3122                          */
3123                         urb->setup_packet = (char *)iso_packet;
3124                         transaction = cvmx_usb_submit_isochronous(&priv->usb,
3125                                                                   pipe, urb);
3126                         /*
3127                          * If submit failed we need to free our private packet
3128                          * list.
3129                          */
3130                         if (!transaction) {
3131                                 urb->setup_packet = NULL;
3132                                 kfree(iso_packet);
3133                         }
3134                 }
3135                 break;
3136         case PIPE_INTERRUPT:
3137                 dev_dbg(dev, "Submit interrupt to %d.%d\n",
3138                         usb_pipedevice(urb->pipe), usb_pipeendpoint(urb->pipe));
3139                 transaction = cvmx_usb_submit_interrupt(&priv->usb, pipe, urb);
3140                 break;
3141         case PIPE_CONTROL:
3142                 dev_dbg(dev, "Submit control to %d.%d\n",
3143                         usb_pipedevice(urb->pipe), usb_pipeendpoint(urb->pipe));
3144                 transaction = cvmx_usb_submit_control(&priv->usb, pipe, urb);
3145                 break;
3146         case PIPE_BULK:
3147                 dev_dbg(dev, "Submit bulk to %d.%d\n",
3148                         usb_pipedevice(urb->pipe), usb_pipeendpoint(urb->pipe));
3149                 transaction = cvmx_usb_submit_bulk(&priv->usb, pipe, urb);
3150                 break;
3151         }
3152         if (!transaction) {
3153                 spin_unlock_irqrestore(&priv->lock, flags);
3154                 dev_dbg(dev, "Failed to submit\n");
3155                 return -ENOMEM;
3156         }
3157         urb->hcpriv = transaction;
3158         spin_unlock_irqrestore(&priv->lock, flags);
3159         return 0;
3160 }
3161
3162 static void octeon_usb_urb_dequeue_work(unsigned long arg)
3163 {
3164         struct urb *urb;
3165         struct urb *next;
3166         unsigned long flags;
3167         struct octeon_hcd *priv = (struct octeon_hcd *)arg;
3168
3169         spin_lock_irqsave(&priv->lock, flags);
3170
3171         list_for_each_entry_safe(urb, next, &priv->dequeue_list, urb_list) {
3172                 list_del_init(&urb->urb_list);
3173                 cvmx_usb_cancel(&priv->usb, urb->ep->hcpriv, urb->hcpriv);
3174         }
3175
3176         spin_unlock_irqrestore(&priv->lock, flags);
3177 }
3178
3179 static int octeon_usb_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
3180 {
3181         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3182         unsigned long flags;
3183
3184         if (!urb->dev)
3185                 return -EINVAL;
3186
3187         spin_lock_irqsave(&priv->lock, flags);
3188
3189         urb->status = status;
3190         list_add_tail(&urb->urb_list, &priv->dequeue_list);
3191
3192         spin_unlock_irqrestore(&priv->lock, flags);
3193
3194         tasklet_schedule(&priv->dequeue_tasklet);
3195
3196         return 0;
3197 }
3198
3199 static void octeon_usb_endpoint_disable(struct usb_hcd *hcd, struct usb_host_endpoint *ep)
3200 {
3201         struct device *dev = hcd->self.controller;
3202
3203         if (ep->hcpriv) {
3204                 struct octeon_hcd *priv = hcd_to_octeon(hcd);
3205                 struct cvmx_usb_pipe *pipe = ep->hcpriv;
3206                 unsigned long flags;
3207                 spin_lock_irqsave(&priv->lock, flags);
3208                 cvmx_usb_cancel_all(&priv->usb, pipe);
3209                 if (cvmx_usb_close_pipe(&priv->usb, pipe))
3210                         dev_dbg(dev, "Closing pipe %p failed\n", pipe);
3211                 spin_unlock_irqrestore(&priv->lock, flags);
3212                 ep->hcpriv = NULL;
3213         }
3214 }
3215
3216 static int octeon_usb_hub_status_data(struct usb_hcd *hcd, char *buf)
3217 {
3218         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3219         struct cvmx_usb_port_status port_status;
3220         unsigned long flags;
3221
3222         spin_lock_irqsave(&priv->lock, flags);
3223         port_status = cvmx_usb_get_status(&priv->usb);
3224         spin_unlock_irqrestore(&priv->lock, flags);
3225         buf[0] = 0;
3226         buf[0] = port_status.connect_change << 1;
3227
3228         return (buf[0] != 0);
3229 }
3230
3231 static int octeon_usb_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, u16 wIndex, char *buf, u16 wLength)
3232 {
3233         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3234         struct device *dev = hcd->self.controller;
3235         struct cvmx_usb_port_status usb_port_status;
3236         int port_status;
3237         struct usb_hub_descriptor *desc;
3238         unsigned long flags;
3239
3240         switch (typeReq) {
3241         case ClearHubFeature:
3242                 dev_dbg(dev, "ClearHubFeature\n");
3243                 switch (wValue) {
3244                 case C_HUB_LOCAL_POWER:
3245                 case C_HUB_OVER_CURRENT:
3246                         /* Nothing required here */
3247                         break;
3248                 default:
3249                         return -EINVAL;
3250                 }
3251                 break;
3252         case ClearPortFeature:
3253                 dev_dbg(dev, "ClearPortFeature\n");
3254                 if (wIndex != 1) {
3255                         dev_dbg(dev, " INVALID\n");
3256                         return -EINVAL;
3257                 }
3258
3259                 switch (wValue) {
3260                 case USB_PORT_FEAT_ENABLE:
3261                         dev_dbg(dev, " ENABLE\n");
3262                         spin_lock_irqsave(&priv->lock, flags);
3263                         cvmx_usb_disable(&priv->usb);
3264                         spin_unlock_irqrestore(&priv->lock, flags);
3265                         break;
3266                 case USB_PORT_FEAT_SUSPEND:
3267                         dev_dbg(dev, " SUSPEND\n");
3268                         /* Not supported on Octeon */
3269                         break;
3270                 case USB_PORT_FEAT_POWER:
3271                         dev_dbg(dev, " POWER\n");
3272                         /* Not supported on Octeon */
3273                         break;
3274                 case USB_PORT_FEAT_INDICATOR:
3275                         dev_dbg(dev, " INDICATOR\n");
3276                         /* Port inidicator not supported */
3277                         break;
3278                 case USB_PORT_FEAT_C_CONNECTION:
3279                         dev_dbg(dev, " C_CONNECTION\n");
3280                         /* Clears drivers internal connect status change flag */
3281                         spin_lock_irqsave(&priv->lock, flags);
3282                         priv->usb.port_status = cvmx_usb_get_status(&priv->usb);
3283                         spin_unlock_irqrestore(&priv->lock, flags);
3284                         break;
3285                 case USB_PORT_FEAT_C_RESET:
3286                         dev_dbg(dev, " C_RESET\n");
3287                         /*
3288                          * Clears the driver's internal Port Reset Change flag.
3289                          */
3290                         spin_lock_irqsave(&priv->lock, flags);
3291                         priv->usb.port_status = cvmx_usb_get_status(&priv->usb);
3292                         spin_unlock_irqrestore(&priv->lock, flags);
3293                         break;
3294                 case USB_PORT_FEAT_C_ENABLE:
3295                         dev_dbg(dev, " C_ENABLE\n");
3296                         /*
3297                          * Clears the driver's internal Port Enable/Disable
3298                          * Change flag.
3299                          */
3300                         spin_lock_irqsave(&priv->lock, flags);
3301                         priv->usb.port_status = cvmx_usb_get_status(&priv->usb);
3302                         spin_unlock_irqrestore(&priv->lock, flags);
3303                         break;
3304                 case USB_PORT_FEAT_C_SUSPEND:
3305                         dev_dbg(dev, " C_SUSPEND\n");
3306                         /*
3307                          * Clears the driver's internal Port Suspend Change
3308                          * flag, which is set when resume signaling on the host
3309                          * port is complete.
3310                          */
3311                         break;
3312                 case USB_PORT_FEAT_C_OVER_CURRENT:
3313                         dev_dbg(dev, " C_OVER_CURRENT\n");
3314                         /* Clears the driver's overcurrent Change flag */
3315                         spin_lock_irqsave(&priv->lock, flags);
3316                         priv->usb.port_status = cvmx_usb_get_status(&priv->usb);
3317                         spin_unlock_irqrestore(&priv->lock, flags);
3318                         break;
3319                 default:
3320                         dev_dbg(dev, " UNKNOWN\n");
3321                         return -EINVAL;
3322                 }
3323                 break;
3324         case GetHubDescriptor:
3325                 dev_dbg(dev, "GetHubDescriptor\n");
3326                 desc = (struct usb_hub_descriptor *)buf;
3327                 desc->bDescLength = 9;
3328                 desc->bDescriptorType = 0x29;
3329                 desc->bNbrPorts = 1;
3330                 desc->wHubCharacteristics = 0x08;
3331                 desc->bPwrOn2PwrGood = 1;
3332                 desc->bHubContrCurrent = 0;
3333                 desc->u.hs.DeviceRemovable[0] = 0;
3334                 desc->u.hs.DeviceRemovable[1] = 0xff;
3335                 break;
3336         case GetHubStatus:
3337                 dev_dbg(dev, "GetHubStatus\n");
3338                 *(__le32 *) buf = 0;
3339                 break;
3340         case GetPortStatus:
3341                 dev_dbg(dev, "GetPortStatus\n");
3342                 if (wIndex != 1) {
3343                         dev_dbg(dev, " INVALID\n");
3344                         return -EINVAL;
3345                 }
3346
3347                 spin_lock_irqsave(&priv->lock, flags);
3348                 usb_port_status = cvmx_usb_get_status(&priv->usb);
3349                 spin_unlock_irqrestore(&priv->lock, flags);
3350                 port_status = 0;
3351
3352                 if (usb_port_status.connect_change) {
3353                         port_status |= (1 << USB_PORT_FEAT_C_CONNECTION);
3354                         dev_dbg(dev, " C_CONNECTION\n");
3355                 }
3356
3357                 if (usb_port_status.port_enabled) {
3358                         port_status |= (1 << USB_PORT_FEAT_C_ENABLE);
3359                         dev_dbg(dev, " C_ENABLE\n");
3360                 }
3361
3362                 if (usb_port_status.connected) {
3363                         port_status |= (1 << USB_PORT_FEAT_CONNECTION);
3364                         dev_dbg(dev, " CONNECTION\n");
3365                 }
3366
3367                 if (usb_port_status.port_enabled) {
3368                         port_status |= (1 << USB_PORT_FEAT_ENABLE);
3369                         dev_dbg(dev, " ENABLE\n");
3370                 }
3371
3372                 if (usb_port_status.port_over_current) {
3373                         port_status |= (1 << USB_PORT_FEAT_OVER_CURRENT);
3374                         dev_dbg(dev, " OVER_CURRENT\n");
3375                 }
3376
3377                 if (usb_port_status.port_powered) {
3378                         port_status |= (1 << USB_PORT_FEAT_POWER);
3379                         dev_dbg(dev, " POWER\n");
3380                 }
3381
3382                 if (usb_port_status.port_speed == CVMX_USB_SPEED_HIGH) {
3383                         port_status |= USB_PORT_STAT_HIGH_SPEED;
3384                         dev_dbg(dev, " HIGHSPEED\n");
3385                 } else if (usb_port_status.port_speed == CVMX_USB_SPEED_LOW) {
3386                         port_status |= (1 << USB_PORT_FEAT_LOWSPEED);
3387                         dev_dbg(dev, " LOWSPEED\n");
3388                 }
3389
3390                 *((__le32 *) buf) = cpu_to_le32(port_status);
3391                 break;
3392         case SetHubFeature:
3393                 dev_dbg(dev, "SetHubFeature\n");
3394                 /* No HUB features supported */
3395                 break;
3396         case SetPortFeature:
3397                 dev_dbg(dev, "SetPortFeature\n");
3398                 if (wIndex != 1) {
3399                         dev_dbg(dev, " INVALID\n");
3400                         return -EINVAL;
3401                 }
3402
3403                 switch (wValue) {
3404                 case USB_PORT_FEAT_SUSPEND:
3405                         dev_dbg(dev, " SUSPEND\n");
3406                         return -EINVAL;
3407                 case USB_PORT_FEAT_POWER:
3408                         dev_dbg(dev, " POWER\n");
3409                         return -EINVAL;
3410                 case USB_PORT_FEAT_RESET:
3411                         dev_dbg(dev, " RESET\n");
3412                         spin_lock_irqsave(&priv->lock, flags);
3413                         cvmx_usb_disable(&priv->usb);
3414                         if (cvmx_usb_enable(&priv->usb))
3415                                 dev_dbg(dev, "Failed to enable the port\n");
3416                         spin_unlock_irqrestore(&priv->lock, flags);
3417                         return 0;
3418                 case USB_PORT_FEAT_INDICATOR:
3419                         dev_dbg(dev, " INDICATOR\n");
3420                         /* Not supported */
3421                         break;
3422                 default:
3423                         dev_dbg(dev, " UNKNOWN\n");
3424                         return -EINVAL;
3425                 }
3426                 break;
3427         default:
3428                 dev_dbg(dev, "Unknown root hub request\n");
3429                 return -EINVAL;
3430         }
3431         return 0;
3432 }
3433
3434
3435 static const struct hc_driver octeon_hc_driver = {
3436         .description            = "Octeon USB",
3437         .product_desc           = "Octeon Host Controller",
3438         .hcd_priv_size          = sizeof(struct octeon_hcd),
3439         .irq                    = octeon_usb_irq,
3440         .flags                  = HCD_MEMORY | HCD_USB2,
3441         .start                  = octeon_usb_start,
3442         .stop                   = octeon_usb_stop,
3443         .urb_enqueue            = octeon_usb_urb_enqueue,
3444         .urb_dequeue            = octeon_usb_urb_dequeue,
3445         .endpoint_disable       = octeon_usb_endpoint_disable,
3446         .get_frame_number       = octeon_usb_get_frame_number,
3447         .hub_status_data        = octeon_usb_hub_status_data,
3448         .hub_control            = octeon_usb_hub_control,
3449 };
3450
3451
3452 static int octeon_usb_driver_probe(struct device *dev)
3453 {
3454         int status;
3455         int usb_num = to_platform_device(dev)->id;
3456         int irq = platform_get_irq(to_platform_device(dev), 0);
3457         struct octeon_hcd *priv;
3458         struct usb_hcd *hcd;
3459         unsigned long flags;
3460
3461         /*
3462          * Set the DMA mask to 64bits so we get buffers already translated for
3463          * DMA.
3464          */
3465         dev->coherent_dma_mask = ~0;
3466         dev->dma_mask = &dev->coherent_dma_mask;
3467
3468         hcd = usb_create_hcd(&octeon_hc_driver, dev, dev_name(dev));
3469         if (!hcd) {
3470                 dev_dbg(dev, "Failed to allocate memory for HCD\n");
3471                 return -1;
3472         }
3473         hcd->uses_new_polling = 1;
3474         priv = (struct octeon_hcd *)hcd->hcd_priv;
3475
3476         spin_lock_init(&priv->lock);
3477
3478         tasklet_init(&priv->dequeue_tasklet, octeon_usb_urb_dequeue_work, (unsigned long)priv);
3479         INIT_LIST_HEAD(&priv->dequeue_list);
3480
3481         status = cvmx_usb_initialize(&priv->usb, usb_num);
3482         if (status) {
3483                 dev_dbg(dev, "USB initialization failed with %d\n", status);
3484                 kfree(hcd);
3485                 return -1;
3486         }
3487
3488         /* This delay is needed for CN3010, but I don't know why... */
3489         mdelay(10);
3490
3491         spin_lock_irqsave(&priv->lock, flags);
3492         cvmx_usb_poll(&priv->usb);
3493         spin_unlock_irqrestore(&priv->lock, flags);
3494
3495         status = usb_add_hcd(hcd, irq, IRQF_SHARED);
3496         if (status) {
3497                 dev_dbg(dev, "USB add HCD failed with %d\n", status);
3498                 kfree(hcd);
3499                 return -1;
3500         }
3501         device_wakeup_enable(hcd->self.controller);
3502
3503         dev_dbg(dev, "Registered HCD for port %d on irq %d\n", usb_num, irq);
3504
3505         return 0;
3506 }
3507
3508 static int octeon_usb_driver_remove(struct device *dev)
3509 {
3510         int status;
3511         struct usb_hcd *hcd = dev_get_drvdata(dev);
3512         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3513         unsigned long flags;
3514
3515         usb_remove_hcd(hcd);
3516         tasklet_kill(&priv->dequeue_tasklet);
3517         spin_lock_irqsave(&priv->lock, flags);
3518         status = cvmx_usb_shutdown(&priv->usb);
3519         spin_unlock_irqrestore(&priv->lock, flags);
3520         if (status)
3521                 dev_dbg(dev, "USB shutdown failed with %d\n", status);
3522
3523         kfree(hcd);
3524
3525         return 0;
3526 }
3527
3528 static struct device_driver octeon_usb_driver = {
3529         .name   = "OcteonUSB",
3530         .bus    = &platform_bus_type,
3531         .probe  = octeon_usb_driver_probe,
3532         .remove = octeon_usb_driver_remove,
3533 };
3534
3535
3536 #define MAX_USB_PORTS   10
3537 static struct platform_device *pdev_glob[MAX_USB_PORTS];
3538 static int octeon_usb_registered;
3539 static int __init octeon_usb_module_init(void)
3540 {
3541         int num_devices = cvmx_usb_get_num_ports();
3542         int device;
3543
3544         if (usb_disabled() || num_devices == 0)
3545                 return -ENODEV;
3546
3547         if (driver_register(&octeon_usb_driver))
3548                 return -ENOMEM;
3549
3550         octeon_usb_registered = 1;
3551
3552         /*
3553          * Only cn52XX and cn56XX have DWC_OTG USB hardware and the
3554          * IOB priority registers.  Under heavy network load USB
3555          * hardware can be starved by the IOB causing a crash.  Give
3556          * it a priority boost if it has been waiting more than 400
3557          * cycles to avoid this situation.
3558          *
3559          * Testing indicates that a cnt_val of 8192 is not sufficient,
3560          * but no failures are seen with 4096.  We choose a value of
3561          * 400 to give a safety factor of 10.
3562          */
3563         if (OCTEON_IS_MODEL(OCTEON_CN52XX) || OCTEON_IS_MODEL(OCTEON_CN56XX)) {
3564                 union cvmx_iob_n2c_l2c_pri_cnt pri_cnt;
3565
3566                 pri_cnt.u64 = 0;
3567                 pri_cnt.s.cnt_enb = 1;
3568                 pri_cnt.s.cnt_val = 400;
3569                 cvmx_write_csr(CVMX_IOB_N2C_L2C_PRI_CNT, pri_cnt.u64);
3570         }
3571
3572         for (device = 0; device < num_devices; device++) {
3573                 struct resource irq_resource;
3574                 struct platform_device *pdev;
3575                 memset(&irq_resource, 0, sizeof(irq_resource));
3576                 irq_resource.start = (device == 0) ? OCTEON_IRQ_USB0 : OCTEON_IRQ_USB1;
3577                 irq_resource.end = irq_resource.start;
3578                 irq_resource.flags = IORESOURCE_IRQ;
3579                 pdev = platform_device_register_simple((char *)octeon_usb_driver.  name, device, &irq_resource, 1);
3580                 if (IS_ERR(pdev)) {
3581                         driver_unregister(&octeon_usb_driver);
3582                         octeon_usb_registered = 0;
3583                         return PTR_ERR(pdev);
3584                 }
3585                 if (device < MAX_USB_PORTS)
3586                         pdev_glob[device] = pdev;
3587
3588         }
3589         return 0;
3590 }
3591
3592 static void __exit octeon_usb_module_cleanup(void)
3593 {
3594         int i;
3595
3596         for (i = 0; i < MAX_USB_PORTS; i++)
3597                 if (pdev_glob[i]) {
3598                         platform_device_unregister(pdev_glob[i]);
3599                         pdev_glob[i] = NULL;
3600                 }
3601         if (octeon_usb_registered)
3602                 driver_unregister(&octeon_usb_driver);
3603 }
3604
3605 MODULE_LICENSE("GPL");
3606 MODULE_AUTHOR("Cavium Networks <support@caviumnetworks.com>");
3607 MODULE_DESCRIPTION("Cavium Networks Octeon USB Host driver.");
3608 module_init(octeon_usb_module_init);
3609 module_exit(octeon_usb_module_cleanup);