drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2011 Intel Corporation.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/inet_lro.h>
  42 #include <linux/slab.h>
  43
  44 #include "nes.h"
  45
  46 static unsigned int nes_lro_max_aggr = NES_LRO_MAX_AGGR;
  47 module_param(nes_lro_max_aggr, uint, 0444);
  48 MODULE_PARM_DESC(nes_lro_max_aggr, "NIC LRO max packet aggregation");
  49
  50 static int wide_ppm_offset;
  51 module_param(wide_ppm_offset, int, 0644);
  52 MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm");
  53
  54 static u32 crit_err_count;
  55 u32 int_mod_timer_init;
  56 u32 int_mod_cq_depth_256;
  57 u32 int_mod_cq_depth_128;
  58 u32 int_mod_cq_depth_32;
  59 u32 int_mod_cq_depth_24;
  60 u32 int_mod_cq_depth_16;
  61 u32 int_mod_cq_depth_4;
  62 u32 int_mod_cq_depth_1;
  63 static const u8 nes_max_critical_error_count = 100;
  64 #include "nes_cm.h"
  65
  66 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  67 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  68 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  69                                 struct nes_adapter *nesadapter, u8  OneG_Mode);
  70 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  71 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  72 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  73 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  74                                    struct nes_hw_aeqe *aeqe);
  75 static void process_critical_error(struct nes_device *nesdev);
  76 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  77 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  78 static void nes_terminate_timeout(unsigned long context);
  79 static void nes_terminate_start_timer(struct nes_qp *nesqp);
  80
  81 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  82 static unsigned char *nes_iwarp_state_str[] = {
  83         "Non-Existent",
  84         "Idle",
  85         "RTS",
  86         "Closing",
  87         "RSVD1",
  88         "Terminate",
  89         "Error",
  90         "RSVD2",
  91 };
  92
  93 static unsigned char *nes_tcp_state_str[] = {
  94         "Non-Existent",
  95         "Closed",
  96         "Listen",
  97         "SYN Sent",
  98         "SYN Rcvd",
  99         "Established",
 100         "Close Wait",
 101         "FIN Wait 1",
 102         "Closing",
 103         "Last Ack",
 104         "FIN Wait 2",
 105         "Time Wait",
 106         "RSVD1",
 107         "RSVD2",
 108         "RSVD3",
 109         "RSVD4",
 110 };
 111 #endif
 112
 113 static inline void print_ip(struct nes_cm_node *cm_node)
 114 {
 115         unsigned char *rem_addr;
 116         if (cm_node) {
 117                 rem_addr = (unsigned char *)&cm_node->rem_addr;
 118                 printk(KERN_ERR PFX "Remote IP addr: %pI4\n", rem_addr);
 119         }
 120 }
 121
 122 /**
 123  * nes_nic_init_timer_defaults
 124  */
 125 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 126 {
 127         unsigned long flags;
 128         struct nes_adapter *nesadapter = nesdev->nesadapter;
 129         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 130
 131         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 132
 133         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 134         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 135         if (jumbomode) {
 136                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 137                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 138                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 139         } else {
 140                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 141                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 142                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 143         }
 144
 145         /* todo use netdev->mtu to set thresholds */
 146         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 147 }
 148
 149
 150 /**
 151  * nes_nic_init_timer
 152  */
 153 static void  nes_nic_init_timer(struct nes_device *nesdev)
 154 {
 155         unsigned long flags;
 156         struct nes_adapter *nesadapter = nesdev->nesadapter;
 157         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 158
 159         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 160
 161         if (shared_timer->timer_in_use_old == 0) {
 162                 nesdev->deepcq_count = 0;
 163                 shared_timer->timer_direction_upward = 0;
 164                 shared_timer->timer_direction_downward = 0;
 165                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 166                 shared_timer->timer_in_use_old = 0;
 167
 168         }
 169         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 170                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 171                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 172                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 173         }
 174         /* todo use netdev->mtu to set thresholds */
 175         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 176 }
 177
 178
 179 /**
 180  * nes_nic_tune_timer
 181  */
 182 static void nes_nic_tune_timer(struct nes_device *nesdev)
 183 {
 184         unsigned long flags;
 185         struct nes_adapter *nesadapter = nesdev->nesadapter;
 186         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 187         u16 cq_count = nesdev->currcq_count;
 188
 189         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 190
 191         if (shared_timer->cq_count_old <= cq_count)
 192                 shared_timer->cq_direction_downward = 0;
 193         else
 194                 shared_timer->cq_direction_downward++;
 195         shared_timer->cq_count_old = cq_count;
 196         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 197                 if (cq_count <= shared_timer->threshold_low &&
 198                     shared_timer->threshold_low > 4) {
 199                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 200                         shared_timer->cq_direction_downward=0;
 201                         nesdev->currcq_count = 0;
 202                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 203                         return;
 204                 }
 205         }
 206
 207         if (cq_count > 1) {
 208                 nesdev->deepcq_count += cq_count;
 209                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 210                         shared_timer->timer_direction_upward++;
 211                         shared_timer->timer_direction_downward = 0;
 212                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 213                         shared_timer->timer_direction_upward = 0;
 214                         shared_timer->timer_direction_downward = 0;
 215                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 216                         shared_timer->timer_direction_downward++;
 217                         shared_timer->timer_direction_upward = 0;
 218                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 219                         shared_timer->timer_in_use -= 2;
 220                         shared_timer->timer_direction_upward = 0;
 221                         shared_timer->timer_direction_downward++;
 222                 } else {
 223                         shared_timer->timer_in_use -= 4;
 224                         shared_timer->timer_direction_upward = 0;
 225                         shared_timer->timer_direction_downward++;
 226                 }
 227
 228                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 229                         shared_timer->timer_in_use += 3;
 230                         shared_timer->timer_direction_upward = 0;
 231                         shared_timer->timer_direction_downward = 0;
 232                 }
 233                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 234                         shared_timer->timer_in_use -= 4 ;
 235                         shared_timer->timer_direction_downward = 0;
 236                         shared_timer->timer_direction_upward = 0;
 237                 }
 238         }
 239
 240         /* boundary checking */
 241         if (shared_timer->timer_in_use > shared_timer->threshold_high)
 242                 shared_timer->timer_in_use = shared_timer->threshold_high;
 243         else if (shared_timer->timer_in_use < shared_timer->threshold_low)
 244                 shared_timer->timer_in_use = shared_timer->threshold_low;
 245
 246         nesdev->currcq_count = 0;
 247
 248         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 249 }
 250
 251
 252 /**
 253  * nes_init_adapter - initialize adapter
 254  */
 255 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 256         struct nes_adapter *nesadapter = NULL;
 257         unsigned long num_pds;
 258         u32 u32temp;
 259         u32 port_count;
 260         u16 max_rq_wrs;
 261         u16 max_sq_wrs;
 262         u32 max_mr;
 263         u32 max_256pbl;
 264         u32 max_4kpbl;
 265         u32 max_qp;
 266         u32 max_irrq;
 267         u32 max_cq;
 268         u32 hte_index_mask;
 269         u32 adapter_size;
 270         u32 arp_table_size;
 271         u16 vendor_id;
 272         u16 device_id;
 273         u8  OneG_Mode;
 274         u8  func_index;
 275
 276         /* search the list of existing adapters */
 277         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 278                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 279                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 280                                 nesdev->pcidev->devfn,
 281                                 PCI_SLOT(nesadapter->devfn),
 282                                 nesadapter->bus_number,
 283                                 PCI_SLOT(nesdev->pcidev->devfn),
 284                                 nesdev->pcidev->bus->number );
 285                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 286                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 287                         nesadapter->ref_count++;
 288                         return nesadapter;
 289                 }
 290         }
 291
 292         /* no adapter found */
 293         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 294         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 295                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 296                                 hw_rev);
 297                 return NULL;
 298         }
 299
 300         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 301                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 302                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 303                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 304                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 305
 306         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 307
 308
 309         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 310                 return NULL;
 311
 312         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 313         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 314
 315         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 316         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 317                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 318                                 max_qp, u32temp);
 319                 max_qp = (u32)1 << (u32temp & 0x001f);
 320         }
 321
 322         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 323         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 324                         max_qp, hte_index_mask);
 325
 326         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 327
 328         max_irrq = 1 << (u32temp & 0x001f);
 329
 330         if (max_qp > max_irrq) {
 331                 max_qp = max_irrq;
 332                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 333                                 max_qp);
 334         }
 335
 336         /* there should be no reason to allocate more pds than qps */
 337         if (num_pds > max_qp)
 338                 num_pds = max_qp;
 339
 340         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 341         max_mr = (u32)8192 << (u32temp & 0x7);
 342
 343         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 344         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 345         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 346         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 347
 348         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 349         arp_table_size = 1 << u32temp;
 350
 351         adapter_size = (sizeof(struct nes_adapter) +
 352                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 353         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 354         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 355         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 356         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 357         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 358         adapter_size += sizeof(struct nes_qp **) * max_qp;
 359
 360         /* allocate a new adapter struct */
 361         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 362         if (nesadapter == NULL) {
 363                 return NULL;
 364         }
 365
 366         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 367                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 368
 369         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 370                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 371                 kfree(nesadapter);
 372                 return NULL;
 373         }
 374
 375         nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
 376                                 (nesadapter->mac_addr_low >> 24);
 377
 378         pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
 379                                  PCI_DEVICE_ID, &device_id);
 380         nesadapter->vendor_part_id = device_id;
 381
 382         if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
 383                                                         OneG_Mode)) {
 384                 kfree(nesadapter);
 385                 return NULL;
 386         }
 387         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 388
 389         memset(nesadapter->pft_mcast_map, 255,
 390                sizeof nesadapter->pft_mcast_map);
 391
 392         /* populate the new nesadapter */
 393         nesadapter->devfn = nesdev->pcidev->devfn;
 394         nesadapter->bus_number = nesdev->pcidev->bus->number;
 395         nesadapter->ref_count = 1;
 396         nesadapter->timer_int_req = 0xffff0000;
 397         nesadapter->OneG_Mode = OneG_Mode;
 398         nesadapter->doorbell_start = nesdev->doorbell_region;
 399
 400         /* nesadapter->tick_delta = clk_divisor; */
 401         nesadapter->hw_rev = hw_rev;
 402         nesadapter->port_count = port_count;
 403
 404         nesadapter->max_qp = max_qp;
 405         nesadapter->hte_index_mask = hte_index_mask;
 406         nesadapter->max_irrq = max_irrq;
 407         nesadapter->max_mr = max_mr;
 408         nesadapter->max_256pbl = max_256pbl - 1;
 409         nesadapter->max_4kpbl = max_4kpbl - 1;
 410         nesadapter->max_cq = max_cq;
 411         nesadapter->free_256pbl = max_256pbl - 1;
 412         nesadapter->free_4kpbl = max_4kpbl - 1;
 413         nesadapter->max_pd = num_pds;
 414         nesadapter->arp_table_size = arp_table_size;
 415
 416         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 417         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 418                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 419                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 420                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 421         } else {
 422                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 423                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 424                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 425                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 426         }
 427         /* Setup and enable the periodic timer */
 428         if (nesadapter->et_rx_coalesce_usecs_irq)
 429                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 430                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 431         else
 432                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 433
 434         nesadapter->base_pd = 1;
 435
 436         nesadapter->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
 437                                        IB_DEVICE_MEM_WINDOW |
 438                                        IB_DEVICE_MEM_MGT_EXTENSIONS;
 439
 440         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 441                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 442         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 443         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 444         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 445         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 446         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 447
 448
 449         /* mark the usual suspect QPs, MR and CQs as in use */
 450         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 451                 set_bit(u32temp, nesadapter->allocated_qps);
 452                 set_bit(u32temp, nesadapter->allocated_cqs);
 453         }
 454         set_bit(0, nesadapter->allocated_mrs);
 455
 456         for (u32temp = 0; u32temp < 20; u32temp++)
 457                 set_bit(u32temp, nesadapter->allocated_pds);
 458         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 459
 460         max_rq_wrs = ((u32temp >> 8) & 3);
 461         switch (max_rq_wrs) {
 462                 case 0:
 463                         max_rq_wrs = 4;
 464                         break;
 465                 case 1:
 466                         max_rq_wrs = 16;
 467                         break;
 468                 case 2:
 469                         max_rq_wrs = 32;
 470                         break;
 471                 case 3:
 472                         max_rq_wrs = 512;
 473                         break;
 474         }
 475
 476         max_sq_wrs = (u32temp & 3);
 477         switch (max_sq_wrs) {
 478                 case 0:
 479                         max_sq_wrs = 4;
 480                         break;
 481                 case 1:
 482                         max_sq_wrs = 16;
 483                         break;
 484                 case 2:
 485                         max_sq_wrs = 32;
 486                         break;
 487                 case 3:
 488                         max_sq_wrs = 512;
 489                         break;
 490         }
 491         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 492         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 493
 494         nesadapter->max_sge = 4;
 495         nesadapter->max_cqe = 32766;
 496
 497         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 498                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 499                 kfree(nesadapter);
 500                 return NULL;
 501         }
 502
 503         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 504         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 505                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 506
 507         /* setup port configuration */
 508         if (nesadapter->port_count == 1) {
 509                 nesadapter->log_port = 0x00000000;
 510                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 511                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 512                 else
 513                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 514         } else {
 515                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 516                         nesadapter->log_port = 0x000000D8;
 517                 } else {
 518                         if (nesadapter->port_count == 2)
 519                                 nesadapter->log_port = 0x00000044;
 520                         else
 521                                 nesadapter->log_port = 0x000000e4;
 522                 }
 523                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 524         }
 525
 526         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
 527                                                 nesadapter->log_port);
 528         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 529                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 530
 531         spin_lock_init(&nesadapter->resource_lock);
 532         spin_lock_init(&nesadapter->phy_lock);
 533         spin_lock_init(&nesadapter->pbl_lock);
 534         spin_lock_init(&nesadapter->periodic_timer_lock);
 535
 536         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 537         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 538         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 539         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 540
 541         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 542                 u32 pcs_control_status0, pcs_control_status1;
 543                 u32 reset_value;
 544                 u32 i = 0;
 545                 u32 int_cnt = 0;
 546                 u32 ext_cnt = 0;
 547                 unsigned long flags;
 548                 u32 j = 0;
 549
 550                 pcs_control_status0 = nes_read_indexed(nesdev,
 551                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 552                 pcs_control_status1 = nes_read_indexed(nesdev,
 553                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 554
 555                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 556                         pcs_control_status0 = nes_read_indexed(nesdev,
 557                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 558                         pcs_control_status1 = nes_read_indexed(nesdev,
 559                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 560                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 561                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 562                                 int_cnt++;
 563                         msleep(1);
 564                 }
 565                 if (int_cnt > 1) {
 566                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 567                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
 568                         mh_detected++;
 569                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 570                         reset_value |= 0x0000003d;
 571                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 572
 573                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 574                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 575                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 576
 577                         pcs_control_status0 = nes_read_indexed(nesdev,
 578                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 579                         pcs_control_status1 = nes_read_indexed(nesdev,
 580                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 581
 582                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 583                                 pcs_control_status0 = nes_read_indexed(nesdev,
 584                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 585                                 pcs_control_status1 = nes_read_indexed(nesdev,
 586                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 587                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 588                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 589                                         if (++ext_cnt > int_cnt) {
 590                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 591                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 592                                                                 0x0000F088);
 593                                                 mh_detected++;
 594                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 595                                                 reset_value |= 0x0000003d;
 596                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 597
 598                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 599                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 600                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 601                                                 break;
 602                                         }
 603                                 }
 604                                 msleep(1);
 605                         }
 606                 }
 607         }
 608
 609         if (nesadapter->hw_rev == NE020_REV) {
 610                 init_timer(&nesadapter->mh_timer);
 611                 nesadapter->mh_timer.function = nes_mh_fix;
 612                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 613                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 614                 add_timer(&nesadapter->mh_timer);
 615         } else {
 616                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 617         }
 618
 619         init_timer(&nesadapter->lc_timer);
 620         nesadapter->lc_timer.function = nes_clc;
 621         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 622         nesadapter->lc_timer.data = (unsigned long)nesdev;
 623         add_timer(&nesadapter->lc_timer);
 624
 625         list_add_tail(&nesadapter->list, &nes_adapter_list);
 626
 627         for (func_index = 0; func_index < 8; func_index++) {
 628                 pci_bus_read_config_word(nesdev->pcidev->bus,
 629                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 630                                         func_index), 0, &vendor_id);
 631                 if (vendor_id == 0xffff)
 632                         break;
 633         }
 634         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 635                 func_index, pci_name(nesdev->pcidev));
 636         nesadapter->adapter_fcn_count = func_index;
 637
 638         return nesadapter;
 639 }
 640
 641
 642 /**
 643  * nes_reset_adapter_ne020
 644  */
 645 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 646 {
 647         u32 port_count;
 648         u32 u32temp;
 649         u32 i;
 650
 651         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 652         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 653         /* TODO: assuming that both SERDES are set the same for now */
 654         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 655         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 656                         u32temp, port_count);
 657         if (*OneG_Mode)
 658                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 659         u32temp &= 0xff00ffc0;
 660         switch (port_count) {
 661                 case 1:
 662                         u32temp |= 0x00ee0000;
 663                         break;
 664                 case 2:
 665                         u32temp |= 0x00cc0000;
 666                         break;
 667                 case 4:
 668                         u32temp |= 0x00000000;
 669                         break;
 670                 default:
 671                         return 0;
 672                         break;
 673         }
 674
 675         /* check and do full reset if needed */
 676         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 677                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 678                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 679
 680                 i = 0;
 681                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 682                         mdelay(1);
 683                 if (i > 10000) {
 684                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 685                         return 0;
 686                 }
 687
 688                 i = 0;
 689                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 690                         mdelay(1);
 691                 if (i > 10000) {
 692                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 693                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 694                         return 0;
 695                 }
 696         }
 697
 698         /* port reset */
 699         switch (port_count) {
 700                 case 1:
 701                         u32temp |= 0x00ee0010;
 702                         break;
 703                 case 2:
 704                         u32temp |= 0x00cc0030;
 705                         break;
 706                 case 4:
 707                         u32temp |= 0x00000030;
 708                         break;
 709         }
 710
 711         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 712         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 713
 714         i = 0;
 715         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 716                 mdelay(1);
 717         if (i > 10000) {
 718                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 719                 return 0;
 720         }
 721
 722         /* serdes 0 */
 723         i = 0;
 724         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 725                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 726                 mdelay(1);
 727         if (i > 5000) {
 728                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 729                 return 0;
 730         }
 731
 732         /* serdes 1 */
 733         if (port_count > 1) {
 734                 i = 0;
 735                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 736                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 737                         mdelay(1);
 738                 if (i > 5000) {
 739                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 740                         return 0;
 741                 }
 742         }
 743
 744         return port_count;
 745 }
 746
 747
 748 /**
 749  * nes_init_serdes
 750  */
 751 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 752                                 struct nes_adapter *nesadapter, u8  OneG_Mode)
 753 {
 754         int i;
 755         u32 u32temp;
 756         u32 sds;
 757
 758         if (hw_rev != NE020_REV) {
 759                 /* init serdes 0 */
 760                 switch (nesadapter->phy_type[0]) {
 761                 case NES_PHY_TYPE_CX4:
 762                         if (wide_ppm_offset)
 763                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA);
 764                         else
 765                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 766                         break;
 767                 case NES_PHY_TYPE_KR:
 768                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 769                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 770                         break;
 771                 case NES_PHY_TYPE_PUMA_1G:
 772                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 773                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
 774                         sds |= 0x00000100;
 775                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds);
 776                         break;
 777                 default:
 778                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 779                         break;
 780                 }
 781
 782                 if (!OneG_Mode)
 783                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 784
 785                 if (port_count < 2)
 786                         return 0;
 787
 788                 /* init serdes 1 */
 789                 if (!(OneG_Mode && (nesadapter->phy_type[1] != NES_PHY_TYPE_PUMA_1G)))
 790                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 791
 792                 switch (nesadapter->phy_type[1]) {
 793                 case NES_PHY_TYPE_ARGUS:
 794                 case NES_PHY_TYPE_SFP_D:
 795                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 796                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 797                         break;
 798                 case NES_PHY_TYPE_CX4:
 799                         if (wide_ppm_offset)
 800                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA);
 801                         break;
 802                 case NES_PHY_TYPE_KR:
 803                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 804                         break;
 805                 case NES_PHY_TYPE_PUMA_1G:
 806                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 807                         sds |= 0x000000100;
 808                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 809                 }
 810                 if (!OneG_Mode) {
 811                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 812                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 813                         sds &= 0xFFFFFFBF;
 814                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 815                 }
 816         } else {
 817                 /* init serdes 0 */
 818                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 819                 i = 0;
 820                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 821                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 822                         mdelay(1);
 823                 if (i > 5000) {
 824                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 825                         return 1;
 826                 }
 827                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 828                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 829                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 830                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 831                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 832                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 833                 if (OneG_Mode)
 834                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 835                 else
 836                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 837
 838                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 839                 if (port_count > 1) {
 840                         /* init serdes 1 */
 841                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 842                         i = 0;
 843                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 844                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 845                                 mdelay(1);
 846                         if (i > 5000) {
 847                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 848                                 /* return 1; */
 849                         }
 850                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 851                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 852                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 853                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 854                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 855                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 856                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 857                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 858                 }
 859         }
 860         return 0;
 861 }
 862
 863
 864 /**
 865  * nes_init_csr_ne020
 866  * Initialize registers for ne020 hardware
 867  */
 868 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 869 {
 870         u32 u32temp;
 871
 872         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 873
 874         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 875         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 876         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 877         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 878         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 879         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 880         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 881         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 882
 883         /* TODO: move these MAC register settings to NIC bringup */
 884         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 885         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 886         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 887         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 888         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 889         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 890         if (port_count > 1) {
 891                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 892                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 893                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 894                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 895                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 896                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 897                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 898         }
 899         if (port_count > 2) {
 900                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 901                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 902                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 903                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 904                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 905                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 906                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 907
 908                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 909                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 910                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 911                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 912                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 913                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 914                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 915         }
 916
 917         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 918         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 919         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
 920                                                          0x00000001);
 921         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 922         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 923         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 924         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 925         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 926
 927         /* TODO: move this to code, get from EEPROM */
 928         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 929         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 930         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 931
 932         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 933         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 934
 935         if (hw_rev != NE020_REV) {
 936                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 937                 u32temp |= 0x80000000;
 938                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 939                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 940                 u32temp &= 0x7fffffff;
 941                 u32temp |= 0x7fff0010;
 942                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 943                 if (port_count > 1) {
 944                         u32temp = nes_read_indexed(nesdev, 0x000023f8);
 945                         u32temp &= 0x7fffffff;
 946                         u32temp |= 0x7fff0010;
 947                         nes_write_indexed(nesdev, 0x000023f8, u32temp);
 948                 }
 949         }
 950 }
 951
 952
 953 /**
 954  * nes_destroy_adapter - destroy the adapter structure
 955  */
 956 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 957 {
 958         struct nes_adapter *tmp_adapter;
 959
 960         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 961                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 962                                 tmp_adapter);
 963         }
 964
 965         nesadapter->ref_count--;
 966         if (!nesadapter->ref_count) {
 967                 if (nesadapter->hw_rev == NE020_REV) {
 968                         del_timer(&nesadapter->mh_timer);
 969                 }
 970                 del_timer(&nesadapter->lc_timer);
 971
 972                 list_del(&nesadapter->list);
 973                 kfree(nesadapter);
 974         }
 975 }
 976
 977
 978 /**
 979  * nes_init_cqp
 980  */
 981 int nes_init_cqp(struct nes_device *nesdev)
 982 {
 983         struct nes_adapter *nesadapter = nesdev->nesadapter;
 984         struct nes_hw_cqp_qp_context *cqp_qp_context;
 985         struct nes_hw_cqp_wqe *cqp_wqe;
 986         struct nes_hw_ceq *ceq;
 987         struct nes_hw_ceq *nic_ceq;
 988         struct nes_hw_aeq *aeq;
 989         void *vmem;
 990         dma_addr_t pmem;
 991         u32 count=0;
 992         u32 cqp_head;
 993         u64 u64temp;
 994         u32 u32temp;
 995
 996         /* allocate CQP memory */
 997         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 998         /* SQ is 512 byte aligned, others are 256 byte aligned */
 999         nesdev->cqp_mem_size = 512 +
1000                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
1001                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
1002                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
1003                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
1004                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
1005                         sizeof(struct nes_hw_cqp_qp_context);
1006
1007         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1008                         &nesdev->cqp_pbase);
1009         if (!nesdev->cqp_vbase) {
1010                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
1011                 return -ENOMEM;
1012         }
1013         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
1014
1015         /* Allocate a twice the number of CQP requests as the SQ size */
1016         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
1017                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
1018         if (nesdev->nes_cqp_requests == NULL) {
1019                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
1020                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1021                                 nesdev->cqp.sq_pbase);
1022                 return -ENOMEM;
1023         }
1024
1025         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
1026                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
1027
1028         spin_lock_init(&nesdev->cqp.lock);
1029         init_waitqueue_head(&nesdev->cqp.waitq);
1030
1031         /* Setup Various Structures */
1032         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
1033                         ~(unsigned long)(512 - 1));
1034         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
1035                         ~(unsigned long long)(512 - 1));
1036
1037         nesdev->cqp.sq_vbase = vmem;
1038         nesdev->cqp.sq_pbase = pmem;
1039         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
1040         nesdev->cqp.sq_head = 0;
1041         nesdev->cqp.sq_tail = 0;
1042         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
1043
1044         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1045         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1046
1047         nesdev->ccq.cq_vbase = vmem;
1048         nesdev->ccq.cq_pbase = pmem;
1049         nesdev->ccq.cq_size = NES_CCQ_SIZE;
1050         nesdev->ccq.cq_head = 0;
1051         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
1052         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
1053
1054         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1055         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1056
1057         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1058         ceq = &nesadapter->ceq[nesdev->ceq_index];
1059         ceq->ceq_vbase = vmem;
1060         ceq->ceq_pbase = pmem;
1061         ceq->ceq_size = NES_CCEQ_SIZE;
1062         ceq->ceq_head = 0;
1063
1064         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1065         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1066
1067         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1068         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1069         nic_ceq->ceq_vbase = vmem;
1070         nic_ceq->ceq_pbase = pmem;
1071         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1072         nic_ceq->ceq_head = 0;
1073
1074         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1075         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1076
1077         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1078         aeq->aeq_vbase = vmem;
1079         aeq->aeq_pbase = pmem;
1080         aeq->aeq_size = nesadapter->max_qp;
1081         aeq->aeq_head = 0;
1082
1083         /* Setup QP Context */
1084         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1085         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1086
1087         cqp_qp_context = vmem;
1088         cqp_qp_context->context_words[0] =
1089                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1090         cqp_qp_context->context_words[1] = 0;
1091         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1092         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1093
1094
1095         /* Write the address to Create CQP */
1096         if ((sizeof(dma_addr_t) > 4)) {
1097                 nes_write_indexed(nesdev,
1098                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1099                                 ((u64)pmem) >> 32);
1100         } else {
1101                 nes_write_indexed(nesdev,
1102                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1103         }
1104         nes_write_indexed(nesdev,
1105                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1106                         (u32)pmem);
1107
1108         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1109         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1110
1111         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1112                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1113                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1114         }
1115
1116         /* Write Create CCQ WQE */
1117         cqp_head = nesdev->cqp.sq_head++;
1118         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1119         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1120         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1121                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1122                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1123         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1124                             (nesdev->ccq.cq_number |
1125                              ((u32)nesdev->ceq_index << 16)));
1126         u64temp = (u64)nesdev->ccq.cq_pbase;
1127         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1128         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1129         u64temp = (unsigned long)&nesdev->ccq;
1130         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1131                         cpu_to_le32((u32)(u64temp >> 1));
1132         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1133                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1134         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1135
1136         /* Write Create CEQ WQE */
1137         cqp_head = nesdev->cqp.sq_head++;
1138         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1139         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1140         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1141                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1142         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1143         u64temp = (u64)ceq->ceq_pbase;
1144         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1145
1146         /* Write Create AEQ WQE */
1147         cqp_head = nesdev->cqp.sq_head++;
1148         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1149         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1150         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1151                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1152         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1153         u64temp = (u64)aeq->aeq_pbase;
1154         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1155
1156         /* Write Create NIC CEQ WQE */
1157         cqp_head = nesdev->cqp.sq_head++;
1158         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1159         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1160         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1161                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1162         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1163         u64temp = (u64)nic_ceq->ceq_pbase;
1164         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1165
1166         /* Poll until CCQP done */
1167         count = 0;
1168         do {
1169                 if (count++ > 1000) {
1170                         printk(KERN_ERR PFX "Error creating CQP\n");
1171                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1172                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1173                         return -1;
1174                 }
1175                 udelay(10);
1176         } while (!(nes_read_indexed(nesdev,
1177                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1178
1179         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1180                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1181
1182         u32temp = 0x04800000;
1183         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1184
1185         /* wait for the CCQ, CEQ, and AEQ to get created */
1186         count = 0;
1187         do {
1188                 if (count++ > 1000) {
1189                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1190                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1191                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1192                         return -1;
1193                 }
1194                 udelay(10);
1195         } while (((nes_read_indexed(nesdev,
1196                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1197
1198         /* dump the QP status value */
1199         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1200                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1201
1202         nesdev->cqp.sq_tail++;
1203
1204         return 0;
1205 }
1206
1207
1208 /**
1209  * nes_destroy_cqp
1210  */
1211 int nes_destroy_cqp(struct nes_device *nesdev)
1212 {
1213         struct nes_hw_cqp_wqe *cqp_wqe;
1214         u32 count = 0;
1215         u32 cqp_head;
1216         unsigned long flags;
1217
1218         do {
1219                 if (count++ > 1000)
1220                         break;
1221                 udelay(10);
1222         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1223
1224         /* Reset CCQ */
1225         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1226                         nesdev->ccq.cq_number);
1227
1228         /* Disable device interrupts */
1229         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1230
1231         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1232
1233         /* Destroy the AEQ */
1234         cqp_head = nesdev->cqp.sq_head++;
1235         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1236         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1237         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1238                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1239         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1240
1241         /* Destroy the NIC CEQ */
1242         cqp_head = nesdev->cqp.sq_head++;
1243         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1244         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1245         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1246                         ((u32)nesdev->nic_ceq_index << 8));
1247
1248         /* Destroy the CEQ */
1249         cqp_head = nesdev->cqp.sq_head++;
1250         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1251         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1252         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1253                         (nesdev->ceq_index << 8));
1254
1255         /* Destroy the CCQ */
1256         cqp_head = nesdev->cqp.sq_head++;
1257         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1258         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1259         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1260         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1261                         ((u32)nesdev->ceq_index << 16));
1262
1263         /* Destroy CQP */
1264         cqp_head = nesdev->cqp.sq_head++;
1265         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1266         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1267         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1268                         NES_CQP_QP_TYPE_CQP);
1269         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1270
1271         barrier();
1272         /* Ring doorbell (5 WQEs) */
1273         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1274
1275         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1276
1277         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1278         count = 0;
1279         do {
1280                 if (count++ > 1000) {
1281                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1282                                         PCI_FUNC(nesdev->pcidev->devfn));
1283                         break;
1284                 }
1285                 udelay(10);
1286         } while (((nes_read_indexed(nesdev,
1287                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1288
1289         /* dump the QP status value */
1290         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1291                         PCI_FUNC(nesdev->pcidev->devfn),
1292                         nes_read_indexed(nesdev,
1293                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1294
1295         kfree(nesdev->nes_cqp_requests);
1296
1297         /* Free the control structures */
1298         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1299                         nesdev->cqp.sq_pbase);
1300
1301         return 0;
1302 }
1303
1304
1305 /**
1306  * nes_init_1g_phy
1307  */
1308 static int nes_init_1g_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1309 {
1310         u32 counter = 0;
1311         u16 phy_data;
1312         int ret = 0;
1313
1314         nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data);
1315         nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000);
1316
1317         /* Reset the PHY */
1318         nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000);
1319         udelay(100);
1320         counter = 0;
1321         do {
1322                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1323                 if (counter++ > 100) {
1324                         ret = -1;
1325                         break;
1326                 }
1327         } while (phy_data & 0x8000);
1328
1329         /* Setting no phy loopback */
1330         phy_data &= 0xbfff;
1331         phy_data |= 0x1140;
1332         nes_write_1G_phy_reg(nesdev, 0, phy_index,  phy_data);
1333         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1334         nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data);
1335         nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data);
1336
1337         /* Setting the interrupt mask */
1338         nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1339         nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee);
1340         nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1341
1342         /* turning on flow control */
1343         nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1344         nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00);
1345         nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1346
1347         /* Clear Half duplex */
1348         nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1349         nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100));
1350         nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1351
1352         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1353         nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300);
1354
1355         return ret;
1356 }
1357
1358
1359 /**
1360  * nes_init_2025_phy
1361  */
1362 static int nes_init_2025_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1363 {
1364         u32 temp_phy_data = 0;
1365         u32 temp_phy_data2 = 0;
1366         u32 counter = 0;
1367         u32 sds;
1368         u32 mac_index = nesdev->mac_index;
1369         int ret = 0;
1370         unsigned int first_attempt = 1;
1371
1372         /* Check firmware heartbeat */
1373         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1374         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1375         udelay(1500);
1376         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1377         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1378
1379         if (temp_phy_data != temp_phy_data2) {
1380                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1381                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1382                 if ((temp_phy_data & 0xff) > 0x20)
1383                         return 0;
1384                 printk(PFX "Reinitialize external PHY\n");
1385         }
1386
1387         /* no heartbeat, configure the PHY */
1388         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000);
1389         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000);
1390         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1391         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1392
1393         switch (phy_type) {
1394         case NES_PHY_TYPE_ARGUS:
1395                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1396                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1397                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1398                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008);
1399                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001);
1400                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1401                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1402
1403                 /* setup LEDs */
1404                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1405                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1406                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1407                 break;
1408
1409         case NES_PHY_TYPE_SFP_D:
1410                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1411                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1412                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x0004);
1413                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0038);
1414                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1415                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1416                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1417
1418                 /* setup LEDs */
1419                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1420                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1421                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1422                 break;
1423
1424         case NES_PHY_TYPE_KR:
1425                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1426                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1427                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1428                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0010);
1429                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1430                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0080);
1431                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1432
1433                 /* setup LEDs */
1434                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x000B);
1435                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x0003);
1436                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0004);
1437
1438                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0022, 0x406D);
1439                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0023, 0x0020);
1440                 break;
1441         }
1442
1443         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528);
1444
1445         /* Bring PHY out of reset */
1446         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002);
1447
1448         /* Check for heartbeat */
1449         counter = 0;
1450         mdelay(690);
1451         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1452         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1453         do {
1454                 if (counter++ > 150) {
1455                         printk(PFX "No PHY heartbeat\n");
1456                         break;
1457                 }
1458                 mdelay(1);
1459                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1460                 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1461         } while ((temp_phy_data2 == temp_phy_data));
1462
1463         /* wait for tracking */
1464         counter = 0;
1465         do {
1466                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1467                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1468                 if (counter++ > 300) {
1469                         if (((temp_phy_data & 0xff) == 0x0) && first_attempt) {
1470                                 first_attempt = 0;
1471                                 counter = 0;
1472                                 /* reset AMCC PHY and try again */
1473                                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x00c0);
1474                                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x0040);
1475                                 continue;
1476                         } else {
1477                                 ret = 1;
1478                                 break;
1479                         }
1480                 }
1481                 mdelay(10);
1482         } while ((temp_phy_data & 0xff) < 0x30);
1483
1484         /* setup signal integrity */
1485         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000);
1486         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE);
1487         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032);
1488         if (phy_type == NES_PHY_TYPE_KR) {
1489                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x000C);
1490         } else {
1491                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002);
1492                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063);
1493         }
1494
1495         /* reset serdes */
1496         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200);
1497         sds |= 0x1;
1498         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1499         sds &= 0xfffffffe;
1500         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1501
1502         counter = 0;
1503         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1504                         && (counter++ < 5000))
1505                 ;
1506
1507         return ret;
1508 }
1509
1510
1511 /**
1512  * nes_init_phy
1513  */
1514 int nes_init_phy(struct nes_device *nesdev)
1515 {
1516         struct nes_adapter *nesadapter = nesdev->nesadapter;
1517         u32 mac_index = nesdev->mac_index;
1518         u32 tx_config = 0;
1519         unsigned long flags;
1520         u8  phy_type = nesadapter->phy_type[mac_index];
1521         u8  phy_index = nesadapter->phy_index[mac_index];
1522         int ret = 0;
1523
1524         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1525         if (phy_type == NES_PHY_TYPE_1G) {
1526                 /* setup 1G MDIO operation */
1527                 tx_config &= 0xFFFFFFE3;
1528                 tx_config |= 0x04;
1529         } else {
1530                 /* setup 10G MDIO operation */
1531                 tx_config &= 0xFFFFFFE3;
1532                 tx_config |= 0x1D;
1533         }
1534         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1535
1536         spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags);
1537
1538         switch (phy_type) {
1539         case NES_PHY_TYPE_1G:
1540                 ret = nes_init_1g_phy(nesdev, phy_type, phy_index);
1541                 break;
1542         case NES_PHY_TYPE_ARGUS:
1543         case NES_PHY_TYPE_SFP_D:
1544         case NES_PHY_TYPE_KR:
1545                 ret = nes_init_2025_phy(nesdev, phy_type, phy_index);
1546                 break;
1547         }
1548
1549         spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags);
1550
1551         return ret;
1552 }
1553
1554
1555 /**
1556  * nes_replenish_nic_rq
1557  */
1558 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1559 {
1560         unsigned long flags;
1561         dma_addr_t bus_address;
1562         struct sk_buff *skb;
1563         struct nes_hw_nic_rq_wqe *nic_rqe;
1564         struct nes_hw_nic *nesnic;
1565         struct nes_device *nesdev;
1566         struct nes_rskb_cb *cb;
1567         u32 rx_wqes_posted = 0;
1568
1569         nesnic = &nesvnic->nic;
1570         nesdev = nesvnic->nesdev;
1571         spin_lock_irqsave(&nesnic->rq_lock, flags);
1572         if (nesnic->replenishing_rq !=0) {
1573                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1574                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1575                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1576                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1577                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1578                         add_timer(&nesvnic->rq_wqes_timer);
1579                 } else
1580                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1581                 return;
1582         }
1583         nesnic->replenishing_rq = 1;
1584         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1585         do {
1586                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1587                 if (skb) {
1588                         skb->dev = nesvnic->netdev;
1589
1590                         bus_address = pci_map_single(nesdev->pcidev,
1591                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1592                         cb = (struct nes_rskb_cb *)&skb->cb[0];
1593                         cb->busaddr = bus_address;
1594                         cb->maplen = nesvnic->max_frame_size;
1595
1596                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1597                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1598                                         cpu_to_le32(nesvnic->max_frame_size);
1599                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1600                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1601                                         cpu_to_le32((u32)bus_address);
1602                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1603                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1604                         nesnic->rx_skb[nesnic->rq_head] = skb;
1605                         nesnic->rq_head++;
1606                         nesnic->rq_head &= nesnic->rq_size - 1;
1607                         atomic_dec(&nesvnic->rx_skbs_needed);
1608                         barrier();
1609                         if (++rx_wqes_posted == 255) {
1610                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1611                                 rx_wqes_posted = 0;
1612                         }
1613                 } else {
1614                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1615                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1616                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1617                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1618                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1619                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1620                                 add_timer(&nesvnic->rq_wqes_timer);
1621                         } else
1622                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1623                         break;
1624                 }
1625         } while (atomic_read(&nesvnic->rx_skbs_needed));
1626         barrier();
1627         if (rx_wqes_posted)
1628                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1629         nesnic->replenishing_rq = 0;
1630 }
1631
1632
1633 /**
1634  * nes_rq_wqes_timeout
1635  */
1636 static void nes_rq_wqes_timeout(unsigned long parm)
1637 {
1638         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1639         printk("%s: Timer fired.\n", __func__);
1640         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1641         if (atomic_read(&nesvnic->rx_skbs_needed))
1642                 nes_replenish_nic_rq(nesvnic);
1643 }
1644
1645
1646 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1647                                void **tcph, u64 *hdr_flags, void *priv)
1648 {
1649         unsigned int ip_len;
1650         struct iphdr *iph;
1651         skb_reset_network_header(skb);
1652         iph = ip_hdr(skb);
1653         if (iph->protocol != IPPROTO_TCP)
1654                 return -1;
1655         ip_len = ip_hdrlen(skb);
1656         skb_set_transport_header(skb, ip_len);
1657         *tcph = tcp_hdr(skb);
1658
1659         *hdr_flags = LRO_IPV4 | LRO_TCP;
1660         *iphdr = iph;
1661         return 0;
1662 }
1663
1664
1665 /**
1666  * nes_init_nic_qp
1667  */
1668 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1669 {
1670         struct nes_hw_cqp_wqe *cqp_wqe;
1671         struct nes_hw_nic_sq_wqe *nic_sqe;
1672         struct nes_hw_nic_qp_context *nic_context;
1673         struct sk_buff *skb;
1674         struct nes_hw_nic_rq_wqe *nic_rqe;
1675         struct nes_vnic *nesvnic = netdev_priv(netdev);
1676         unsigned long flags;
1677         void *vmem;
1678         dma_addr_t pmem;
1679         u64 u64temp;
1680         int ret;
1681         u32 cqp_head;
1682         u32 counter;
1683         u32 wqe_count;
1684         struct nes_rskb_cb *cb;
1685         u8 jumbomode=0;
1686
1687         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1688         nesvnic->nic_mem_size = 256 +
1689                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1690                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1691                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1692                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1693                         sizeof(struct nes_hw_nic_qp_context);
1694
1695         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1696                         &nesvnic->nic_pbase);
1697         if (!nesvnic->nic_vbase) {
1698                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1699                 return -ENOMEM;
1700         }
1701         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1702         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1703                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1704
1705         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1706                         ~(unsigned long)(256 - 1));
1707         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1708                         ~(unsigned long long)(256 - 1));
1709
1710         /* Setup the first Fragment buffers */
1711         nesvnic->nic.first_frag_vbase = vmem;
1712
1713         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1714                 nesvnic->nic.frag_paddr[counter] = pmem;
1715                 pmem += sizeof(struct nes_first_frag);
1716         }
1717
1718         /* setup the SQ */
1719         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1720
1721         nesvnic->nic.sq_vbase = (void *)vmem;
1722         nesvnic->nic.sq_pbase = pmem;
1723         nesvnic->nic.sq_head = 0;
1724         nesvnic->nic.sq_tail = 0;
1725         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1726         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1727                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1728                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1729                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1730                                 NES_NIC_SQ_WQE_COMPLETION);
1731                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1732                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1733                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1734                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1735                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1736                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1737         }
1738
1739         nesvnic->get_cqp_request = nes_get_cqp_request;
1740         nesvnic->post_cqp_request = nes_post_cqp_request;
1741         nesvnic->mcrq_mcast_filter = NULL;
1742
1743         spin_lock_init(&nesvnic->nic.rq_lock);
1744
1745         /* setup the RQ */
1746         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1747         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1748
1749
1750         nesvnic->nic.rq_vbase = vmem;
1751         nesvnic->nic.rq_pbase = pmem;
1752         nesvnic->nic.rq_head = 0;
1753         nesvnic->nic.rq_tail = 0;
1754         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1755
1756         /* setup the CQ */
1757         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1758         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1759
1760         if (nesdev->nesadapter->netdev_count > 2)
1761                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1762         else
1763                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1764
1765         nesvnic->nic_cq.cq_vbase = vmem;
1766         nesvnic->nic_cq.cq_pbase = pmem;
1767         nesvnic->nic_cq.cq_head = 0;
1768         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1769
1770         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1771
1772         /* Send CreateCQ request to CQP */
1773         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1774         cqp_head = nesdev->cqp.sq_head;
1775
1776         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1777         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1778
1779         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1780                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1781                         ((u32)nesvnic->nic_cq.cq_size << 16));
1782         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1783                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1784         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1785         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1786         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1787         u64temp = (unsigned long)&nesvnic->nic_cq;
1788         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1789         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1790                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1791         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1792         if (++cqp_head >= nesdev->cqp.sq_size)
1793                 cqp_head = 0;
1794         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1795         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1796
1797         /* Send CreateQP request to CQP */
1798         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1799         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1800                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1801                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1802         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1803                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1804                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1805         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1806                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1807         }
1808
1809         u64temp = (u64)nesvnic->nic.sq_pbase;
1810         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1811         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1812         u64temp = (u64)nesvnic->nic.rq_pbase;
1813         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1814         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1815
1816         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1817                         NES_CQP_QP_TYPE_NIC);
1818         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1819         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1820                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1821         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1822
1823         if (++cqp_head >= nesdev->cqp.sq_size)
1824                 cqp_head = 0;
1825         nesdev->cqp.sq_head = cqp_head;
1826
1827         barrier();
1828
1829         /* Ring doorbell (2 WQEs) */
1830         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1831
1832         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1833         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1834                         nesvnic->nic.qp_id);
1835
1836         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1837                         NES_EVENT_TIMEOUT);
1838         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1839                         nesvnic->nic.qp_id, ret);
1840         if (!ret) {
1841                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1842                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1843                                 nesvnic->nic_pbase);
1844                 return -EIO;
1845         }
1846
1847         /* Populate the RQ */
1848         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1849                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1850                 if (!skb) {
1851                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1852
1853                         nes_destroy_nic_qp(nesvnic);
1854                         return -ENOMEM;
1855                 }
1856
1857                 skb->dev = netdev;
1858
1859                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1860                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1861                 cb = (struct nes_rskb_cb *)&skb->cb[0];
1862                 cb->busaddr = pmem;
1863                 cb->maplen = nesvnic->max_frame_size;
1864
1865                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1866                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1867                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1868                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1869                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1870                 nesvnic->nic.rx_skb[counter] = skb;
1871         }
1872
1873         wqe_count = NES_NIC_WQ_SIZE - 1;
1874         nesvnic->nic.rq_head = wqe_count;
1875         barrier();
1876         do {
1877                 counter = min(wqe_count, ((u32)255));
1878                 wqe_count -= counter;
1879                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1880         } while (wqe_count);
1881         init_timer(&nesvnic->rq_wqes_timer);
1882         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1883         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1884         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1885         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1886         {
1887                 nes_nic_init_timer(nesdev);
1888                 if (netdev->mtu > 1500)
1889                         jumbomode = 1;
1890                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1891         }
1892         if ((nesdev->nesadapter->allow_unaligned_fpdus) &&
1893                 (nes_init_mgt_qp(nesdev, netdev, nesvnic))) {
1894                         nes_debug(NES_DBG_INIT, "%s: Out of memory for pau nic\n", netdev->name);
1895                         nes_destroy_nic_qp(nesvnic);
1896                 return -ENOMEM;
1897         }
1898
1899         nesvnic->lro_mgr.max_aggr       = nes_lro_max_aggr;
1900         nesvnic->lro_mgr.max_desc       = NES_MAX_LRO_DESCRIPTORS;
1901         nesvnic->lro_mgr.lro_arr        = nesvnic->lro_desc;
1902         nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1903         nesvnic->lro_mgr.features       = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1904         nesvnic->lro_mgr.dev            = netdev;
1905         nesvnic->lro_mgr.ip_summed      = CHECKSUM_UNNECESSARY;
1906         nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1907         return 0;
1908 }
1909
1910
1911 /**
1912  * nes_destroy_nic_qp
1913  */
1914 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1915 {
1916         u64 u64temp;
1917         dma_addr_t bus_address;
1918         struct nes_device *nesdev = nesvnic->nesdev;
1919         struct nes_hw_cqp_wqe *cqp_wqe;
1920         struct nes_hw_nic_sq_wqe *nic_sqe;
1921         __le16 *wqe_fragment_length;
1922         u16  wqe_fragment_index;
1923         u32 cqp_head;
1924         u32 wqm_cfg0;
1925         unsigned long flags;
1926         struct sk_buff *rx_skb;
1927         struct nes_rskb_cb *cb;
1928         int ret;
1929
1930         if (nesdev->nesadapter->allow_unaligned_fpdus)
1931                 nes_destroy_mgt(nesvnic);
1932
1933         /* clear wqe stall before destroying NIC QP */
1934         wqm_cfg0 = nes_read_indexed(nesdev, NES_IDX_WQM_CONFIG0);
1935         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0 & 0xFFFF7FFF);
1936
1937         /* Free remaining NIC receive buffers */
1938         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1939                 rx_skb = nesvnic->nic.rx_skb[nesvnic->nic.rq_tail];
1940                 cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
1941                 pci_unmap_single(nesdev->pcidev, cb->busaddr, cb->maplen,
1942                         PCI_DMA_FROMDEVICE);
1943
1944                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1945                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1946         }
1947
1948         /* Free remaining NIC transmit buffers */
1949         while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1950                 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1951                 wqe_fragment_index = 1;
1952                 wqe_fragment_length = (__le16 *)
1953                         &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1954                 /* bump past the vlan tag */
1955                 wqe_fragment_length++;
1956                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1957                         u64temp = (u64)le32_to_cpu(
1958                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1959                                 wqe_fragment_index*2]);
1960                         u64temp += ((u64)le32_to_cpu(
1961                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1962                                 + wqe_fragment_index*2]))<<32;
1963                         bus_address = (dma_addr_t)u64temp;
1964                         if (test_and_clear_bit(nesvnic->nic.sq_tail,
1965                                         nesvnic->nic.first_frag_overflow)) {
1966                                 pci_unmap_single(nesdev->pcidev,
1967                                                 bus_address,
1968                                                 le16_to_cpu(wqe_fragment_length[
1969                                                         wqe_fragment_index++]),
1970                                                 PCI_DMA_TODEVICE);
1971                         }
1972                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1973                                 if (wqe_fragment_length[wqe_fragment_index]) {
1974                                         u64temp = le32_to_cpu(
1975                                                 nic_sqe->wqe_words[
1976                                                 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1977                                                 wqe_fragment_index*2]);
1978                                         u64temp += ((u64)le32_to_cpu(
1979                                                 nic_sqe->wqe_words[
1980                                                 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1981                                                 wqe_fragment_index*2]))<<32;
1982                                         bus_address = (dma_addr_t)u64temp;
1983                                         pci_unmap_page(nesdev->pcidev,
1984                                                         bus_address,
1985                                                         le16_to_cpu(
1986                                                         wqe_fragment_length[
1987                                                         wqe_fragment_index]),
1988                                                         PCI_DMA_TODEVICE);
1989                                 } else
1990                                         break;
1991                         }
1992                 }
1993                 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1994                         dev_kfree_skb(
1995                                 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1996
1997                 nesvnic->nic.sq_tail = (nesvnic->nic.sq_tail + 1)
1998                                         & (nesvnic->nic.sq_size - 1);
1999         }
2000
2001         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2002
2003         /* Destroy NIC QP */
2004         cqp_head = nesdev->cqp.sq_head;
2005         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
2006         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
2007
2008         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
2009                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
2010         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
2011                 nesvnic->nic.qp_id);
2012
2013         if (++cqp_head >= nesdev->cqp.sq_size)
2014                 cqp_head = 0;
2015
2016         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
2017
2018         /* Destroy NIC CQ */
2019         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
2020         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
2021                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
2022         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
2023                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
2024
2025         if (++cqp_head >= nesdev->cqp.sq_size)
2026                 cqp_head = 0;
2027
2028         nesdev->cqp.sq_head = cqp_head;
2029         barrier();
2030
2031         /* Ring doorbell (2 WQEs) */
2032         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
2033
2034         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2035         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
2036                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
2037                         cqp_head, nesdev->cqp.sq_head,
2038                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
2039
2040         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
2041                         NES_EVENT_TIMEOUT);
2042
2043         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
2044                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
2045                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
2046         if (!ret) {
2047                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
2048                                 nesvnic->nic.qp_id);
2049         }
2050
2051         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
2052                         nesvnic->nic_pbase);
2053
2054         /* restore old wqm_cfg0 value */
2055         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0);
2056 }
2057
2058 /**
2059  * nes_napi_isr
2060  */
2061 int nes_napi_isr(struct nes_device *nesdev)
2062 {
2063         struct nes_adapter *nesadapter = nesdev->nesadapter;
2064         u32 int_stat;
2065
2066         if (nesdev->napi_isr_ran) {
2067                 /* interrupt status has already been read in ISR */
2068                 int_stat = nesdev->int_stat;
2069         } else {
2070                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
2071                 nesdev->int_stat = int_stat;
2072                 nesdev->napi_isr_ran = 1;
2073         }
2074
2075         int_stat &= nesdev->int_req;
2076         /* iff NIC, process here, else wait for DPC */
2077         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
2078                 nesdev->napi_isr_ran = 0;
2079                 nes_write32(nesdev->regs + NES_INT_STAT,
2080                         (int_stat &
2081                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2082
2083                 /* Process the CEQs */
2084                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
2085
2086                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
2087                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2088                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
2089                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
2090                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
2091                                 /* Enable Periodic timer interrupts */
2092                                 nesdev->int_req |= NES_INT_TIMER;
2093                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
2094                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
2095                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
2096                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
2097                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2098                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2099                         }
2100
2101                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2102                         {
2103                                 nes_nic_init_timer(nesdev);
2104                         }
2105                         /* Enable interrupts, except CEQs */
2106                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2107                 } else {
2108                         /* Enable interrupts, make sure timer is off */
2109                         nesdev->int_req &= ~NES_INT_TIMER;
2110                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2111                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2112                 }
2113                 nesdev->deepcq_count = 0;
2114                 return 1;
2115         } else {
2116                 return 0;
2117         }
2118 }
2119
2120 static void process_critical_error(struct nes_device *nesdev)
2121 {
2122         u32 debug_error;
2123         u32 nes_idx_debug_error_masks0 = 0;
2124         u16 error_module = 0;
2125
2126         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
2127         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
2128                         (u16)debug_error);
2129         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
2130                         0x01010000 | (debug_error & 0x0000ffff));
2131         if (crit_err_count++ > 10)
2132                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
2133         error_module = (u16) (debug_error & 0x1F00) >> 8;
2134         if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
2135                         nes_max_critical_error_count) {
2136                 printk(KERN_ERR PFX "Masking off critical error for module "
2137                         "0x%02X\n", (u16)error_module);
2138                 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
2139                         NES_IDX_DEBUG_ERROR_MASKS0);
2140                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2141                         nes_idx_debug_error_masks0 | (1 << error_module));
2142         }
2143 }
2144 /**
2145  * nes_dpc
2146  */
2147 void nes_dpc(unsigned long param)
2148 {
2149         struct nes_device *nesdev = (struct nes_device *)param;
2150         struct nes_adapter *nesadapter = nesdev->nesadapter;
2151         u32 counter;
2152         u32 loop_counter = 0;
2153         u32 int_status_bit;
2154         u32 int_stat;
2155         u32 timer_stat;
2156         u32 temp_int_stat;
2157         u32 intf_int_stat;
2158         u32 processed_intf_int = 0;
2159         u16 processed_timer_int = 0;
2160         u16 completion_ints = 0;
2161         u16 timer_ints = 0;
2162
2163         /* nes_debug(NES_DBG_ISR, "\n"); */
2164
2165         do {
2166                 timer_stat = 0;
2167                 if (nesdev->napi_isr_ran) {
2168                         nesdev->napi_isr_ran = 0;
2169                         int_stat = nesdev->int_stat;
2170                 } else
2171                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2172                 if (processed_intf_int != 0)
2173                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
2174                 else
2175                         int_stat &= nesdev->int_req;
2176                 if (processed_timer_int == 0) {
2177                         processed_timer_int = 1;
2178                         if (int_stat & NES_INT_TIMER) {
2179                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2180                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
2181                                         int_stat &= ~NES_INT_TIMER;
2182                                 }
2183                         }
2184                 } else {
2185                         int_stat &= ~NES_INT_TIMER;
2186                 }
2187
2188                 if (int_stat) {
2189                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2190                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2191                                 /* Ack the interrupts */
2192                                 nes_write32(nesdev->regs+NES_INT_STAT,
2193                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2194                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2195                         }
2196
2197                         temp_int_stat = int_stat;
2198                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2199                                 if (int_stat & int_status_bit) {
2200                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2201                                         temp_int_stat &= ~int_status_bit;
2202                                         completion_ints = 1;
2203                                 }
2204                                 if (!(temp_int_stat & 0x0000ffff))
2205                                         break;
2206                                 int_status_bit <<= 1;
2207                         }
2208
2209                         /* Process the AEQ for this pci function */
2210                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2211                         if (int_stat & int_status_bit) {
2212                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2213                         }
2214
2215                         /* Process the MAC interrupt for this pci function */
2216                         int_status_bit = 1 << (24 + nesdev->mac_index);
2217                         if (int_stat & int_status_bit) {
2218                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
2219                         }
2220
2221                         if (int_stat & NES_INT_TIMER) {
2222                                 if (timer_stat & nesdev->timer_int_req) {
2223                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
2224                                                         (timer_stat & nesdev->timer_int_req) |
2225                                                         ~(nesdev->nesadapter->timer_int_req));
2226                                         timer_ints = 1;
2227                                 }
2228                         }
2229
2230                         if (int_stat & NES_INT_INTF) {
2231                                 processed_intf_int = 1;
2232                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2233                                 intf_int_stat &= nesdev->intf_int_req;
2234                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2235                                         process_critical_error(nesdev);
2236                                 }
2237                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2238                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2239                                         BUG();
2240                                 }
2241                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2242                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2243                                         BUG();
2244                                 }
2245                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2246                         }
2247
2248                         if (int_stat & NES_INT_TSW) {
2249                         }
2250                 }
2251                 /* Don't use the interface interrupt bit stay in loop */
2252                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2253                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2254         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2255
2256         if (timer_ints == 1) {
2257                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2258                         if (completion_ints == 0) {
2259                                 nesdev->timer_only_int_count++;
2260                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2261                                         nesdev->timer_only_int_count = 0;
2262                                         nesdev->int_req &= ~NES_INT_TIMER;
2263                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2264                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2265                                 } else {
2266                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2267                                 }
2268                         } else {
2269                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2270                                 {
2271                                         nes_nic_init_timer(nesdev);
2272                                 }
2273                                 nesdev->timer_only_int_count = 0;
2274                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2275                         }
2276                 } else {
2277                         nesdev->timer_only_int_count = 0;
2278                         nesdev->int_req &= ~NES_INT_TIMER;
2279                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2280                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2281                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2282                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2283                 }
2284         } else {
2285                 if ( (completion_ints == 1) &&
2286                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2287                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2288                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2289                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2290                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2291                         nesdev->timer_only_int_count = 0;
2292                         nesdev->int_req |= NES_INT_TIMER;
2293                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2294                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2295                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2296                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2297                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2298                 } else {
2299                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2300                 }
2301         }
2302         nesdev->deepcq_count = 0;
2303 }
2304
2305
2306 /**
2307  * nes_process_ceq
2308  */
2309 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2310 {
2311         u64 u64temp;
2312         struct nes_hw_cq *cq;
2313         u32 head;
2314         u32 ceq_size;
2315
2316         /* nes_debug(NES_DBG_CQ, "\n"); */
2317         head = ceq->ceq_head;
2318         ceq_size = ceq->ceq_size;
2319
2320         do {
2321                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2322                                 NES_CEQE_VALID) {
2323                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2324                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2325                         u64temp <<= 1;
2326                         cq = *((struct nes_hw_cq **)&u64temp);
2327                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2328                         barrier();
2329                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2330
2331                         /* call the event handler */
2332                         cq->ce_handler(nesdev, cq);
2333
2334                         if (++head >= ceq_size)
2335                                 head = 0;
2336                 } else {
2337                         break;
2338                 }
2339
2340         } while (1);
2341
2342         ceq->ceq_head = head;
2343 }
2344
2345
2346 /**
2347  * nes_process_aeq
2348  */
2349 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2350 {
2351         /* u64 u64temp; */
2352         u32 head;
2353         u32 aeq_size;
2354         u32 aeqe_misc;
2355         u32 aeqe_cq_id;
2356         struct nes_hw_aeqe volatile *aeqe;
2357
2358         head = aeq->aeq_head;
2359         aeq_size = aeq->aeq_size;
2360
2361         do {
2362                 aeqe = &aeq->aeq_vbase[head];
2363                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2364                         break;
2365                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2366                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2367                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2368                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2369                                 /* dealing with an accelerated QP related AE */
2370                                 /*
2371                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2372                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2373                                  */
2374                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2375                         } else {
2376                                 /* TODO: dealing with a CQP related AE */
2377                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2378                                                 (u16)(aeqe_misc >> 16));
2379                         }
2380                 }
2381
2382                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2383
2384                 if (++head >= aeq_size)
2385                         head = 0;
2386
2387                 nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16);
2388         }
2389         while (1);
2390         aeq->aeq_head = head;
2391 }
2392
2393 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2394 {
2395         struct nes_adapter *nesadapter = nesdev->nesadapter;
2396         u32 reset_value;
2397         u32 i=0;
2398         u32 u32temp;
2399
2400         if (nesadapter->hw_rev == NE020_REV) {
2401                 return;
2402         }
2403         mh_detected++;
2404
2405         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2406
2407         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2408                 reset_value |= 0x0000001d;
2409         else
2410                 reset_value |= 0x0000002d;
2411
2412         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2413                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2414                         nesadapter->link_interrupt_count[0] = 0;
2415                         nesadapter->link_interrupt_count[1] = 0;
2416                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2417                         if (0x00000040 & u32temp)
2418                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2419                         else
2420                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2421
2422                         reset_value |= 0x0000003d;
2423                 }
2424                 nesadapter->link_interrupt_count[mac_index] = 0;
2425         }
2426
2427         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2428
2429         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2430                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2431
2432         if (0x0000003d == (reset_value & 0x0000003d)) {
2433                 u32 pcs_control_status0, pcs_control_status1;
2434
2435                 for (i = 0; i < 10; i++) {
2436                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2437                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2438                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2439                              && (pcs_control_status0 & 0x00100000))
2440                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2441                                 && (pcs_control_status1 & 0x00100000)))
2442                                 continue;
2443                         else
2444                                 break;
2445                 }
2446                 if (10 == i) {
2447                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2448                         if (0x00000040 & u32temp)
2449                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2450                         else
2451                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2452
2453                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2454
2455                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2456                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2457                 }
2458         }
2459 }
2460
2461 /**
2462  * nes_process_mac_intr
2463  */
2464 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2465 {
2466         unsigned long flags;
2467         u32 pcs_control_status;
2468         struct nes_adapter *nesadapter = nesdev->nesadapter;
2469         struct nes_vnic *nesvnic;
2470         u32 mac_status;
2471         u32 mac_index = nesdev->mac_index;
2472         u32 u32temp;
2473         u16 phy_data;
2474         u16 temp_phy_data;
2475         u32 pcs_val  = 0x0f0f0000;
2476         u32 pcs_mask = 0x0f1f0000;
2477         u32 cdr_ctrl;
2478
2479         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2480         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2481                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2482                 return;
2483         }
2484         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2485
2486         /* ack the MAC interrupt */
2487         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2488         /* Clear the interrupt */
2489         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2490
2491         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2492
2493         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2494                 nesdev->link_status_interrupts++;
2495                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS)))
2496                         nes_reset_link(nesdev, mac_index);
2497
2498                 /* read the PHY interrupt status register */
2499                 if ((nesadapter->OneG_Mode) &&
2500                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2501                         do {
2502                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2503                                                 nesadapter->phy_index[mac_index], &phy_data);
2504                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2505                                                 nesadapter->phy_index[mac_index], phy_data);
2506                         } while (phy_data&0x8000);
2507
2508                         temp_phy_data = 0;
2509                         do {
2510                                 nes_read_1G_phy_reg(nesdev, 0x11,
2511                                                 nesadapter->phy_index[mac_index], &phy_data);
2512                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2513                                                 nesadapter->phy_index[mac_index], phy_data);
2514                                 if (temp_phy_data == phy_data)
2515                                         break;
2516                                 temp_phy_data = phy_data;
2517                         } while (1);
2518
2519                         nes_read_1G_phy_reg(nesdev, 0x1e,
2520                                         nesadapter->phy_index[mac_index], &phy_data);
2521                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2522                                         nesadapter->phy_index[mac_index], phy_data);
2523
2524                         nes_read_1G_phy_reg(nesdev, 1,
2525                                         nesadapter->phy_index[mac_index], &phy_data);
2526                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2527                                         nesadapter->phy_index[mac_index], phy_data);
2528
2529                         if (temp_phy_data & 0x1000) {
2530                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2531                                 phy_data = 4;
2532                         } else {
2533                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2534                         }
2535                 }
2536                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2537                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2538                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2539
2540                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2541                         switch (mac_index) {
2542                         case 1:
2543                         case 3:
2544                                 pcs_control_status = nes_read_indexed(nesdev,
2545                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2546                                 break;
2547                         default:
2548                                 pcs_control_status = nes_read_indexed(nesdev,
2549                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2550                                 break;
2551                         }
2552                 } else {
2553                         pcs_control_status = nes_read_indexed(nesdev,
2554                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2555                         pcs_control_status = nes_read_indexed(nesdev,
2556                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2557                 }
2558
2559                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2560                                 mac_index, pcs_control_status);
2561                 if ((nesadapter->OneG_Mode) &&
2562                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2563                         u32temp = 0x01010000;
2564                         if (nesadapter->port_count > 2) {
2565                                 u32temp |= 0x02020000;
2566                         }
2567                         if ((pcs_control_status & u32temp)!= u32temp) {
2568                                 phy_data = 0;
2569                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2570                         }
2571                 } else {
2572                         switch (nesadapter->phy_type[mac_index]) {
2573                         case NES_PHY_TYPE_ARGUS:
2574                         case NES_PHY_TYPE_SFP_D:
2575                         case NES_PHY_TYPE_KR:
2576                                 /* clear the alarms */
2577                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2578                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2579                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2580                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2581                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2582                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2583                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2584                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2585                                 /* check link status */
2586                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2587                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2588
2589                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2590                                 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2591                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2592                                 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2593
2594                                 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2595
2596                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2597                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2598                                 break;
2599
2600                         case NES_PHY_TYPE_PUMA_1G:
2601                                 if (mac_index < 2)
2602                                         pcs_val = pcs_mask = 0x01010000;
2603                                 else
2604                                         pcs_val = pcs_mask = 0x02020000;
2605                                 /* fall through */
2606                         default:
2607                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2608                                 break;
2609                         }
2610                 }
2611
2612                 if (phy_data & 0x0004) {
2613                         if (wide_ppm_offset &&
2614                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2615                             (nesadapter->hw_rev != NE020_REV)) {
2616                                 cdr_ctrl = nes_read_indexed(nesdev,
2617                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2618                                                             mac_index * 0x200);
2619                                 nes_write_indexed(nesdev,
2620                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2621                                                   mac_index * 0x200,
2622                                                   cdr_ctrl | 0x000F0000);
2623                         }
2624                         nesadapter->mac_link_down[mac_index] = 0;
2625                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2626                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2627                                                 nesvnic->linkup);
2628                                 if (nesvnic->linkup == 0) {
2629                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2630                                                         nesvnic->netdev->name, nesvnic->netdev);
2631                                         if (netif_queue_stopped(nesvnic->netdev))
2632                                                 netif_start_queue(nesvnic->netdev);
2633                                         nesvnic->linkup = 1;
2634                                         netif_carrier_on(nesvnic->netdev);
2635
2636                                         spin_lock(&nesvnic->port_ibevent_lock);
2637                                         if (nesvnic->of_device_registered) {
2638                                                 if (nesdev->iw_status == 0) {
2639                                                         nesdev->iw_status = 1;
2640                                                         nes_port_ibevent(nesvnic);
2641                                                 }
2642                                         }
2643                                         spin_unlock(&nesvnic->port_ibevent_lock);
2644                                 }
2645                         }
2646                 } else {
2647                         if (wide_ppm_offset &&
2648                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2649                             (nesadapter->hw_rev != NE020_REV)) {
2650                                 cdr_ctrl = nes_read_indexed(nesdev,
2651                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2652                                                             mac_index * 0x200);
2653                                 nes_write_indexed(nesdev,
2654                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2655                                                   mac_index * 0x200,
2656                                                   cdr_ctrl & 0xFFF0FFFF);
2657                         }
2658                         nesadapter->mac_link_down[mac_index] = 1;
2659                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2660                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2661                                                 nesvnic->linkup);
2662                                 if (nesvnic->linkup == 1) {
2663                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2664                                                         nesvnic->netdev->name, nesvnic->netdev);
2665                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2666                                                 netif_stop_queue(nesvnic->netdev);
2667                                         nesvnic->linkup = 0;
2668                                         netif_carrier_off(nesvnic->netdev);
2669
2670                                         spin_lock(&nesvnic->port_ibevent_lock);
2671                                         if (nesvnic->of_device_registered) {
2672                                                 if (nesdev->iw_status == 1) {
2673                                                         nesdev->iw_status = 0;
2674                                                         nes_port_ibevent(nesvnic);
2675                                                 }
2676                                         }
2677                                         spin_unlock(&nesvnic->port_ibevent_lock);
2678                                 }
2679                         }
2680                 }
2681                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_SFP_D) {
2682                         nesdev->link_recheck = 1;
2683                         mod_delayed_work(system_wq, &nesdev->work,
2684                                          NES_LINK_RECHECK_DELAY);
2685                 }
2686         }
2687
2688         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2689
2690         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2691 }
2692
2693 void nes_recheck_link_status(struct work_struct *work)
2694 {
2695         unsigned long flags;
2696         struct nes_device *nesdev = container_of(work, struct nes_device, work.work);
2697         struct nes_adapter *nesadapter = nesdev->nesadapter;
2698         struct nes_vnic *nesvnic;
2699         u32 mac_index = nesdev->mac_index;
2700         u16 phy_data;
2701         u16 temp_phy_data;
2702
2703         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2704
2705         /* check link status */
2706         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2707         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2708
2709         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2710         nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2711         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2712         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2713
2714         phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2715
2716         nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2717                 __func__, phy_data,
2718                 nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2719
2720         if (phy_data & 0x0004) {
2721                 nesadapter->mac_link_down[mac_index] = 0;
2722                 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2723                         if (nesvnic->linkup == 0) {
2724                                 printk(PFX "The Link is now up for port %s, netdev %p.\n",
2725                                                 nesvnic->netdev->name, nesvnic->netdev);
2726                                 if (netif_queue_stopped(nesvnic->netdev))
2727                                         netif_start_queue(nesvnic->netdev);
2728                                 nesvnic->linkup = 1;
2729                                 netif_carrier_on(nesvnic->netdev);
2730
2731                                 spin_lock(&nesvnic->port_ibevent_lock);
2732                                 if (nesvnic->of_device_registered) {
2733                                         if (nesdev->iw_status == 0) {
2734                                                 nesdev->iw_status = 1;
2735                                                 nes_port_ibevent(nesvnic);
2736                                         }
2737                                 }
2738                                 spin_unlock(&nesvnic->port_ibevent_lock);
2739                         }
2740                 }
2741
2742         } else {
2743                 nesadapter->mac_link_down[mac_index] = 1;
2744                 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2745                         if (nesvnic->linkup == 1) {
2746                                 printk(PFX "The Link is now down for port %s, netdev %p.\n",
2747                                                 nesvnic->netdev->name, nesvnic->netdev);
2748                                 if (!(netif_queue_stopped(nesvnic->netdev)))
2749                                         netif_stop_queue(nesvnic->netdev);
2750                                 nesvnic->linkup = 0;
2751                                 netif_carrier_off(nesvnic->netdev);
2752
2753                                 spin_lock(&nesvnic->port_ibevent_lock);
2754                                 if (nesvnic->of_device_registered) {
2755                                         if (nesdev->iw_status == 1) {
2756                                                 nesdev->iw_status = 0;
2757                                                 nes_port_ibevent(nesvnic);
2758                                         }
2759                                 }
2760                                 spin_unlock(&nesvnic->port_ibevent_lock);
2761                         }
2762                 }
2763         }
2764         if (nesdev->link_recheck++ < NES_LINK_RECHECK_MAX)
2765                 schedule_delayed_work(&nesdev->work, NES_LINK_RECHECK_DELAY);
2766         else
2767                 nesdev->link_recheck = 0;
2768
2769         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2770 }
2771
2772
2773 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2774 {
2775         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2776
2777         napi_schedule(&nesvnic->napi);
2778 }
2779
2780
2781 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2782 * getting out of nic_ce_handler
2783 */
2784 #define MAX_RQES_TO_PROCESS     384
2785
2786 /**
2787  * nes_nic_ce_handler
2788  */
2789 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2790 {
2791         u64 u64temp;
2792         dma_addr_t bus_address;
2793         struct nes_hw_nic *nesnic;
2794         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2795         struct nes_adapter *nesadapter = nesdev->nesadapter;
2796         struct nes_hw_nic_rq_wqe *nic_rqe;
2797         struct nes_hw_nic_sq_wqe *nic_sqe;
2798         struct sk_buff *skb;
2799         struct sk_buff *rx_skb;
2800         struct nes_rskb_cb *cb;
2801         __le16 *wqe_fragment_length;
2802         u32 head;
2803         u32 cq_size;
2804         u32 rx_pkt_size;
2805         u32 cqe_count=0;
2806         u32 cqe_errv;
2807         u32 cqe_misc;
2808         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2809         u16 vlan_tag;
2810         u16 pkt_type;
2811         u16 rqes_processed = 0;
2812         u8 sq_cqes = 0;
2813         u8 nes_use_lro = 0;
2814
2815         head = cq->cq_head;
2816         cq_size = cq->cq_size;
2817         cq->cqes_pending = 1;
2818         if (nesvnic->netdev->features & NETIF_F_LRO)
2819                 nes_use_lro = 1;
2820         do {
2821                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2822                                 NES_NIC_CQE_VALID) {
2823                         nesnic = &nesvnic->nic;
2824                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2825                         if (cqe_misc & NES_NIC_CQE_SQ) {
2826                                 sq_cqes++;
2827                                 wqe_fragment_index = 1;
2828                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2829                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2830                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2831                                 /* bump past the vlan tag */
2832                                 wqe_fragment_length++;
2833                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2834                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2835                                                         wqe_fragment_index * 2]);
2836                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2837                                                         wqe_fragment_index * 2])) << 32;
2838                                         bus_address = (dma_addr_t)u64temp;
2839                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2840                                                 pci_unmap_single(nesdev->pcidev,
2841                                                                 bus_address,
2842                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2843                                                                 PCI_DMA_TODEVICE);
2844                                         }
2845                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2846                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2847                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2848                                                                                 wqe_fragment_index * 2]);
2849                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2850                                                                                 + wqe_fragment_index * 2])) <<32;
2851                                                         bus_address = (dma_addr_t)u64temp;
2852                                                         pci_unmap_page(nesdev->pcidev,
2853                                                                         bus_address,
2854                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2855                                                                         PCI_DMA_TODEVICE);
2856                                                 } else
2857                                                         break;
2858                                         }
2859                                 }
2860                                 if (skb)
2861                                         dev_kfree_skb_any(skb);
2862                                 nesnic->sq_tail++;
2863                                 nesnic->sq_tail &= nesnic->sq_size-1;
2864                                 if (sq_cqes > 128) {
2865                                         barrier();
2866                                         /* restart the queue if it had been stopped */
2867                                         if (netif_queue_stopped(nesvnic->netdev))
2868                                                 netif_wake_queue(nesvnic->netdev);
2869                                         sq_cqes = 0;
2870                                 }
2871                         } else {
2872                                 rqes_processed ++;
2873
2874                                 cq->rx_cqes_completed++;
2875                                 cq->rx_pkts_indicated++;
2876                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2877                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2878                                 /* Get the skb */
2879                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2880                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2881                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2882                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2883                                 pci_unmap_single(nesdev->pcidev, bus_address,
2884                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2885                                 cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
2886                                 cb->busaddr = 0;
2887                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2888                                 /* rx_skb->len = rx_pkt_size; */
2889                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2890                                 skb_put(rx_skb, rx_pkt_size);
2891                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2892                                 nesnic->rq_tail++;
2893                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2894
2895                                 atomic_inc(&nesvnic->rx_skbs_needed);
2896                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2897                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2898                                                         cq->cq_number | (cqe_count << 16));
2899                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2900                                         nesdev->currcq_count += cqe_count;
2901                                         cqe_count = 0;
2902                                         nes_replenish_nic_rq(nesvnic);
2903                                 }
2904                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2905                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2906                                 rx_skb->ip_summed = CHECKSUM_NONE;
2907
2908                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2909                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2910                                         if ((cqe_errv &
2911                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2912                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2913                                                 if (nesvnic->netdev->features & NETIF_F_RXCSUM)
2914                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2915                                         } else
2916                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2917                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2918                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2919
2920                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2921                                         if ((cqe_errv &
2922                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2923                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2924                                                 if (nesvnic->netdev->features & NETIF_F_RXCSUM) {
2925                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2926                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2927                                                                   nesvnic->netdev->name); */
2928                                                 }
2929                                         } else
2930                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2931                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2932                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2933                                         }
2934                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2935                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2936
2937                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2938                                         if (nes_cm_recv(rx_skb, nesvnic->netdev))
2939                                                 rx_skb = NULL;
2940                                 }
2941                                 if (rx_skb == NULL)
2942                                         goto skip_rx_indicate0;
2943
2944
2945                                 if (cqe_misc & NES_NIC_CQE_TAG_VALID) {
2946                                         vlan_tag = (u16)(le32_to_cpu(
2947                                                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2948                                                         >> 16);
2949                                         nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2950                                                         nesvnic->netdev->name, vlan_tag);
2951
2952                                         __vlan_hwaccel_put_tag(rx_skb, vlan_tag);
2953                                 }
2954                                 if (nes_use_lro)
2955                                         lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2956                                 else
2957                                         netif_receive_skb(rx_skb);
2958
2959 skip_rx_indicate0:
2960                                 ;
2961                                 /* nesvnic->netstats.rx_packets++; */
2962                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2963                         }
2964
2965                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2966                         /* Accounting... */
2967                         cqe_count++;
2968                         if (++head >= cq_size)
2969                                 head = 0;
2970                         if (cqe_count == 255) {
2971                                 /* Replenish Nic CQ */
2972                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2973                                                 cq->cq_number | (cqe_count << 16));
2974                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2975                                 nesdev->currcq_count += cqe_count;
2976                                 cqe_count = 0;
2977                         }
2978
2979                         if (cq->rx_cqes_completed >= nesvnic->budget)
2980                                 break;
2981                 } else {
2982                         cq->cqes_pending = 0;
2983                         break;
2984                 }
2985
2986         } while (1);
2987
2988         if (nes_use_lro)
2989                 lro_flush_all(&nesvnic->lro_mgr);
2990         if (sq_cqes) {
2991                 barrier();
2992                 /* restart the queue if it had been stopped */
2993                 if (netif_queue_stopped(nesvnic->netdev))
2994                         netif_wake_queue(nesvnic->netdev);
2995         }
2996         cq->cq_head = head;
2997         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2998                         cq->cq_number, cqe_count, cq->cq_head); */
2999         cq->cqe_allocs_pending = cqe_count;
3000         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
3001         {
3002                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
3003                 nesdev->currcq_count += cqe_count;
3004                 nes_nic_tune_timer(nesdev);
3005         }
3006         if (atomic_read(&nesvnic->rx_skbs_needed))
3007                 nes_replenish_nic_rq(nesvnic);
3008 }
3009
3010
3011
3012 /**
3013  * nes_cqp_ce_handler
3014  */
3015 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
3016 {
3017         u64 u64temp;
3018         unsigned long flags;
3019         struct nes_hw_cqp *cqp = NULL;
3020         struct nes_cqp_request *cqp_request;
3021         struct nes_hw_cqp_wqe *cqp_wqe;
3022         u32 head;
3023         u32 cq_size;
3024         u32 cqe_count=0;
3025         u32 error_code;
3026         u32 opcode;
3027         u32 ctx_index;
3028         /* u32 counter; */
3029
3030         head = cq->cq_head;
3031         cq_size = cq->cq_size;
3032
3033         do {
3034                 /* process the CQE */
3035                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
3036                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
3037
3038                 opcode = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]);
3039                 if (opcode & NES_CQE_VALID) {
3040                         cqp = &nesdev->cqp;
3041
3042                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
3043                         if (error_code) {
3044                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
3045                                                 " Major/Minor codes = 0x%04X:%04X.\n",
3046                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
3047                                                 (u16)(error_code >> 16),
3048                                                 (u16)error_code);
3049                         }
3050
3051                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
3052                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
3053                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
3054                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
3055
3056                         cqp_request = (struct nes_cqp_request *)(unsigned long)u64temp;
3057                         if (cqp_request) {
3058                                 if (cqp_request->waiting) {
3059                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
3060                                         cqp_request->major_code = (u16)(error_code >> 16);
3061                                         cqp_request->minor_code = (u16)error_code;
3062                                         barrier();
3063                                         cqp_request->request_done = 1;
3064                                         wake_up(&cqp_request->waitq);
3065                                         nes_put_cqp_request(nesdev, cqp_request);
3066                                 } else {
3067                                         if (cqp_request->callback)
3068                                                 cqp_request->cqp_callback(nesdev, cqp_request);
3069                                         nes_free_cqp_request(nesdev, cqp_request);
3070                                 }
3071                         } else {
3072                                 wake_up(&nesdev->cqp.waitq);
3073                         }
3074
3075                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
3076                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
3077                         if (++cqp->sq_tail >= cqp->sq_size)
3078                                 cqp->sq_tail = 0;
3079
3080                         /* Accounting... */
3081                         cqe_count++;
3082                         if (++head >= cq_size)
3083                                 head = 0;
3084                 } else {
3085                         break;
3086                 }
3087         } while (1);
3088         cq->cq_head = head;
3089
3090         spin_lock_irqsave(&nesdev->cqp.lock, flags);
3091         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
3092                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
3093                         (nesdev->cqp.sq_size - 1)) != 1)) {
3094                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
3095                                 struct nes_cqp_request, list);
3096                 list_del_init(&cqp_request->list);
3097                 head = nesdev->cqp.sq_head++;
3098                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
3099                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
3100                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
3101                 barrier();
3102
3103                 opcode = cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX];
3104                 if ((opcode & NES_CQP_OPCODE_MASK) == NES_CQP_DOWNLOAD_SEGMENT)
3105                         ctx_index = NES_CQP_WQE_DL_COMP_CTX_LOW_IDX;
3106                 else
3107                         ctx_index = NES_CQP_WQE_COMP_CTX_LOW_IDX;
3108                 cqp_wqe->wqe_words[ctx_index] =
3109                         cpu_to_le32((u32)((unsigned long)cqp_request));
3110                 cqp_wqe->wqe_words[ctx_index + 1] =
3111                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
3112                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
3113                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
3114                 /* Ring doorbell (1 WQEs) */
3115                 barrier();
3116                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
3117         }
3118         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3119
3120         /* Arm the CCQ */
3121         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
3122                         cq->cq_number);
3123         nes_read32(nesdev->regs+NES_CQE_ALLOC);
3124 }
3125
3126 static u8 *locate_mpa(u8 *pkt, u32 aeq_info)
3127 {
3128         if (aeq_info & NES_AEQE_Q2_DATA_ETHERNET) {
3129                 /* skip over ethernet header */
3130                 pkt += ETH_HLEN;
3131
3132                 /* Skip over IP and TCP headers */
3133                 pkt += 4 * (pkt[0] & 0x0f);
3134                 pkt += 4 * ((pkt[12] >> 4) & 0x0f);
3135         }
3136         return pkt;
3137 }
3138
3139 /* Determine if incoming error pkt is rdma layer */
3140 static u32 iwarp_opcode(struct nes_qp *nesqp, u32 aeq_info)
3141 {
3142         u8 *pkt;
3143         u16 *mpa;
3144         u32 opcode = 0xffffffff;
3145
3146         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3147                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3148                 mpa = (u16 *)locate_mpa(pkt, aeq_info);
3149                 opcode = be16_to_cpu(mpa[1]) & 0xf;
3150         }
3151
3152         return opcode;
3153 }
3154
3155 /* Build iWARP terminate header */
3156 static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 aeq_info)
3157 {
3158         u8 *pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3159         u16 ddp_seg_len;
3160         int copy_len = 0;
3161         u8 is_tagged = 0;
3162         u8 flush_code = 0;
3163         struct nes_terminate_hdr *termhdr;
3164
3165         termhdr = (struct nes_terminate_hdr *)nesqp->hwqp.q2_vbase;
3166         memset(termhdr, 0, 64);
3167
3168         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3169
3170                 /* Use data from offending packet to fill in ddp & rdma hdrs */
3171                 pkt = locate_mpa(pkt, aeq_info);
3172                 ddp_seg_len = be16_to_cpu(*(u16 *)pkt);
3173                 if (ddp_seg_len) {
3174                         copy_len = 2;
3175                         termhdr->hdrct = DDP_LEN_FLAG;
3176                         if (pkt[2] & 0x80) {
3177                                 is_tagged = 1;
3178                                 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
3179                                         copy_len += TERM_DDP_LEN_TAGGED;
3180                                         termhdr->hdrct |= DDP_HDR_FLAG;
3181                                 }
3182                         } else {
3183                                 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
3184                                         copy_len += TERM_DDP_LEN_UNTAGGED;
3185                                         termhdr->hdrct |= DDP_HDR_FLAG;
3186                                 }
3187
3188                                 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {
3189                                         if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {
3190                                                 copy_len += TERM_RDMA_LEN;
3191                                                 termhdr->hdrct |= RDMA_HDR_FLAG;
3192                                         }
3193                                 }
3194                         }
3195                 }
3196         }
3197
3198         switch (async_event_id) {
3199         case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3200                 switch (iwarp_opcode(nesqp, aeq_info)) {
3201                 case IWARP_OPCODE_WRITE:
3202                         flush_code = IB_WC_LOC_PROT_ERR;
3203                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3204                         termhdr->error_code = DDP_TAGGED_INV_STAG;
3205                         break;
3206                 default:
3207                         flush_code = IB_WC_REM_ACCESS_ERR;
3208                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3209                         termhdr->error_code = RDMAP_INV_STAG;
3210                 }
3211                 break;
3212         case NES_AEQE_AEID_AMP_INVALID_STAG:
3213                 flush_code = IB_WC_REM_ACCESS_ERR;
3214                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3215                 termhdr->error_code = RDMAP_INV_STAG;
3216                 break;
3217         case NES_AEQE_AEID_AMP_BAD_QP:
3218                 flush_code = IB_WC_LOC_QP_OP_ERR;
3219                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3220                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3221                 break;
3222         case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3223         case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3224                 switch (iwarp_opcode(nesqp, aeq_info)) {
3225                 case IWARP_OPCODE_SEND_INV:
3226                 case IWARP_OPCODE_SEND_SE_INV:
3227                         flush_code = IB_WC_REM_OP_ERR;
3228                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3229                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3230                         break;
3231                 default:
3232                         flush_code = IB_WC_REM_ACCESS_ERR;
3233                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3234                         termhdr->error_code = RDMAP_INV_STAG;
3235                 }
3236                 break;
3237         case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3238                 if (aeq_info & (NES_AEQE_Q2_DATA_ETHERNET | NES_AEQE_Q2_DATA_MPA)) {
3239                         flush_code = IB_WC_LOC_PROT_ERR;
3240                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3241                         termhdr->error_code = DDP_TAGGED_BOUNDS;
3242                 } else {
3243                         flush_code = IB_WC_REM_ACCESS_ERR;
3244                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3245                         termhdr->error_code = RDMAP_INV_BOUNDS;
3246                 }
3247                 break;
3248         case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3249         case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3250         case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3251                 flush_code = IB_WC_REM_ACCESS_ERR;
3252                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3253                 termhdr->error_code = RDMAP_ACCESS;
3254                 break;
3255         case NES_AEQE_AEID_AMP_TO_WRAP:
3256                 flush_code = IB_WC_REM_ACCESS_ERR;
3257                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3258                 termhdr->error_code = RDMAP_TO_WRAP;
3259                 break;
3260         case NES_AEQE_AEID_AMP_BAD_PD:
3261                 switch (iwarp_opcode(nesqp, aeq_info)) {
3262                 case IWARP_OPCODE_WRITE:
3263                         flush_code = IB_WC_LOC_PROT_ERR;
3264                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3265                         termhdr->error_code = DDP_TAGGED_UNASSOC_STAG;
3266                         break;
3267                 case IWARP_OPCODE_SEND_INV:
3268                 case IWARP_OPCODE_SEND_SE_INV:
3269                         flush_code = IB_WC_REM_ACCESS_ERR;
3270                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3271                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3272                         break;
3273                 default:
3274                         flush_code = IB_WC_REM_ACCESS_ERR;
3275                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3276                         termhdr->error_code = RDMAP_UNASSOC_STAG;
3277                 }
3278                 break;
3279         case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3280                 flush_code = IB_WC_LOC_LEN_ERR;
3281                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3282                 termhdr->error_code = MPA_MARKER;
3283                 break;
3284         case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3285                 flush_code = IB_WC_GENERAL_ERR;
3286                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3287                 termhdr->error_code = MPA_CRC;
3288                 break;
3289         case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3290         case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3291                 flush_code = IB_WC_LOC_LEN_ERR;
3292                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3293                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3294                 break;
3295         case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3296         case NES_AEQE_AEID_DDP_NO_L_BIT:
3297                 flush_code = IB_WC_FATAL_ERR;
3298                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3299                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3300                 break;
3301         case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3302         case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3303                 flush_code = IB_WC_GENERAL_ERR;
3304                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3305                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_RANGE;
3306                 break;
3307         case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3308                 flush_code = IB_WC_LOC_LEN_ERR;
3309                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3310                 termhdr->error_code = DDP_UNTAGGED_INV_TOO_LONG;
3311                 break;
3312         case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3313                 flush_code = IB_WC_GENERAL_ERR;
3314                 if (is_tagged) {
3315                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3316                         termhdr->error_code = DDP_TAGGED_INV_DDP_VER;
3317                 } else {
3318                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3319                         termhdr->error_code = DDP_UNTAGGED_INV_DDP_VER;
3320                 }
3321                 break;
3322         case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3323                 flush_code = IB_WC_GENERAL_ERR;
3324                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3325                 termhdr->error_code = DDP_UNTAGGED_INV_MO;
3326                 break;
3327         case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3328                 flush_code = IB_WC_REM_OP_ERR;
3329                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3330                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_NO_BUF;
3331                 break;
3332         case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3333                 flush_code = IB_WC_GENERAL_ERR;
3334                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3335                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3336                 break;
3337         case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3338                 flush_code = IB_WC_GENERAL_ERR;
3339                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3340                 termhdr->error_code = RDMAP_INV_RDMAP_VER;
3341                 break;
3342         case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3343                 flush_code = IB_WC_LOC_QP_OP_ERR;
3344                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3345                 termhdr->error_code = RDMAP_UNEXPECTED_OP;
3346                 break;
3347         default:
3348                 flush_code = IB_WC_FATAL_ERR;
3349                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3350                 termhdr->error_code = RDMAP_UNSPECIFIED;
3351                 break;
3352         }
3353
3354         if (copy_len)
3355                 memcpy(termhdr + 1, pkt, copy_len);
3356
3357         if ((flush_code) && ((NES_AEQE_INBOUND_RDMA & aeq_info) == 0)) {
3358                 if (aeq_info & NES_AEQE_SQ)
3359                         nesqp->term_sq_flush_code = flush_code;
3360                 else
3361                         nesqp->term_rq_flush_code = flush_code;
3362         }
3363
3364         return sizeof(struct nes_terminate_hdr) + copy_len;
3365 }
3366
3367 static void nes_terminate_connection(struct nes_device *nesdev, struct nes_qp *nesqp,
3368                  struct nes_hw_aeqe *aeqe, enum ib_event_type eventtype)
3369 {
3370         u64 context;
3371         unsigned long flags;
3372         u32 aeq_info;
3373         u16 async_event_id;
3374         u8 tcp_state;
3375         u8 iwarp_state;
3376         u32 termlen = 0;
3377         u32 mod_qp_flags = NES_CQP_QP_IWARP_STATE_TERMINATE |
3378                            NES_CQP_QP_TERM_DONT_SEND_FIN;
3379         struct nes_adapter *nesadapter = nesdev->nesadapter;
3380
3381         if (nesqp->term_flags & NES_TERM_SENT)
3382                 return; /* Sanity check */
3383
3384         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3385         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3386         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3387         async_event_id = (u16)aeq_info;
3388
3389         context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3390                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3391         if (!context) {
3392                 WARN_ON(!context);
3393                 return;
3394         }
3395
3396         nesqp = (struct nes_qp *)(unsigned long)context;
3397         spin_lock_irqsave(&nesqp->lock, flags);
3398         nesqp->hw_iwarp_state = iwarp_state;
3399         nesqp->hw_tcp_state = tcp_state;
3400         nesqp->last_aeq = async_event_id;
3401         nesqp->terminate_eventtype = eventtype;
3402         spin_unlock_irqrestore(&nesqp->lock, flags);
3403
3404         if (nesadapter->send_term_ok)
3405                 termlen = nes_bld_terminate_hdr(nesqp, async_event_id, aeq_info);
3406         else
3407                 mod_qp_flags |= NES_CQP_QP_TERM_DONT_SEND_TERM_MSG;
3408
3409         if (!nesdev->iw_status)  {
3410                 nesqp->term_flags = NES_TERM_DONE;
3411                 nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_ERROR, 0, 0);
3412                 nes_cm_disconn(nesqp);
3413         } else {
3414                 nes_terminate_start_timer(nesqp);
3415                 nesqp->term_flags |= NES_TERM_SENT;
3416                 nes_hw_modify_qp(nesdev, nesqp, mod_qp_flags, termlen, 0);
3417         }
3418 }
3419
3420 static void nes_terminate_send_fin(struct nes_device *nesdev,
3421                           struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3422 {
3423         u32 aeq_info;
3424         u16 async_event_id;
3425         u8 tcp_state;
3426         u8 iwarp_state;
3427         unsigned long flags;
3428
3429         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3430         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3431         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3432         async_event_id = (u16)aeq_info;
3433
3434         spin_lock_irqsave(&nesqp->lock, flags);
3435         nesqp->hw_iwarp_state = iwarp_state;
3436         nesqp->hw_tcp_state = tcp_state;
3437         nesqp->last_aeq = async_event_id;
3438         spin_unlock_irqrestore(&nesqp->lock, flags);
3439
3440         /* Send the fin only */
3441         nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_TERMINATE |
3442                 NES_CQP_QP_TERM_DONT_SEND_TERM_MSG, 0, 0);
3443 }
3444
3445 /* Cleanup after a terminate sent or received */
3446 static void nes_terminate_done(struct nes_qp *nesqp, int timeout_occurred)
3447 {
3448         u32 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR;
3449         unsigned long flags;
3450         struct nes_vnic *nesvnic = to_nesvnic(nesqp->ibqp.device);
3451         struct nes_device *nesdev = nesvnic->nesdev;
3452         u8 first_time = 0;
3453
3454         spin_lock_irqsave(&nesqp->lock, flags);
3455         if (nesqp->hte_added) {
3456                 nesqp->hte_added = 0;
3457                 next_iwarp_state |= NES_CQP_QP_DEL_HTE;
3458         }
3459
3460         first_time = (nesqp->term_flags & NES_TERM_DONE) == 0;
3461         nesqp->term_flags |= NES_TERM_DONE;
3462         spin_unlock_irqrestore(&nesqp->lock, flags);
3463
3464         /* Make sure we go through this only once */
3465         if (first_time) {
3466                 if (timeout_occurred == 0)
3467                         del_timer(&nesqp->terminate_timer);
3468                 else
3469                         next_iwarp_state |= NES_CQP_QP_RESET;
3470
3471                 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3472                 nes_cm_disconn(nesqp);
3473         }
3474 }
3475
3476 static void nes_terminate_received(struct nes_device *nesdev,
3477                                 struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3478 {
3479         u32 aeq_info;
3480         u8 *pkt;
3481         u32 *mpa;
3482         u8 ddp_ctl;
3483         u8 rdma_ctl;
3484         u16 aeq_id = 0;
3485
3486         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3487         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3488                 /* Terminate is not a performance path so the silicon */
3489                 /* did not validate the frame - do it now */
3490                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3491                 mpa = (u32 *)locate_mpa(pkt, aeq_info);
3492                 ddp_ctl = (be32_to_cpu(mpa[0]) >> 8) & 0xff;
3493                 rdma_ctl = be32_to_cpu(mpa[0]) & 0xff;
3494                 if ((ddp_ctl & 0xc0) != 0x40)
3495                         aeq_id = NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC;
3496                 else if ((ddp_ctl & 0x03) != 1)
3497                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION;
3498                 else if (be32_to_cpu(mpa[2]) != 2)
3499                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_QN;
3500                 else if (be32_to_cpu(mpa[3]) != 1)
3501                         aeq_id = NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN;
3502                 else if (be32_to_cpu(mpa[4]) != 0)
3503                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_MO;
3504                 else if ((rdma_ctl & 0xc0) != 0x40)
3505                         aeq_id = NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION;
3506
3507                 if (aeq_id) {
3508                         /* Bad terminate recvd - send back a terminate */
3509                         aeq_info = (aeq_info & 0xffff0000) | aeq_id;
3510                         aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3511                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3512                         return;
3513                 }
3514         }
3515
3516         nesqp->term_flags |= NES_TERM_RCVD;
3517         nesqp->terminate_eventtype = IB_EVENT_QP_FATAL;
3518         nes_terminate_start_timer(nesqp);
3519         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3520 }
3521
3522 /* Timeout routine in case terminate fails to complete */
3523 static void nes_terminate_timeout(unsigned long context)
3524 {
3525         struct nes_qp *nesqp = (struct nes_qp *)(unsigned long)context;
3526
3527         nes_terminate_done(nesqp, 1);
3528 }
3529
3530 /* Set a timer in case hw cannot complete the terminate sequence */
3531 static void nes_terminate_start_timer(struct nes_qp *nesqp)
3532 {
3533         init_timer(&nesqp->terminate_timer);
3534         nesqp->terminate_timer.function = nes_terminate_timeout;
3535         nesqp->terminate_timer.expires = jiffies + HZ;
3536         nesqp->terminate_timer.data = (unsigned long)nesqp;
3537         add_timer(&nesqp->terminate_timer);
3538 }
3539
3540 /**
3541  * nes_process_iwarp_aeqe
3542  */
3543 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
3544                                    struct nes_hw_aeqe *aeqe)
3545 {
3546         u64 context;
3547         unsigned long flags;
3548         struct nes_qp *nesqp;
3549         struct nes_hw_cq *hw_cq;
3550         struct nes_cq *nescq;
3551         int resource_allocated;
3552         struct nes_adapter *nesadapter = nesdev->nesadapter;
3553         u32 aeq_info;
3554         u32 next_iwarp_state = 0;
3555         u32 aeqe_cq_id;
3556         u16 async_event_id;
3557         u8 tcp_state;
3558         u8 iwarp_state;
3559         struct ib_event ibevent;
3560
3561         nes_debug(NES_DBG_AEQ, "\n");
3562         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3563         if ((NES_AEQE_INBOUND_RDMA & aeq_info) || (!(NES_AEQE_QP & aeq_info))) {
3564                 context  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
3565                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
3566         } else {
3567                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3568                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3569                 BUG_ON(!context);
3570         }
3571
3572         /* context is nesqp unless async_event_id == CQ ERROR */
3573         nesqp = (struct nes_qp *)(unsigned long)context;
3574         async_event_id = (u16)aeq_info;
3575         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3576         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3577         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
3578                         " Tcp state = %d, iWARP state = %d\n",
3579                         async_event_id,
3580                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
3581                         tcp_state, iwarp_state);
3582
3583         aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
3584         if (aeq_info & NES_AEQE_QP) {
3585                 if (!nes_is_resource_allocated(nesadapter,
3586                                 nesadapter->allocated_qps,
3587                                 aeqe_cq_id))
3588                         return;
3589         }
3590
3591         switch (async_event_id) {
3592                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
3593                         if (nesqp->term_flags)
3594                                 return; /* Ignore it, wait for close complete */
3595
3596                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
3597                                 if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) &&
3598                                         (nesqp->ibqp_state == IB_QPS_RTS)) {
3599                                         spin_lock_irqsave(&nesqp->lock, flags);
3600                                         nesqp->hw_iwarp_state = iwarp_state;
3601                                         nesqp->hw_tcp_state = tcp_state;
3602                                         nesqp->last_aeq = async_event_id;
3603                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
3604                                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
3605                                         spin_unlock_irqrestore(&nesqp->lock, flags);
3606                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3607                                         nes_cm_disconn(nesqp);
3608                                 }
3609                                 nesqp->cm_id->add_ref(nesqp->cm_id);
3610                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
3611                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
3612                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
3613                                                 " need ae to finish up, original_last_aeq = 0x%04X."
3614                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
3615                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
3616                                                 async_event_id, nesqp->last_aeq, tcp_state);
3617                         }
3618                         break;
3619                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
3620                         spin_lock_irqsave(&nesqp->lock, flags);
3621                         nesqp->hw_iwarp_state = iwarp_state;
3622                         nesqp->hw_tcp_state = tcp_state;
3623                         nesqp->last_aeq = async_event_id;
3624                         spin_unlock_irqrestore(&nesqp->lock, flags);
3625                         nes_cm_disconn(nesqp);
3626                         break;
3627
3628                 case NES_AEQE_AEID_RESET_SENT:
3629                         tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3630                         spin_lock_irqsave(&nesqp->lock, flags);
3631                         nesqp->hw_iwarp_state = iwarp_state;
3632                         nesqp->hw_tcp_state = tcp_state;
3633                         nesqp->last_aeq = async_event_id;
3634                         nesqp->hte_added = 0;
3635                         spin_unlock_irqrestore(&nesqp->lock, flags);
3636                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE;
3637                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3638                         nes_cm_disconn(nesqp);
3639                         break;
3640
3641                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
3642                         if (atomic_read(&nesqp->close_timer_started))
3643                                 return;
3644                         spin_lock_irqsave(&nesqp->lock, flags);
3645                         nesqp->hw_iwarp_state = iwarp_state;
3646                         nesqp->hw_tcp_state = tcp_state;
3647                         nesqp->last_aeq = async_event_id;
3648                         spin_unlock_irqrestore(&nesqp->lock, flags);
3649                         nes_cm_disconn(nesqp);
3650                         break;
3651
3652                 case NES_AEQE_AEID_TERMINATE_SENT:
3653                         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3654                         break;
3655
3656                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3657                         nes_terminate_received(nesdev, nesqp, aeqe);
3658                         break;
3659
3660                 case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3661                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3662                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3663                 case NES_AEQE_AEID_AMP_INVALID_STAG:
3664                 case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3665                 case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3666                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3667                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3668                 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3669                 case NES_AEQE_AEID_AMP_TO_WRAP:
3670                         printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_ACCESS_ERR\n",
3671                                         nesqp->hwqp.qp_id, async_event_id);
3672                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_ACCESS_ERR);
3673                         break;
3674
3675                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3676                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3677                 case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3678                 case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3679                         if (iwarp_opcode(nesqp, aeq_info) > IWARP_OPCODE_TERM) {
3680                                 aeq_info &= 0xffff0000;
3681                                 aeq_info |= NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE;
3682                                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3683                         }
3684
3685                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
3686                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3687                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3688                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3689                 case NES_AEQE_AEID_AMP_BAD_QP:
3690                 case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3691                 case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3692                 case NES_AEQE_AEID_DDP_NO_L_BIT:
3693                 case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3694                 case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3695                 case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3696                 case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3697                 case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3698                 case NES_AEQE_AEID_AMP_BAD_PD:
3699                 case NES_AEQE_AEID_AMP_FASTREG_SHARED:
3700                 case NES_AEQE_AEID_AMP_FASTREG_VALID_STAG:
3701                 case NES_AEQE_AEID_AMP_FASTREG_MW_STAG:
3702                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_RIGHTS:
3703                 case NES_AEQE_AEID_AMP_FASTREG_PBL_TABLE_OVERFLOW:
3704                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_LENGTH:
3705                 case NES_AEQE_AEID_AMP_INVALIDATE_SHARED:
3706                 case NES_AEQE_AEID_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS:
3707                 case NES_AEQE_AEID_AMP_MWBIND_VALID_STAG:
3708                 case NES_AEQE_AEID_AMP_MWBIND_OF_MR_STAG:
3709                 case NES_AEQE_AEID_AMP_MWBIND_TO_ZERO_BASED_STAG:
3710                 case NES_AEQE_AEID_AMP_MWBIND_TO_MW_STAG:
3711                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_RIGHTS:
3712                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_BOUNDS:
3713                 case NES_AEQE_AEID_AMP_MWBIND_TO_INVALID_PARENT:
3714                 case NES_AEQE_AEID_AMP_MWBIND_BIND_DISABLED:
3715                 case NES_AEQE_AEID_BAD_CLOSE:
3716                 case NES_AEQE_AEID_RDMA_READ_WHILE_ORD_ZERO:
3717                 case NES_AEQE_AEID_STAG_ZERO_INVALID:
3718                 case NES_AEQE_AEID_ROE_INVALID_RDMA_READ_REQUEST:
3719                 case NES_AEQE_AEID_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
3720                         printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_FATAL\n",
3721                                         nesqp->hwqp.qp_id, async_event_id);
3722                         print_ip(nesqp->cm_node);
3723                         if (!atomic_read(&nesqp->close_timer_started))
3724                                 nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3725                         break;
3726
3727                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3728                         context <<= 1;
3729                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3730                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3731                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3732                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3733                         if (resource_allocated) {
3734                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3735                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3736                                 hw_cq = (struct nes_hw_cq *)(unsigned long)context;
3737                                 if (hw_cq) {
3738                                         nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3739                                         if (nescq->ibcq.event_handler) {
3740                                                 ibevent.device = nescq->ibcq.device;
3741                                                 ibevent.event = IB_EVENT_CQ_ERR;
3742                                                 ibevent.element.cq = &nescq->ibcq;
3743                                                 nescq->ibcq.event_handler(&ibevent, nescq->ibcq.cq_context);
3744                                         }
3745                                 }
3746                         }
3747                         break;
3748
3749                 default:
3750                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3751                                         async_event_id);
3752                         break;
3753         }
3754
3755 }
3756
3757 /**
3758  * nes_iwarp_ce_handler
3759  */
3760 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3761 {
3762         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3763
3764         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3765                         nescq->hw_cq.cq_number); */
3766         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3767
3768         if (nescq->ibcq.comp_handler)
3769                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3770
3771         return;
3772 }
3773
3774
3775 /**
3776  * nes_manage_apbvt()
3777  */
3778 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3779                 u32 nic_index, u32 add_port)
3780 {
3781         struct nes_device *nesdev = nesvnic->nesdev;
3782         struct nes_hw_cqp_wqe *cqp_wqe;
3783         struct nes_cqp_request *cqp_request;
3784         int ret = 0;
3785         u16 major_code;
3786
3787         /* Send manage APBVT request to CQP */
3788         cqp_request = nes_get_cqp_request(nesdev);
3789         if (cqp_request == NULL) {
3790                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3791                 return -ENOMEM;
3792         }
3793         cqp_request->waiting = 1;
3794         cqp_wqe = &cqp_request->cqp_wqe;
3795
3796         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3797                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3798                         accel_local_port, accel_local_port, nic_index);
3799
3800         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3801         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3802                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3803         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3804                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3805
3806         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3807
3808         atomic_set(&cqp_request->refcount, 2);
3809         nes_post_cqp_request(nesdev, cqp_request);
3810
3811         if (add_port == NES_MANAGE_APBVT_ADD)
3812                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3813                                 NES_EVENT_TIMEOUT);
3814         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3815                         ret, cqp_request->major_code, cqp_request->minor_code);
3816         major_code = cqp_request->major_code;
3817
3818         nes_put_cqp_request(nesdev, cqp_request);
3819
3820         if (!ret)
3821                 return -ETIME;
3822         else if (major_code)
3823                 return -EIO;
3824         else
3825                 return 0;
3826 }
3827
3828
3829 /**
3830  * nes_manage_arp_cache
3831  */
3832 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3833                 u32 ip_addr, u32 action)
3834 {
3835         struct nes_hw_cqp_wqe *cqp_wqe;
3836         struct nes_vnic *nesvnic = netdev_priv(netdev);
3837         struct nes_device *nesdev;
3838         struct nes_cqp_request *cqp_request;
3839         int arp_index;
3840
3841         nesdev = nesvnic->nesdev;
3842         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3843         if (arp_index == -1) {
3844                 return;
3845         }
3846
3847         /* update the ARP entry */
3848         cqp_request = nes_get_cqp_request(nesdev);
3849         if (cqp_request == NULL) {
3850                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3851                 return;
3852         }
3853         cqp_request->waiting = 0;
3854         cqp_wqe = &cqp_request->cqp_wqe;
3855         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3856
3857         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3858                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3859         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3860                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3861         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3862
3863         if (action == NES_ARP_ADD) {
3864                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3865                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3866                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3867                                 (((u32)mac_addr[4]) << 8)  | (u32)mac_addr[5]);
3868                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3869                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3870         } else {
3871                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3872                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3873         }
3874
3875         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3876                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3877
3878         atomic_set(&cqp_request->refcount, 1);
3879         nes_post_cqp_request(nesdev, cqp_request);
3880 }
3881
3882
3883 /**
3884  * flush_wqes
3885  */
3886 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3887                 u32 which_wq, u32 wait_completion)
3888 {
3889         struct nes_cqp_request *cqp_request;
3890         struct nes_hw_cqp_wqe *cqp_wqe;
3891         u32 sq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3892         u32 rq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3893         int ret;
3894
3895         cqp_request = nes_get_cqp_request(nesdev);
3896         if (cqp_request == NULL) {
3897                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3898                 return;
3899         }
3900         if (wait_completion) {
3901                 cqp_request->waiting = 1;
3902                 atomic_set(&cqp_request->refcount, 2);
3903         } else {
3904                 cqp_request->waiting = 0;
3905         }
3906         cqp_wqe = &cqp_request->cqp_wqe;
3907         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3908
3909         /* If wqe in error was identified, set code to be put into cqe */
3910         if ((nesqp->term_sq_flush_code) && (which_wq & NES_CQP_FLUSH_SQ)) {
3911                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3912                 sq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_sq_flush_code;
3913                 nesqp->term_sq_flush_code = 0;
3914         }
3915
3916         if ((nesqp->term_rq_flush_code) && (which_wq & NES_CQP_FLUSH_RQ)) {
3917                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3918                 rq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_rq_flush_code;
3919                 nesqp->term_rq_flush_code = 0;
3920         }
3921
3922         if (which_wq & NES_CQP_FLUSH_MAJ_MIN) {
3923                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_SQ_CODE] = cpu_to_le32(sq_code);
3924                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_RQ_CODE] = cpu_to_le32(rq_code);
3925         }
3926
3927         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3928                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3929         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3930
3931         nes_post_cqp_request(nesdev, cqp_request);
3932
3933         if (wait_completion) {
3934                 /* Wait for CQP */
3935                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3936                                 NES_EVENT_TIMEOUT);
3937                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3938                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3939                                 ret, cqp_request->major_code, cqp_request->minor_code);
3940                 nes_put_cqp_request(nesdev, cqp_request);
3941         }
3942 }