drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2009 Intel-NE, Inc.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/inet_lro.h>
  42
  43 #include "nes.h"
  44
  45 static unsigned int nes_lro_max_aggr = NES_LRO_MAX_AGGR;
  46 module_param(nes_lro_max_aggr, uint, 0444);
  47 MODULE_PARM_DESC(nes_lro_max_aggr, "NIC LRO max packet aggregation");
  48
  49 static int wide_ppm_offset;
  50 module_param(wide_ppm_offset, int, 0644);
  51 MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm");
  52
  53 static u32 crit_err_count;
  54 u32 int_mod_timer_init;
  55 u32 int_mod_cq_depth_256;
  56 u32 int_mod_cq_depth_128;
  57 u32 int_mod_cq_depth_32;
  58 u32 int_mod_cq_depth_24;
  59 u32 int_mod_cq_depth_16;
  60 u32 int_mod_cq_depth_4;
  61 u32 int_mod_cq_depth_1;
  62 static const u8 nes_max_critical_error_count = 100;
  63 #include "nes_cm.h"
  64
  65 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  66 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  67 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  68                                 struct nes_adapter *nesadapter, u8  OneG_Mode);
  69 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  70 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  71 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  72 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  73                                    struct nes_hw_aeqe *aeqe);
  74 static void process_critical_error(struct nes_device *nesdev);
  75 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  76 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  77 static void nes_terminate_timeout(unsigned long context);
  78 static void nes_terminate_start_timer(struct nes_qp *nesqp);
  79
  80 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  81 static unsigned char *nes_iwarp_state_str[] = {
  82         "Non-Existant",
  83         "Idle",
  84         "RTS",
  85         "Closing",
  86         "RSVD1",
  87         "Terminate",
  88         "Error",
  89         "RSVD2",
  90 };
  91
  92 static unsigned char *nes_tcp_state_str[] = {
  93         "Non-Existant",
  94         "Closed",
  95         "Listen",
  96         "SYN Sent",
  97         "SYN Rcvd",
  98         "Established",
  99         "Close Wait",
 100         "FIN Wait 1",
 101         "Closing",
 102         "Last Ack",
 103         "FIN Wait 2",
 104         "Time Wait",
 105         "RSVD1",
 106         "RSVD2",
 107         "RSVD3",
 108         "RSVD4",
 109 };
 110 #endif
 111
 112
 113 /**
 114  * nes_nic_init_timer_defaults
 115  */
 116 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 117 {
 118         unsigned long flags;
 119         struct nes_adapter *nesadapter = nesdev->nesadapter;
 120         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 121
 122         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 123
 124         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 125         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 126         if (jumbomode) {
 127                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 128                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 129                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 130         } else {
 131                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 132                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 133                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 134         }
 135
 136         /* todo use netdev->mtu to set thresholds */
 137         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 138 }
 139
 140
 141 /**
 142  * nes_nic_init_timer
 143  */
 144 static void  nes_nic_init_timer(struct nes_device *nesdev)
 145 {
 146         unsigned long flags;
 147         struct nes_adapter *nesadapter = nesdev->nesadapter;
 148         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 149
 150         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 151
 152         if (shared_timer->timer_in_use_old == 0) {
 153                 nesdev->deepcq_count = 0;
 154                 shared_timer->timer_direction_upward = 0;
 155                 shared_timer->timer_direction_downward = 0;
 156                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 157                 shared_timer->timer_in_use_old = 0;
 158
 159         }
 160         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 161                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 162                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 163                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 164         }
 165         /* todo use netdev->mtu to set thresholds */
 166         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 167 }
 168
 169
 170 /**
 171  * nes_nic_tune_timer
 172  */
 173 static void nes_nic_tune_timer(struct nes_device *nesdev)
 174 {
 175         unsigned long flags;
 176         struct nes_adapter *nesadapter = nesdev->nesadapter;
 177         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 178         u16 cq_count = nesdev->currcq_count;
 179
 180         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 181
 182         if (shared_timer->cq_count_old <= cq_count)
 183                 shared_timer->cq_direction_downward = 0;
 184         else
 185                 shared_timer->cq_direction_downward++;
 186         shared_timer->cq_count_old = cq_count;
 187         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 188                 if (cq_count <= shared_timer->threshold_low &&
 189                     shared_timer->threshold_low > 4) {
 190                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 191                         shared_timer->cq_direction_downward=0;
 192                         nesdev->currcq_count = 0;
 193                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 194                         return;
 195                 }
 196         }
 197
 198         if (cq_count > 1) {
 199                 nesdev->deepcq_count += cq_count;
 200                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 201                         shared_timer->timer_direction_upward++;
 202                         shared_timer->timer_direction_downward = 0;
 203                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 204                         shared_timer->timer_direction_upward = 0;
 205                         shared_timer->timer_direction_downward = 0;
 206                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 207                         shared_timer->timer_direction_downward++;
 208                         shared_timer->timer_direction_upward = 0;
 209                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 210                         shared_timer->timer_in_use -= 2;
 211                         shared_timer->timer_direction_upward = 0;
 212                         shared_timer->timer_direction_downward++;
 213                 } else {
 214                         shared_timer->timer_in_use -= 4;
 215                         shared_timer->timer_direction_upward = 0;
 216                         shared_timer->timer_direction_downward++;
 217                 }
 218
 219                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 220                         shared_timer->timer_in_use += 3;
 221                         shared_timer->timer_direction_upward = 0;
 222                         shared_timer->timer_direction_downward = 0;
 223                 }
 224                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 225                         shared_timer->timer_in_use -= 4 ;
 226                         shared_timer->timer_direction_downward = 0;
 227                         shared_timer->timer_direction_upward = 0;
 228                 }
 229         }
 230
 231         /* boundary checking */
 232         if (shared_timer->timer_in_use > shared_timer->threshold_high)
 233                 shared_timer->timer_in_use = shared_timer->threshold_high;
 234         else if (shared_timer->timer_in_use < shared_timer->threshold_low)
 235                 shared_timer->timer_in_use = shared_timer->threshold_low;
 236
 237         nesdev->currcq_count = 0;
 238
 239         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 240 }
 241
 242
 243 /**
 244  * nes_init_adapter - initialize adapter
 245  */
 246 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 247         struct nes_adapter *nesadapter = NULL;
 248         unsigned long num_pds;
 249         u32 u32temp;
 250         u32 port_count;
 251         u16 max_rq_wrs;
 252         u16 max_sq_wrs;
 253         u32 max_mr;
 254         u32 max_256pbl;
 255         u32 max_4kpbl;
 256         u32 max_qp;
 257         u32 max_irrq;
 258         u32 max_cq;
 259         u32 hte_index_mask;
 260         u32 adapter_size;
 261         u32 arp_table_size;
 262         u16 vendor_id;
 263         u16 device_id;
 264         u8  OneG_Mode;
 265         u8  func_index;
 266
 267         /* search the list of existing adapters */
 268         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 269                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 270                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 271                                 nesdev->pcidev->devfn,
 272                                 PCI_SLOT(nesadapter->devfn),
 273                                 nesadapter->bus_number,
 274                                 PCI_SLOT(nesdev->pcidev->devfn),
 275                                 nesdev->pcidev->bus->number );
 276                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 277                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 278                         nesadapter->ref_count++;
 279                         return nesadapter;
 280                 }
 281         }
 282
 283         /* no adapter found */
 284         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 285         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 286                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 287                                 hw_rev);
 288                 return NULL;
 289         }
 290
 291         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 292                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 293                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 294                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 295                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 296
 297         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 298
 299
 300         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 301                 return NULL;
 302
 303         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 304         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 305
 306         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 307         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 308                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 309                                 max_qp, u32temp);
 310                 max_qp = (u32)1 << (u32temp & 0x001f);
 311         }
 312
 313         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 314         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 315                         max_qp, hte_index_mask);
 316
 317         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 318
 319         max_irrq = 1 << (u32temp & 0x001f);
 320
 321         if (max_qp > max_irrq) {
 322                 max_qp = max_irrq;
 323                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 324                                 max_qp);
 325         }
 326
 327         /* there should be no reason to allocate more pds than qps */
 328         if (num_pds > max_qp)
 329                 num_pds = max_qp;
 330
 331         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 332         max_mr = (u32)8192 << (u32temp & 0x7);
 333
 334         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 335         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 336         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 337         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 338
 339         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 340         arp_table_size = 1 << u32temp;
 341
 342         adapter_size = (sizeof(struct nes_adapter) +
 343                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 344         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 345         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 346         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 347         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 348         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 349         adapter_size += sizeof(struct nes_qp **) * max_qp;
 350
 351         /* allocate a new adapter struct */
 352         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 353         if (nesadapter == NULL) {
 354                 return NULL;
 355         }
 356
 357         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 358                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 359
 360         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 361                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 362                 kfree(nesadapter);
 363                 return NULL;
 364         }
 365
 366         nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
 367                                 (nesadapter->mac_addr_low >> 24);
 368
 369         pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
 370                                  PCI_DEVICE_ID, &device_id);
 371         nesadapter->vendor_part_id = device_id;
 372
 373         if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
 374                                                         OneG_Mode)) {
 375                 kfree(nesadapter);
 376                 return NULL;
 377         }
 378         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 379
 380         memset(nesadapter->pft_mcast_map, 255,
 381                sizeof nesadapter->pft_mcast_map);
 382
 383         /* populate the new nesadapter */
 384         nesadapter->devfn = nesdev->pcidev->devfn;
 385         nesadapter->bus_number = nesdev->pcidev->bus->number;
 386         nesadapter->ref_count = 1;
 387         nesadapter->timer_int_req = 0xffff0000;
 388         nesadapter->OneG_Mode = OneG_Mode;
 389         nesadapter->doorbell_start = nesdev->doorbell_region;
 390
 391         /* nesadapter->tick_delta = clk_divisor; */
 392         nesadapter->hw_rev = hw_rev;
 393         nesadapter->port_count = port_count;
 394
 395         nesadapter->max_qp = max_qp;
 396         nesadapter->hte_index_mask = hte_index_mask;
 397         nesadapter->max_irrq = max_irrq;
 398         nesadapter->max_mr = max_mr;
 399         nesadapter->max_256pbl = max_256pbl - 1;
 400         nesadapter->max_4kpbl = max_4kpbl - 1;
 401         nesadapter->max_cq = max_cq;
 402         nesadapter->free_256pbl = max_256pbl - 1;
 403         nesadapter->free_4kpbl = max_4kpbl - 1;
 404         nesadapter->max_pd = num_pds;
 405         nesadapter->arp_table_size = arp_table_size;
 406
 407         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 408         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 409                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 410                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 411                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 412         } else {
 413                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 414                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 415                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 416                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 417         }
 418         /* Setup and enable the periodic timer */
 419         if (nesadapter->et_rx_coalesce_usecs_irq)
 420                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 421                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 422         else
 423                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 424
 425         nesadapter->base_pd = 1;
 426
 427         nesadapter->device_cap_flags =
 428                 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_WINDOW;
 429
 430         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 431                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 432         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 433         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 434         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 435         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 436         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 437
 438
 439         /* mark the usual suspect QPs and CQs as in use */
 440         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 441                 set_bit(u32temp, nesadapter->allocated_qps);
 442                 set_bit(u32temp, nesadapter->allocated_cqs);
 443         }
 444
 445         for (u32temp = 0; u32temp < 20; u32temp++)
 446                 set_bit(u32temp, nesadapter->allocated_pds);
 447         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 448
 449         max_rq_wrs = ((u32temp >> 8) & 3);
 450         switch (max_rq_wrs) {
 451                 case 0:
 452                         max_rq_wrs = 4;
 453                         break;
 454                 case 1:
 455                         max_rq_wrs = 16;
 456                         break;
 457                 case 2:
 458                         max_rq_wrs = 32;
 459                         break;
 460                 case 3:
 461                         max_rq_wrs = 512;
 462                         break;
 463         }
 464
 465         max_sq_wrs = (u32temp & 3);
 466         switch (max_sq_wrs) {
 467                 case 0:
 468                         max_sq_wrs = 4;
 469                         break;
 470                 case 1:
 471                         max_sq_wrs = 16;
 472                         break;
 473                 case 2:
 474                         max_sq_wrs = 32;
 475                         break;
 476                 case 3:
 477                         max_sq_wrs = 512;
 478                         break;
 479         }
 480         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 481         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 482
 483         nesadapter->max_sge = 4;
 484         nesadapter->max_cqe = 32767;
 485
 486         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 487                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 488                 kfree(nesadapter);
 489                 return NULL;
 490         }
 491
 492         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 493         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 494                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 495
 496         /* setup port configuration */
 497         if (nesadapter->port_count == 1) {
 498                 nesadapter->log_port = 0x00000000;
 499                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 500                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 501                 else
 502                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 503         } else {
 504                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 505                         nesadapter->log_port = 0x000000D8;
 506                 } else {
 507                         if (nesadapter->port_count == 2)
 508                                 nesadapter->log_port = 0x00000044;
 509                         else
 510                                 nesadapter->log_port = 0x000000e4;
 511                 }
 512                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 513         }
 514
 515         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
 516                                                 nesadapter->log_port);
 517         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 518                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 519
 520         spin_lock_init(&nesadapter->resource_lock);
 521         spin_lock_init(&nesadapter->phy_lock);
 522         spin_lock_init(&nesadapter->pbl_lock);
 523         spin_lock_init(&nesadapter->periodic_timer_lock);
 524
 525         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 526         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 527         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 528         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 529
 530         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 531                 u32 pcs_control_status0, pcs_control_status1;
 532                 u32 reset_value;
 533                 u32 i = 0;
 534                 u32 int_cnt = 0;
 535                 u32 ext_cnt = 0;
 536                 unsigned long flags;
 537                 u32 j = 0;
 538
 539                 pcs_control_status0 = nes_read_indexed(nesdev,
 540                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 541                 pcs_control_status1 = nes_read_indexed(nesdev,
 542                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 543
 544                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 545                         pcs_control_status0 = nes_read_indexed(nesdev,
 546                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 547                         pcs_control_status1 = nes_read_indexed(nesdev,
 548                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 549                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 550                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 551                                 int_cnt++;
 552                         msleep(1);
 553                 }
 554                 if (int_cnt > 1) {
 555                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 556                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
 557                         mh_detected++;
 558                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 559                         reset_value |= 0x0000003d;
 560                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 561
 562                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 563                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 564                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 565
 566                         pcs_control_status0 = nes_read_indexed(nesdev,
 567                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 568                         pcs_control_status1 = nes_read_indexed(nesdev,
 569                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 570
 571                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 572                                 pcs_control_status0 = nes_read_indexed(nesdev,
 573                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 574                                 pcs_control_status1 = nes_read_indexed(nesdev,
 575                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 576                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 577                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 578                                         if (++ext_cnt > int_cnt) {
 579                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 580                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 581                                                                 0x0000F088);
 582                                                 mh_detected++;
 583                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 584                                                 reset_value |= 0x0000003d;
 585                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 586
 587                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 588                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 589                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 590                                                 break;
 591                                         }
 592                                 }
 593                                 msleep(1);
 594                         }
 595                 }
 596         }
 597
 598         if (nesadapter->hw_rev == NE020_REV) {
 599                 init_timer(&nesadapter->mh_timer);
 600                 nesadapter->mh_timer.function = nes_mh_fix;
 601                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 602                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 603                 add_timer(&nesadapter->mh_timer);
 604         } else {
 605                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 606         }
 607
 608         init_timer(&nesadapter->lc_timer);
 609         nesadapter->lc_timer.function = nes_clc;
 610         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 611         nesadapter->lc_timer.data = (unsigned long)nesdev;
 612         add_timer(&nesadapter->lc_timer);
 613
 614         list_add_tail(&nesadapter->list, &nes_adapter_list);
 615
 616         for (func_index = 0; func_index < 8; func_index++) {
 617                 pci_bus_read_config_word(nesdev->pcidev->bus,
 618                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 619                                         func_index), 0, &vendor_id);
 620                 if (vendor_id == 0xffff)
 621                         break;
 622         }
 623         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 624                 func_index, pci_name(nesdev->pcidev));
 625         nesadapter->adapter_fcn_count = func_index;
 626
 627         return nesadapter;
 628 }
 629
 630
 631 /**
 632  * nes_reset_adapter_ne020
 633  */
 634 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 635 {
 636         u32 port_count;
 637         u32 u32temp;
 638         u32 i;
 639
 640         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 641         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 642         /* TODO: assuming that both SERDES are set the same for now */
 643         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 644         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 645                         u32temp, port_count);
 646         if (*OneG_Mode)
 647                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 648         u32temp &= 0xff00ffc0;
 649         switch (port_count) {
 650                 case 1:
 651                         u32temp |= 0x00ee0000;
 652                         break;
 653                 case 2:
 654                         u32temp |= 0x00cc0000;
 655                         break;
 656                 case 4:
 657                         u32temp |= 0x00000000;
 658                         break;
 659                 default:
 660                         return 0;
 661                         break;
 662         }
 663
 664         /* check and do full reset if needed */
 665         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 666                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 667                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 668
 669                 i = 0;
 670                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 671                         mdelay(1);
 672                 if (i > 10000) {
 673                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 674                         return 0;
 675                 }
 676
 677                 i = 0;
 678                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 679                         mdelay(1);
 680                 if (i > 10000) {
 681                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 682                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 683                         return 0;
 684                 }
 685         }
 686
 687         /* port reset */
 688         switch (port_count) {
 689                 case 1:
 690                         u32temp |= 0x00ee0010;
 691                         break;
 692                 case 2:
 693                         u32temp |= 0x00cc0030;
 694                         break;
 695                 case 4:
 696                         u32temp |= 0x00000030;
 697                         break;
 698         }
 699
 700         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 701         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 702
 703         i = 0;
 704         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 705                 mdelay(1);
 706         if (i > 10000) {
 707                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 708                 return 0;
 709         }
 710
 711         /* serdes 0 */
 712         i = 0;
 713         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 714                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 715                 mdelay(1);
 716         if (i > 5000) {
 717                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 718                 return 0;
 719         }
 720
 721         /* serdes 1 */
 722         if (port_count > 1) {
 723                 i = 0;
 724                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 725                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 726                         mdelay(1);
 727                 if (i > 5000) {
 728                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 729                         return 0;
 730                 }
 731         }
 732
 733         return port_count;
 734 }
 735
 736
 737 /**
 738  * nes_init_serdes
 739  */
 740 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 741                                 struct nes_adapter *nesadapter, u8  OneG_Mode)
 742 {
 743         int i;
 744         u32 u32temp;
 745         u32 sds;
 746
 747         if (hw_rev != NE020_REV) {
 748                 /* init serdes 0 */
 749                 if (wide_ppm_offset && (nesadapter->phy_type[0] == NES_PHY_TYPE_CX4))
 750                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA);
 751                 else
 752                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 753
 754                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 755                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
 756                         sds |= 0x00000100;
 757                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds);
 758                 }
 759                 if (!OneG_Mode)
 760                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 761
 762                 if (port_count < 2)
 763                         return 0;
 764
 765                 /* init serdes 1 */
 766                 if (!(OneG_Mode && (nesadapter->phy_type[1] != NES_PHY_TYPE_PUMA_1G)))
 767                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 768
 769                 switch (nesadapter->phy_type[1]) {
 770                 case NES_PHY_TYPE_ARGUS:
 771                 case NES_PHY_TYPE_SFP_D:
 772                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 773                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 774                         break;
 775                 case NES_PHY_TYPE_CX4:
 776                         if (wide_ppm_offset)
 777                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA);
 778                         break;
 779                 case NES_PHY_TYPE_PUMA_1G:
 780                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 781                         sds |= 0x000000100;
 782                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 783                 }
 784                 if (!OneG_Mode) {
 785                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 786                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 787                         sds &= 0xFFFFFFBF;
 788                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 789                 }
 790         } else {
 791                 /* init serdes 0 */
 792                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 793                 i = 0;
 794                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 795                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 796                         mdelay(1);
 797                 if (i > 5000) {
 798                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 799                         return 1;
 800                 }
 801                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 802                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 803                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 804                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 805                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 806                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 807                 if (OneG_Mode)
 808                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 809                 else
 810                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 811
 812                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 813                 if (port_count > 1) {
 814                         /* init serdes 1 */
 815                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 816                         i = 0;
 817                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 818                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 819                                 mdelay(1);
 820                         if (i > 5000) {
 821                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 822                                 /* return 1; */
 823                         }
 824                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 825                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 826                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 827                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 828                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 829                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 830                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 831                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 832                 }
 833         }
 834         return 0;
 835 }
 836
 837
 838 /**
 839  * nes_init_csr_ne020
 840  * Initialize registers for ne020 hardware
 841  */
 842 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 843 {
 844         u32 u32temp;
 845
 846         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 847
 848         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 849         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 850         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 851         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 852         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 853         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 854         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 855         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 856
 857         /* TODO: move these MAC register settings to NIC bringup */
 858         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 859         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 860         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 861         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 862         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 863         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 864         if (port_count > 1) {
 865                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 866                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 867                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 868                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 869                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 870                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 871                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 872         }
 873         if (port_count > 2) {
 874                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 875                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 876                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 877                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 878                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 879                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 880                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 881
 882                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 883                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 884                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 885                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 886                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 887                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 888                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 889         }
 890
 891         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 892         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 893         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
 894                                                          0x00000001);
 895         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 896         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 897         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 898         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 899         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 900
 901         /* TODO: move this to code, get from EEPROM */
 902         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 903         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 904         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 905
 906         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 907         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 908
 909         if (hw_rev != NE020_REV) {
 910                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 911                 u32temp |= 0x80000000;
 912                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 913                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 914                 u32temp &= 0x7fffffff;
 915                 u32temp |= 0x7fff0010;
 916                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 917                 if (port_count > 1) {
 918                         u32temp = nes_read_indexed(nesdev, 0x000023f8);
 919                         u32temp &= 0x7fffffff;
 920                         u32temp |= 0x7fff0010;
 921                         nes_write_indexed(nesdev, 0x000023f8, u32temp);
 922                 }
 923         }
 924 }
 925
 926
 927 /**
 928  * nes_destroy_adapter - destroy the adapter structure
 929  */
 930 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 931 {
 932         struct nes_adapter *tmp_adapter;
 933
 934         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 935                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 936                                 tmp_adapter);
 937         }
 938
 939         nesadapter->ref_count--;
 940         if (!nesadapter->ref_count) {
 941                 if (nesadapter->hw_rev == NE020_REV) {
 942                         del_timer(&nesadapter->mh_timer);
 943                 }
 944                 del_timer(&nesadapter->lc_timer);
 945
 946                 list_del(&nesadapter->list);
 947                 kfree(nesadapter);
 948         }
 949 }
 950
 951
 952 /**
 953  * nes_init_cqp
 954  */
 955 int nes_init_cqp(struct nes_device *nesdev)
 956 {
 957         struct nes_adapter *nesadapter = nesdev->nesadapter;
 958         struct nes_hw_cqp_qp_context *cqp_qp_context;
 959         struct nes_hw_cqp_wqe *cqp_wqe;
 960         struct nes_hw_ceq *ceq;
 961         struct nes_hw_ceq *nic_ceq;
 962         struct nes_hw_aeq *aeq;
 963         void *vmem;
 964         dma_addr_t pmem;
 965         u32 count=0;
 966         u32 cqp_head;
 967         u64 u64temp;
 968         u32 u32temp;
 969
 970         /* allocate CQP memory */
 971         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 972         /* SQ is 512 byte aligned, others are 256 byte aligned */
 973         nesdev->cqp_mem_size = 512 +
 974                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 975                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 976                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 977                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 978                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 979                         sizeof(struct nes_hw_cqp_qp_context);
 980
 981         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
 982                         &nesdev->cqp_pbase);
 983         if (!nesdev->cqp_vbase) {
 984                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
 985                 return -ENOMEM;
 986         }
 987         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
 988
 989         /* Allocate a twice the number of CQP requests as the SQ size */
 990         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
 991                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
 992         if (nesdev->nes_cqp_requests == NULL) {
 993                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
 994                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
 995                                 nesdev->cqp.sq_pbase);
 996                 return -ENOMEM;
 997         }
 998
 999         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
1000                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
1001
1002         spin_lock_init(&nesdev->cqp.lock);
1003         init_waitqueue_head(&nesdev->cqp.waitq);
1004
1005         /* Setup Various Structures */
1006         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
1007                         ~(unsigned long)(512 - 1));
1008         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
1009                         ~(unsigned long long)(512 - 1));
1010
1011         nesdev->cqp.sq_vbase = vmem;
1012         nesdev->cqp.sq_pbase = pmem;
1013         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
1014         nesdev->cqp.sq_head = 0;
1015         nesdev->cqp.sq_tail = 0;
1016         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
1017
1018         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1019         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1020
1021         nesdev->ccq.cq_vbase = vmem;
1022         nesdev->ccq.cq_pbase = pmem;
1023         nesdev->ccq.cq_size = NES_CCQ_SIZE;
1024         nesdev->ccq.cq_head = 0;
1025         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
1026         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
1027
1028         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1029         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1030
1031         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1032         ceq = &nesadapter->ceq[nesdev->ceq_index];
1033         ceq->ceq_vbase = vmem;
1034         ceq->ceq_pbase = pmem;
1035         ceq->ceq_size = NES_CCEQ_SIZE;
1036         ceq->ceq_head = 0;
1037
1038         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1039         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1040
1041         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1042         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1043         nic_ceq->ceq_vbase = vmem;
1044         nic_ceq->ceq_pbase = pmem;
1045         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1046         nic_ceq->ceq_head = 0;
1047
1048         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1049         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1050
1051         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1052         aeq->aeq_vbase = vmem;
1053         aeq->aeq_pbase = pmem;
1054         aeq->aeq_size = nesadapter->max_qp;
1055         aeq->aeq_head = 0;
1056
1057         /* Setup QP Context */
1058         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1059         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1060
1061         cqp_qp_context = vmem;
1062         cqp_qp_context->context_words[0] =
1063                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1064         cqp_qp_context->context_words[1] = 0;
1065         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1066         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1067
1068
1069         /* Write the address to Create CQP */
1070         if ((sizeof(dma_addr_t) > 4)) {
1071                 nes_write_indexed(nesdev,
1072                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1073                                 ((u64)pmem) >> 32);
1074         } else {
1075                 nes_write_indexed(nesdev,
1076                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1077         }
1078         nes_write_indexed(nesdev,
1079                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1080                         (u32)pmem);
1081
1082         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1083         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1084
1085         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1086                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1087                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1088         }
1089
1090         /* Write Create CCQ WQE */
1091         cqp_head = nesdev->cqp.sq_head++;
1092         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1093         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1094         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1095                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1096                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1097         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1098                             (nesdev->ccq.cq_number |
1099                              ((u32)nesdev->ceq_index << 16)));
1100         u64temp = (u64)nesdev->ccq.cq_pbase;
1101         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1102         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1103         u64temp = (unsigned long)&nesdev->ccq;
1104         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1105                         cpu_to_le32((u32)(u64temp >> 1));
1106         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1107                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1108         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1109
1110         /* Write Create CEQ WQE */
1111         cqp_head = nesdev->cqp.sq_head++;
1112         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1113         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1114         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1115                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1116         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1117         u64temp = (u64)ceq->ceq_pbase;
1118         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1119
1120         /* Write Create AEQ WQE */
1121         cqp_head = nesdev->cqp.sq_head++;
1122         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1123         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1124         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1125                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1126         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1127         u64temp = (u64)aeq->aeq_pbase;
1128         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1129
1130         /* Write Create NIC CEQ WQE */
1131         cqp_head = nesdev->cqp.sq_head++;
1132         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1133         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1134         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1135                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1136         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1137         u64temp = (u64)nic_ceq->ceq_pbase;
1138         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1139
1140         /* Poll until CCQP done */
1141         count = 0;
1142         do {
1143                 if (count++ > 1000) {
1144                         printk(KERN_ERR PFX "Error creating CQP\n");
1145                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1146                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1147                         return -1;
1148                 }
1149                 udelay(10);
1150         } while (!(nes_read_indexed(nesdev,
1151                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1152
1153         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1154                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1155
1156         u32temp = 0x04800000;
1157         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1158
1159         /* wait for the CCQ, CEQ, and AEQ to get created */
1160         count = 0;
1161         do {
1162                 if (count++ > 1000) {
1163                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1164                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1165                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1166                         return -1;
1167                 }
1168                 udelay(10);
1169         } while (((nes_read_indexed(nesdev,
1170                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1171
1172         /* dump the QP status value */
1173         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1174                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1175
1176         nesdev->cqp.sq_tail++;
1177
1178         return 0;
1179 }
1180
1181
1182 /**
1183  * nes_destroy_cqp
1184  */
1185 int nes_destroy_cqp(struct nes_device *nesdev)
1186 {
1187         struct nes_hw_cqp_wqe *cqp_wqe;
1188         u32 count = 0;
1189         u32 cqp_head;
1190         unsigned long flags;
1191
1192         do {
1193                 if (count++ > 1000)
1194                         break;
1195                 udelay(10);
1196         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1197
1198         /* Reset CCQ */
1199         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1200                         nesdev->ccq.cq_number);
1201
1202         /* Disable device interrupts */
1203         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1204
1205         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1206
1207         /* Destroy the AEQ */
1208         cqp_head = nesdev->cqp.sq_head++;
1209         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1210         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1211         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1212                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1213         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1214
1215         /* Destroy the NIC CEQ */
1216         cqp_head = nesdev->cqp.sq_head++;
1217         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1218         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1219         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1220                         ((u32)nesdev->nic_ceq_index << 8));
1221
1222         /* Destroy the CEQ */
1223         cqp_head = nesdev->cqp.sq_head++;
1224         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1225         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1226         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1227                         (nesdev->ceq_index << 8));
1228
1229         /* Destroy the CCQ */
1230         cqp_head = nesdev->cqp.sq_head++;
1231         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1232         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1233         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1234         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1235                         ((u32)nesdev->ceq_index << 16));
1236
1237         /* Destroy CQP */
1238         cqp_head = nesdev->cqp.sq_head++;
1239         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1240         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1241         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1242                         NES_CQP_QP_TYPE_CQP);
1243         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1244
1245         barrier();
1246         /* Ring doorbell (5 WQEs) */
1247         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1248
1249         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1250
1251         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1252         count = 0;
1253         do {
1254                 if (count++ > 1000) {
1255                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1256                                         PCI_FUNC(nesdev->pcidev->devfn));
1257                         break;
1258                 }
1259                 udelay(10);
1260         } while (((nes_read_indexed(nesdev,
1261                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1262
1263         /* dump the QP status value */
1264         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1265                         PCI_FUNC(nesdev->pcidev->devfn),
1266                         nes_read_indexed(nesdev,
1267                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1268
1269         kfree(nesdev->nes_cqp_requests);
1270
1271         /* Free the control structures */
1272         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1273                         nesdev->cqp.sq_pbase);
1274
1275         return 0;
1276 }
1277
1278
1279 /**
1280  * nes_init_phy
1281  */
1282 int nes_init_phy(struct nes_device *nesdev)
1283 {
1284         struct nes_adapter *nesadapter = nesdev->nesadapter;
1285         u32 counter = 0;
1286         u32 sds;
1287         u32 mac_index = nesdev->mac_index;
1288         u32 tx_config = 0;
1289         u16 phy_data;
1290         u32 temp_phy_data = 0;
1291         u32 temp_phy_data2 = 0;
1292         u8  phy_type = nesadapter->phy_type[mac_index];
1293         u8  phy_index = nesadapter->phy_index[mac_index];
1294
1295         if ((nesadapter->OneG_Mode) &&
1296             (phy_type != NES_PHY_TYPE_PUMA_1G)) {
1297                 nes_debug(NES_DBG_PHY, "1G PHY, mac_index = %d.\n", mac_index);
1298                 if (phy_type == NES_PHY_TYPE_1G) {
1299                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1300                         tx_config &= 0xFFFFFFE3;
1301                         tx_config |= 0x04;
1302                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1303                 }
1304
1305                 nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data);
1306                 nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000);
1307
1308                 /* Reset the PHY */
1309                 nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000);
1310                 udelay(100);
1311                 counter = 0;
1312                 do {
1313                         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1314                         if (counter++ > 100)
1315                                 break;
1316                 } while (phy_data & 0x8000);
1317
1318                 /* Setting no phy loopback */
1319                 phy_data &= 0xbfff;
1320                 phy_data |= 0x1140;
1321                 nes_write_1G_phy_reg(nesdev, 0, phy_index,  phy_data);
1322                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1323                 nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data);
1324                 nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data);
1325
1326                 /* Setting the interrupt mask */
1327                 nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1328                 nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee);
1329                 nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1330
1331                 /* turning on flow control */
1332                 nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1333                 nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00);
1334                 nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1335
1336                 /* Clear Half duplex */
1337                 nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1338                 nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100));
1339                 nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1340
1341                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1342                 nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300);
1343
1344                 return 0;
1345         }
1346
1347         if ((phy_type == NES_PHY_TYPE_IRIS) ||
1348             (phy_type == NES_PHY_TYPE_ARGUS) ||
1349             (phy_type == NES_PHY_TYPE_SFP_D)) {
1350                 /* setup 10G MDIO operation */
1351                 tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1352                 tx_config &= 0xFFFFFFE3;
1353                 tx_config |= 0x15;
1354                 nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1355         }
1356         if ((phy_type == NES_PHY_TYPE_ARGUS) ||
1357             (phy_type == NES_PHY_TYPE_SFP_D)) {
1358                 /* Check firmware heartbeat */
1359                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1360                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1361                 udelay(1500);
1362                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1363                 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1364
1365                 if (temp_phy_data != temp_phy_data2)
1366                         return 0;
1367
1368                 /* no heartbeat, configure the PHY */
1369                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000);
1370                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000);
1371                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1372                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1373                 if (phy_type == NES_PHY_TYPE_ARGUS) {
1374                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1375                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008);
1376                         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001);
1377                 } else {
1378                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x0004);
1379                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0038);
1380                         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1381                 }
1382                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1383                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1384
1385                 /* setup LEDs */
1386                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1387                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1388                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1389
1390                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528);
1391
1392                 /* Bring PHY out of reset */
1393                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002);
1394
1395                 /* Check for heartbeat */
1396                 counter = 0;
1397                 mdelay(690);
1398                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1399                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1400                 do {
1401                         if (counter++ > 150) {
1402                                 nes_debug(NES_DBG_PHY, "No PHY heartbeat\n");
1403                                 break;
1404                         }
1405                         mdelay(1);
1406                         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1407                         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1408                 } while ((temp_phy_data2 == temp_phy_data));
1409
1410                 /* wait for tracking */
1411                 counter = 0;
1412                 do {
1413                         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1414                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1415                         if (counter++ > 300) {
1416                                 nes_debug(NES_DBG_PHY, "PHY did not track\n");
1417                                 break;
1418                         }
1419                         mdelay(10);
1420                 } while (((temp_phy_data & 0xff) != 0x50) && ((temp_phy_data & 0xff) != 0x70));
1421
1422                 /* setup signal integrity */
1423                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000);
1424                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE);
1425                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032);
1426                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002);
1427                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063);
1428
1429                 /* reset serdes */
1430                 sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 +
1431                                        mac_index * 0x200);
1432                 sds |= 0x1;
1433                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 +
1434                                   mac_index * 0x200, sds);
1435                 sds &= 0xfffffffe;
1436                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 +
1437                                   mac_index * 0x200, sds);
1438
1439                 counter = 0;
1440                 while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1441                                 && (counter++ < 5000))
1442                         ;
1443         }
1444         return 0;
1445 }
1446
1447
1448 /**
1449  * nes_replenish_nic_rq
1450  */
1451 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1452 {
1453         unsigned long flags;
1454         dma_addr_t bus_address;
1455         struct sk_buff *skb;
1456         struct nes_hw_nic_rq_wqe *nic_rqe;
1457         struct nes_hw_nic *nesnic;
1458         struct nes_device *nesdev;
1459         u32 rx_wqes_posted = 0;
1460
1461         nesnic = &nesvnic->nic;
1462         nesdev = nesvnic->nesdev;
1463         spin_lock_irqsave(&nesnic->rq_lock, flags);
1464         if (nesnic->replenishing_rq !=0) {
1465                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1466                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1467                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1468                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1469                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1470                         add_timer(&nesvnic->rq_wqes_timer);
1471                 } else
1472                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1473                 return;
1474         }
1475         nesnic->replenishing_rq = 1;
1476         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1477         do {
1478                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1479                 if (skb) {
1480                         skb->dev = nesvnic->netdev;
1481
1482                         bus_address = pci_map_single(nesdev->pcidev,
1483                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1484
1485                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1486                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1487                                         cpu_to_le32(nesvnic->max_frame_size);
1488                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1489                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1490                                         cpu_to_le32((u32)bus_address);
1491                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1492                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1493                         nesnic->rx_skb[nesnic->rq_head] = skb;
1494                         nesnic->rq_head++;
1495                         nesnic->rq_head &= nesnic->rq_size - 1;
1496                         atomic_dec(&nesvnic->rx_skbs_needed);
1497                         barrier();
1498                         if (++rx_wqes_posted == 255) {
1499                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1500                                 rx_wqes_posted = 0;
1501                         }
1502                 } else {
1503                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1504                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1505                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1506                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1507                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1508                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1509                                 add_timer(&nesvnic->rq_wqes_timer);
1510                         } else
1511                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1512                         break;
1513                 }
1514         } while (atomic_read(&nesvnic->rx_skbs_needed));
1515         barrier();
1516         if (rx_wqes_posted)
1517                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1518         nesnic->replenishing_rq = 0;
1519 }
1520
1521
1522 /**
1523  * nes_rq_wqes_timeout
1524  */
1525 static void nes_rq_wqes_timeout(unsigned long parm)
1526 {
1527         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1528         printk("%s: Timer fired.\n", __func__);
1529         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1530         if (atomic_read(&nesvnic->rx_skbs_needed))
1531                 nes_replenish_nic_rq(nesvnic);
1532 }
1533
1534
1535 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1536                                void **tcph, u64 *hdr_flags, void *priv)
1537 {
1538         unsigned int ip_len;
1539         struct iphdr *iph;
1540         skb_reset_network_header(skb);
1541         iph = ip_hdr(skb);
1542         if (iph->protocol != IPPROTO_TCP)
1543                 return -1;
1544         ip_len = ip_hdrlen(skb);
1545         skb_set_transport_header(skb, ip_len);
1546         *tcph = tcp_hdr(skb);
1547
1548         *hdr_flags = LRO_IPV4 | LRO_TCP;
1549         *iphdr = iph;
1550         return 0;
1551 }
1552
1553
1554 /**
1555  * nes_init_nic_qp
1556  */
1557 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1558 {
1559         struct nes_hw_cqp_wqe *cqp_wqe;
1560         struct nes_hw_nic_sq_wqe *nic_sqe;
1561         struct nes_hw_nic_qp_context *nic_context;
1562         struct sk_buff *skb;
1563         struct nes_hw_nic_rq_wqe *nic_rqe;
1564         struct nes_vnic *nesvnic = netdev_priv(netdev);
1565         unsigned long flags;
1566         void *vmem;
1567         dma_addr_t pmem;
1568         u64 u64temp;
1569         int ret;
1570         u32 cqp_head;
1571         u32 counter;
1572         u32 wqe_count;
1573         u8 jumbomode=0;
1574
1575         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1576         nesvnic->nic_mem_size = 256 +
1577                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1578                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1579                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1580                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1581                         sizeof(struct nes_hw_nic_qp_context);
1582
1583         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1584                         &nesvnic->nic_pbase);
1585         if (!nesvnic->nic_vbase) {
1586                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1587                 return -ENOMEM;
1588         }
1589         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1590         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1591                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1592
1593         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1594                         ~(unsigned long)(256 - 1));
1595         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1596                         ~(unsigned long long)(256 - 1));
1597
1598         /* Setup the first Fragment buffers */
1599         nesvnic->nic.first_frag_vbase = vmem;
1600
1601         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1602                 nesvnic->nic.frag_paddr[counter] = pmem;
1603                 pmem += sizeof(struct nes_first_frag);
1604         }
1605
1606         /* setup the SQ */
1607         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1608
1609         nesvnic->nic.sq_vbase = (void *)vmem;
1610         nesvnic->nic.sq_pbase = pmem;
1611         nesvnic->nic.sq_head = 0;
1612         nesvnic->nic.sq_tail = 0;
1613         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1614         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1615                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1616                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1617                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1618                                 NES_NIC_SQ_WQE_COMPLETION);
1619                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1620                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1621                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1622                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1623                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1624                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1625         }
1626
1627         nesvnic->get_cqp_request = nes_get_cqp_request;
1628         nesvnic->post_cqp_request = nes_post_cqp_request;
1629         nesvnic->mcrq_mcast_filter = NULL;
1630
1631         spin_lock_init(&nesvnic->nic.rq_lock);
1632
1633         /* setup the RQ */
1634         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1635         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1636
1637
1638         nesvnic->nic.rq_vbase = vmem;
1639         nesvnic->nic.rq_pbase = pmem;
1640         nesvnic->nic.rq_head = 0;
1641         nesvnic->nic.rq_tail = 0;
1642         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1643
1644         /* setup the CQ */
1645         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1646         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1647
1648         if (nesdev->nesadapter->netdev_count > 2)
1649                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1650         else
1651                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1652
1653         nesvnic->nic_cq.cq_vbase = vmem;
1654         nesvnic->nic_cq.cq_pbase = pmem;
1655         nesvnic->nic_cq.cq_head = 0;
1656         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1657
1658         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1659
1660         /* Send CreateCQ request to CQP */
1661         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1662         cqp_head = nesdev->cqp.sq_head;
1663
1664         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1665         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1666
1667         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1668                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1669                         ((u32)nesvnic->nic_cq.cq_size << 16));
1670         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1671                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1672         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1673         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1674         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1675         u64temp = (unsigned long)&nesvnic->nic_cq;
1676         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1677         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1678                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1679         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1680         if (++cqp_head >= nesdev->cqp.sq_size)
1681                 cqp_head = 0;
1682         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1683         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1684
1685         /* Send CreateQP request to CQP */
1686         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1687         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1688                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1689                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1690         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1691                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1692                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1693         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1694                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1695         }
1696
1697         u64temp = (u64)nesvnic->nic.sq_pbase;
1698         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1699         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1700         u64temp = (u64)nesvnic->nic.rq_pbase;
1701         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1702         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1703
1704         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1705                         NES_CQP_QP_TYPE_NIC);
1706         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1707         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1708                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1709         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1710
1711         if (++cqp_head >= nesdev->cqp.sq_size)
1712                 cqp_head = 0;
1713         nesdev->cqp.sq_head = cqp_head;
1714
1715         barrier();
1716
1717         /* Ring doorbell (2 WQEs) */
1718         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1719
1720         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1721         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1722                         nesvnic->nic.qp_id);
1723
1724         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1725                         NES_EVENT_TIMEOUT);
1726         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1727                         nesvnic->nic.qp_id, ret);
1728         if (!ret) {
1729                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1730                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1731                                 nesvnic->nic_pbase);
1732                 return -EIO;
1733         }
1734
1735         /* Populate the RQ */
1736         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1737                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1738                 if (!skb) {
1739                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1740
1741                         nes_destroy_nic_qp(nesvnic);
1742                         return -ENOMEM;
1743                 }
1744
1745                 skb->dev = netdev;
1746
1747                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1748                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1749
1750                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1751                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1752                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1753                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1754                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1755                 nesvnic->nic.rx_skb[counter] = skb;
1756         }
1757
1758         wqe_count = NES_NIC_WQ_SIZE - 1;
1759         nesvnic->nic.rq_head = wqe_count;
1760         barrier();
1761         do {
1762                 counter = min(wqe_count, ((u32)255));
1763                 wqe_count -= counter;
1764                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1765         } while (wqe_count);
1766         init_timer(&nesvnic->rq_wqes_timer);
1767         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1768         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1769         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1770         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1771         {
1772                 nes_nic_init_timer(nesdev);
1773                 if (netdev->mtu > 1500)
1774                         jumbomode = 1;
1775                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1776         }
1777         nesvnic->lro_mgr.max_aggr       = nes_lro_max_aggr;
1778         nesvnic->lro_mgr.max_desc       = NES_MAX_LRO_DESCRIPTORS;
1779         nesvnic->lro_mgr.lro_arr        = nesvnic->lro_desc;
1780         nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1781         nesvnic->lro_mgr.features       = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1782         nesvnic->lro_mgr.dev            = netdev;
1783         nesvnic->lro_mgr.ip_summed      = CHECKSUM_UNNECESSARY;
1784         nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1785         return 0;
1786 }
1787
1788
1789 /**
1790  * nes_destroy_nic_qp
1791  */
1792 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1793 {
1794         u64 u64temp;
1795         dma_addr_t bus_address;
1796         struct nes_device *nesdev = nesvnic->nesdev;
1797         struct nes_hw_cqp_wqe *cqp_wqe;
1798         struct nes_hw_nic_sq_wqe *nic_sqe;
1799         struct nes_hw_nic_rq_wqe *nic_rqe;
1800         __le16 *wqe_fragment_length;
1801         u16  wqe_fragment_index;
1802         u64 wqe_frag;
1803         u32 cqp_head;
1804         unsigned long flags;
1805         int ret;
1806
1807         /* Free remaining NIC receive buffers */
1808         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1809                 nic_rqe   = &nesvnic->nic.rq_vbase[nesvnic->nic.rq_tail];
1810                 wqe_frag  = (u64)le32_to_cpu(
1811                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
1812                 wqe_frag |= ((u64)le32_to_cpu(
1813                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX]))<<32;
1814                 pci_unmap_single(nesdev->pcidev, (dma_addr_t)wqe_frag,
1815                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1816                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1817                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1818         }
1819
1820         /* Free remaining NIC transmit buffers */
1821         while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1822                 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1823                 wqe_fragment_index = 1;
1824                 wqe_fragment_length = (__le16 *)
1825                         &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1826                 /* bump past the vlan tag */
1827                 wqe_fragment_length++;
1828                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1829                         u64temp = (u64)le32_to_cpu(
1830                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1831                                 wqe_fragment_index*2]);
1832                         u64temp += ((u64)le32_to_cpu(
1833                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1834                                 + wqe_fragment_index*2]))<<32;
1835                         bus_address = (dma_addr_t)u64temp;
1836                         if (test_and_clear_bit(nesvnic->nic.sq_tail,
1837                                         nesvnic->nic.first_frag_overflow)) {
1838                                 pci_unmap_single(nesdev->pcidev,
1839                                                 bus_address,
1840                                                 le16_to_cpu(wqe_fragment_length[
1841                                                         wqe_fragment_index++]),
1842                                                 PCI_DMA_TODEVICE);
1843                         }
1844                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1845                                 if (wqe_fragment_length[wqe_fragment_index]) {
1846                                         u64temp = le32_to_cpu(
1847                                                 nic_sqe->wqe_words[
1848                                                 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1849                                                 wqe_fragment_index*2]);
1850                                         u64temp += ((u64)le32_to_cpu(
1851                                                 nic_sqe->wqe_words[
1852                                                 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1853                                                 wqe_fragment_index*2]))<<32;
1854                                         bus_address = (dma_addr_t)u64temp;
1855                                         pci_unmap_page(nesdev->pcidev,
1856                                                         bus_address,
1857                                                         le16_to_cpu(
1858                                                         wqe_fragment_length[
1859                                                         wqe_fragment_index]),
1860                                                         PCI_DMA_TODEVICE);
1861                                 } else
1862                                         break;
1863                         }
1864                 }
1865                 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1866                         dev_kfree_skb(
1867                                 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1868
1869                 nesvnic->nic.sq_tail = (++nesvnic->nic.sq_tail)
1870                                         & (nesvnic->nic.sq_size - 1);
1871         }
1872
1873         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1874
1875         /* Destroy NIC QP */
1876         cqp_head = nesdev->cqp.sq_head;
1877         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1878         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1879
1880         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1881                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1882         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1883                 nesvnic->nic.qp_id);
1884
1885         if (++cqp_head >= nesdev->cqp.sq_size)
1886                 cqp_head = 0;
1887
1888         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1889
1890         /* Destroy NIC CQ */
1891         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1892         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1893                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1894         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1895                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1896
1897         if (++cqp_head >= nesdev->cqp.sq_size)
1898                 cqp_head = 0;
1899
1900         nesdev->cqp.sq_head = cqp_head;
1901         barrier();
1902
1903         /* Ring doorbell (2 WQEs) */
1904         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1905
1906         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1907         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1908                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
1909                         cqp_head, nesdev->cqp.sq_head,
1910                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
1911
1912         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1913                         NES_EVENT_TIMEOUT);
1914
1915         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
1916                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
1917                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
1918         if (!ret) {
1919                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
1920                                 nesvnic->nic.qp_id);
1921         }
1922
1923         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1924                         nesvnic->nic_pbase);
1925 }
1926
1927 /**
1928  * nes_napi_isr
1929  */
1930 int nes_napi_isr(struct nes_device *nesdev)
1931 {
1932         struct nes_adapter *nesadapter = nesdev->nesadapter;
1933         u32 int_stat;
1934
1935         if (nesdev->napi_isr_ran) {
1936                 /* interrupt status has already been read in ISR */
1937                 int_stat = nesdev->int_stat;
1938         } else {
1939                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
1940                 nesdev->int_stat = int_stat;
1941                 nesdev->napi_isr_ran = 1;
1942         }
1943
1944         int_stat &= nesdev->int_req;
1945         /* iff NIC, process here, else wait for DPC */
1946         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
1947                 nesdev->napi_isr_ran = 0;
1948                 nes_write32(nesdev->regs + NES_INT_STAT,
1949                         (int_stat &
1950                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
1951
1952                 /* Process the CEQs */
1953                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
1954
1955                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
1956                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1957                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
1958                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
1959                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
1960                                 /* Enable Periodic timer interrupts */
1961                                 nesdev->int_req |= NES_INT_TIMER;
1962                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
1963                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
1964                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
1965                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
1966                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1967                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1968                         }
1969
1970                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1971                         {
1972                                 nes_nic_init_timer(nesdev);
1973                         }
1974                         /* Enable interrupts, except CEQs */
1975                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1976                 } else {
1977                         /* Enable interrupts, make sure timer is off */
1978                         nesdev->int_req &= ~NES_INT_TIMER;
1979                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1980                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1981                 }
1982                 nesdev->deepcq_count = 0;
1983                 return 1;
1984         } else {
1985                 return 0;
1986         }
1987 }
1988
1989 static void process_critical_error(struct nes_device *nesdev)
1990 {
1991         u32 debug_error;
1992         u32 nes_idx_debug_error_masks0 = 0;
1993         u16 error_module = 0;
1994
1995         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
1996         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
1997                         (u16)debug_error);
1998         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
1999                         0x01010000 | (debug_error & 0x0000ffff));
2000         if (crit_err_count++ > 10)
2001                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
2002         error_module = (u16) (debug_error & 0x1F00) >> 8;
2003         if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
2004                         nes_max_critical_error_count) {
2005                 printk(KERN_ERR PFX "Masking off critical error for module "
2006                         "0x%02X\n", (u16)error_module);
2007                 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
2008                         NES_IDX_DEBUG_ERROR_MASKS0);
2009                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2010                         nes_idx_debug_error_masks0 | (1 << error_module));
2011         }
2012 }
2013 /**
2014  * nes_dpc
2015  */
2016 void nes_dpc(unsigned long param)
2017 {
2018         struct nes_device *nesdev = (struct nes_device *)param;
2019         struct nes_adapter *nesadapter = nesdev->nesadapter;
2020         u32 counter;
2021         u32 loop_counter = 0;
2022         u32 int_status_bit;
2023         u32 int_stat;
2024         u32 timer_stat;
2025         u32 temp_int_stat;
2026         u32 intf_int_stat;
2027         u32 processed_intf_int = 0;
2028         u16 processed_timer_int = 0;
2029         u16 completion_ints = 0;
2030         u16 timer_ints = 0;
2031
2032         /* nes_debug(NES_DBG_ISR, "\n"); */
2033
2034         do {
2035                 timer_stat = 0;
2036                 if (nesdev->napi_isr_ran) {
2037                         nesdev->napi_isr_ran = 0;
2038                         int_stat = nesdev->int_stat;
2039                 } else
2040                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2041                 if (processed_intf_int != 0)
2042                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
2043                 else
2044                         int_stat &= nesdev->int_req;
2045                 if (processed_timer_int == 0) {
2046                         processed_timer_int = 1;
2047                         if (int_stat & NES_INT_TIMER) {
2048                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2049                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
2050                                         int_stat &= ~NES_INT_TIMER;
2051                                 }
2052                         }
2053                 } else {
2054                         int_stat &= ~NES_INT_TIMER;
2055                 }
2056
2057                 if (int_stat) {
2058                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2059                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2060                                 /* Ack the interrupts */
2061                                 nes_write32(nesdev->regs+NES_INT_STAT,
2062                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2063                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2064                         }
2065
2066                         temp_int_stat = int_stat;
2067                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2068                                 if (int_stat & int_status_bit) {
2069                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2070                                         temp_int_stat &= ~int_status_bit;
2071                                         completion_ints = 1;
2072                                 }
2073                                 if (!(temp_int_stat & 0x0000ffff))
2074                                         break;
2075                                 int_status_bit <<= 1;
2076                         }
2077
2078                         /* Process the AEQ for this pci function */
2079                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2080                         if (int_stat & int_status_bit) {
2081                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2082                         }
2083
2084                         /* Process the MAC interrupt for this pci function */
2085                         int_status_bit = 1 << (24 + nesdev->mac_index);
2086                         if (int_stat & int_status_bit) {
2087                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
2088                         }
2089
2090                         if (int_stat & NES_INT_TIMER) {
2091                                 if (timer_stat & nesdev->timer_int_req) {
2092                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
2093                                                         (timer_stat & nesdev->timer_int_req) |
2094                                                         ~(nesdev->nesadapter->timer_int_req));
2095                                         timer_ints = 1;
2096                                 }
2097                         }
2098
2099                         if (int_stat & NES_INT_INTF) {
2100                                 processed_intf_int = 1;
2101                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2102                                 intf_int_stat &= nesdev->intf_int_req;
2103                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2104                                         process_critical_error(nesdev);
2105                                 }
2106                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2107                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2108                                         BUG();
2109                                 }
2110                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2111                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2112                                         BUG();
2113                                 }
2114                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2115                         }
2116
2117                         if (int_stat & NES_INT_TSW) {
2118                         }
2119                 }
2120                 /* Don't use the interface interrupt bit stay in loop */
2121                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2122                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2123         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2124
2125         if (timer_ints == 1) {
2126                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2127                         if (completion_ints == 0) {
2128                                 nesdev->timer_only_int_count++;
2129                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2130                                         nesdev->timer_only_int_count = 0;
2131                                         nesdev->int_req &= ~NES_INT_TIMER;
2132                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2133                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2134                                 } else {
2135                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2136                                 }
2137                         } else {
2138                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2139                                 {
2140                                         nes_nic_init_timer(nesdev);
2141                                 }
2142                                 nesdev->timer_only_int_count = 0;
2143                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2144                         }
2145                 } else {
2146                         nesdev->timer_only_int_count = 0;
2147                         nesdev->int_req &= ~NES_INT_TIMER;
2148                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2149                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2150                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2151                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2152                 }
2153         } else {
2154                 if ( (completion_ints == 1) &&
2155                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2156                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2157                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2158                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2159                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2160                         nesdev->timer_only_int_count = 0;
2161                         nesdev->int_req |= NES_INT_TIMER;
2162                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2163                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2164                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2165                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2166                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2167                 } else {
2168                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2169                 }
2170         }
2171         nesdev->deepcq_count = 0;
2172 }
2173
2174
2175 /**
2176  * nes_process_ceq
2177  */
2178 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2179 {
2180         u64 u64temp;
2181         struct nes_hw_cq *cq;
2182         u32 head;
2183         u32 ceq_size;
2184
2185         /* nes_debug(NES_DBG_CQ, "\n"); */
2186         head = ceq->ceq_head;
2187         ceq_size = ceq->ceq_size;
2188
2189         do {
2190                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2191                                 NES_CEQE_VALID) {
2192                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2193                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2194                         u64temp <<= 1;
2195                         cq = *((struct nes_hw_cq **)&u64temp);
2196                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2197                         barrier();
2198                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2199
2200                         /* call the event handler */
2201                         cq->ce_handler(nesdev, cq);
2202
2203                         if (++head >= ceq_size)
2204                                 head = 0;
2205                 } else {
2206                         break;
2207                 }
2208
2209         } while (1);
2210
2211         ceq->ceq_head = head;
2212 }
2213
2214
2215 /**
2216  * nes_process_aeq
2217  */
2218 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2219 {
2220         /* u64 u64temp; */
2221         u32 head;
2222         u32 aeq_size;
2223         u32 aeqe_misc;
2224         u32 aeqe_cq_id;
2225         struct nes_hw_aeqe volatile *aeqe;
2226
2227         head = aeq->aeq_head;
2228         aeq_size = aeq->aeq_size;
2229
2230         do {
2231                 aeqe = &aeq->aeq_vbase[head];
2232                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2233                         break;
2234                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2235                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2236                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2237                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2238                                 /* dealing with an accelerated QP related AE */
2239                                 /*
2240                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2241                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2242                                  */
2243                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2244                         } else {
2245                                 /* TODO: dealing with a CQP related AE */
2246                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2247                                                 (u16)(aeqe_misc >> 16));
2248                         }
2249                 }
2250
2251                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2252
2253                 if (++head >= aeq_size)
2254                         head = 0;
2255
2256                 nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16);
2257         }
2258         while (1);
2259         aeq->aeq_head = head;
2260 }
2261
2262 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2263 {
2264         struct nes_adapter *nesadapter = nesdev->nesadapter;
2265         u32 reset_value;
2266         u32 i=0;
2267         u32 u32temp;
2268
2269         if (nesadapter->hw_rev == NE020_REV) {
2270                 return;
2271         }
2272         mh_detected++;
2273
2274         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2275
2276         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2277                 reset_value |= 0x0000001d;
2278         else
2279                 reset_value |= 0x0000002d;
2280
2281         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2282                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2283                         nesadapter->link_interrupt_count[0] = 0;
2284                         nesadapter->link_interrupt_count[1] = 0;
2285                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2286                         if (0x00000040 & u32temp)
2287                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2288                         else
2289                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2290
2291                         reset_value |= 0x0000003d;
2292                 }
2293                 nesadapter->link_interrupt_count[mac_index] = 0;
2294         }
2295
2296         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2297
2298         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2299                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2300
2301         if (0x0000003d == (reset_value & 0x0000003d)) {
2302                 u32 pcs_control_status0, pcs_control_status1;
2303
2304                 for (i = 0; i < 10; i++) {
2305                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2306                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2307                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2308                              && (pcs_control_status0 & 0x00100000))
2309                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2310                                 && (pcs_control_status1 & 0x00100000)))
2311                                 continue;
2312                         else
2313                                 break;
2314                 }
2315                 if (10 == i) {
2316                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2317                         if (0x00000040 & u32temp)
2318                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2319                         else
2320                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2321
2322                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2323
2324                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2325                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2326                 }
2327         }
2328 }
2329
2330 /**
2331  * nes_process_mac_intr
2332  */
2333 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2334 {
2335         unsigned long flags;
2336         u32 pcs_control_status;
2337         struct nes_adapter *nesadapter = nesdev->nesadapter;
2338         struct nes_vnic *nesvnic;
2339         u32 mac_status;
2340         u32 mac_index = nesdev->mac_index;
2341         u32 u32temp;
2342         u16 phy_data;
2343         u16 temp_phy_data;
2344         u32 pcs_val  = 0x0f0f0000;
2345         u32 pcs_mask = 0x0f1f0000;
2346         u32 cdr_ctrl;
2347
2348         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2349         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2350                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2351                 return;
2352         }
2353         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2354         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2355
2356         /* ack the MAC interrupt */
2357         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2358         /* Clear the interrupt */
2359         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2360
2361         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2362
2363         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2364                 nesdev->link_status_interrupts++;
2365                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) {
2366                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2367                         nes_reset_link(nesdev, mac_index);
2368                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2369                 }
2370                 /* read the PHY interrupt status register */
2371                 if ((nesadapter->OneG_Mode) &&
2372                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2373                         do {
2374                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2375                                                 nesadapter->phy_index[mac_index], &phy_data);
2376                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2377                                                 nesadapter->phy_index[mac_index], phy_data);
2378                         } while (phy_data&0x8000);
2379
2380                         temp_phy_data = 0;
2381                         do {
2382                                 nes_read_1G_phy_reg(nesdev, 0x11,
2383                                                 nesadapter->phy_index[mac_index], &phy_data);
2384                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2385                                                 nesadapter->phy_index[mac_index], phy_data);
2386                                 if (temp_phy_data == phy_data)
2387                                         break;
2388                                 temp_phy_data = phy_data;
2389                         } while (1);
2390
2391                         nes_read_1G_phy_reg(nesdev, 0x1e,
2392                                         nesadapter->phy_index[mac_index], &phy_data);
2393                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2394                                         nesadapter->phy_index[mac_index], phy_data);
2395
2396                         nes_read_1G_phy_reg(nesdev, 1,
2397                                         nesadapter->phy_index[mac_index], &phy_data);
2398                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2399                                         nesadapter->phy_index[mac_index], phy_data);
2400
2401                         if (temp_phy_data & 0x1000) {
2402                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2403                                 phy_data = 4;
2404                         } else {
2405                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2406                         }
2407                 }
2408                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2409                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2410                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2411
2412                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2413                         switch (mac_index) {
2414                         case 1:
2415                         case 3:
2416                                 pcs_control_status = nes_read_indexed(nesdev,
2417                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2418                                 break;
2419                         default:
2420                                 pcs_control_status = nes_read_indexed(nesdev,
2421                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2422                                 break;
2423                         }
2424                 } else {
2425                         pcs_control_status = nes_read_indexed(nesdev,
2426                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2427                         pcs_control_status = nes_read_indexed(nesdev,
2428                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2429                 }
2430
2431                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2432                                 mac_index, pcs_control_status);
2433                 if ((nesadapter->OneG_Mode) &&
2434                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2435                         u32temp = 0x01010000;
2436                         if (nesadapter->port_count > 2) {
2437                                 u32temp |= 0x02020000;
2438                         }
2439                         if ((pcs_control_status & u32temp)!= u32temp) {
2440                                 phy_data = 0;
2441                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2442                         }
2443                 } else {
2444                         switch (nesadapter->phy_type[mac_index]) {
2445                         case NES_PHY_TYPE_IRIS:
2446                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2447                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2448                                 u32temp = 20;
2449                                 do {
2450                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2451                                         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2452                                         if ((phy_data == temp_phy_data) || (!(--u32temp)))
2453                                                 break;
2454                                         temp_phy_data = phy_data;
2455                                 } while (1);
2456                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2457                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2458                                 break;
2459
2460                         case NES_PHY_TYPE_ARGUS:
2461                         case NES_PHY_TYPE_SFP_D:
2462                                 /* clear the alarms */
2463                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2464                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2465                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2466                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2467                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2468                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2469                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2470                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2471                                 /* check link status */
2472                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2473                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2474
2475                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2476                                 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2477                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2478                                 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2479
2480                                 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2481
2482                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2483                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2484                                 break;
2485
2486                         case NES_PHY_TYPE_PUMA_1G:
2487                                 if (mac_index < 2)
2488                                         pcs_val = pcs_mask = 0x01010000;
2489                                 else
2490                                         pcs_val = pcs_mask = 0x02020000;
2491                                 /* fall through */
2492                         default:
2493                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2494                                 break;
2495                         }
2496                 }
2497
2498                 if (phy_data & 0x0004) {
2499                         if (wide_ppm_offset &&
2500                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2501                             (nesadapter->hw_rev != NE020_REV)) {
2502                                 cdr_ctrl = nes_read_indexed(nesdev,
2503                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2504                                                             mac_index * 0x200);
2505                                 nes_write_indexed(nesdev,
2506                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2507                                                   mac_index * 0x200,
2508                                                   cdr_ctrl | 0x000F0000);
2509                         }
2510                         nesadapter->mac_link_down[mac_index] = 0;
2511                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2512                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2513                                                 nesvnic->linkup);
2514                                 if (nesvnic->linkup == 0) {
2515                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2516                                                         nesvnic->netdev->name, nesvnic->netdev);
2517                                         if (netif_queue_stopped(nesvnic->netdev))
2518                                                 netif_start_queue(nesvnic->netdev);
2519                                         nesvnic->linkup = 1;
2520                                         netif_carrier_on(nesvnic->netdev);
2521                                 }
2522                         }
2523                 } else {
2524                         if (wide_ppm_offset &&
2525                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2526                             (nesadapter->hw_rev != NE020_REV)) {
2527                                 cdr_ctrl = nes_read_indexed(nesdev,
2528                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2529                                                             mac_index * 0x200);
2530                                 nes_write_indexed(nesdev,
2531                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2532                                                   mac_index * 0x200,
2533                                                   cdr_ctrl & 0xFFF0FFFF);
2534                         }
2535                         nesadapter->mac_link_down[mac_index] = 1;
2536                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2537                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2538                                                 nesvnic->linkup);
2539                                 if (nesvnic->linkup == 1) {
2540                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2541                                                         nesvnic->netdev->name, nesvnic->netdev);
2542                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2543                                                 netif_stop_queue(nesvnic->netdev);
2544                                         nesvnic->linkup = 0;
2545                                         netif_carrier_off(nesvnic->netdev);
2546                                 }
2547                         }
2548                 }
2549         }
2550
2551         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2552 }
2553
2554
2555
2556 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2557 {
2558         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2559
2560         napi_schedule(&nesvnic->napi);
2561 }
2562
2563
2564 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2565 * getting out of nic_ce_handler
2566 */
2567 #define MAX_RQES_TO_PROCESS     384
2568
2569 /**
2570  * nes_nic_ce_handler
2571  */
2572 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2573 {
2574         u64 u64temp;
2575         dma_addr_t bus_address;
2576         struct nes_hw_nic *nesnic;
2577         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2578         struct nes_adapter *nesadapter = nesdev->nesadapter;
2579         struct nes_hw_nic_rq_wqe *nic_rqe;
2580         struct nes_hw_nic_sq_wqe *nic_sqe;
2581         struct sk_buff *skb;
2582         struct sk_buff *rx_skb;
2583         __le16 *wqe_fragment_length;
2584         u32 head;
2585         u32 cq_size;
2586         u32 rx_pkt_size;
2587         u32 cqe_count=0;
2588         u32 cqe_errv;
2589         u32 cqe_misc;
2590         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2591         u16 vlan_tag;
2592         u16 pkt_type;
2593         u16 rqes_processed = 0;
2594         u8 sq_cqes = 0;
2595         u8 nes_use_lro = 0;
2596
2597         head = cq->cq_head;
2598         cq_size = cq->cq_size;
2599         cq->cqes_pending = 1;
2600         if (nesvnic->netdev->features & NETIF_F_LRO)
2601                 nes_use_lro = 1;
2602         do {
2603                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2604                                 NES_NIC_CQE_VALID) {
2605                         nesnic = &nesvnic->nic;
2606                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2607                         if (cqe_misc & NES_NIC_CQE_SQ) {
2608                                 sq_cqes++;
2609                                 wqe_fragment_index = 1;
2610                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2611                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2612                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2613                                 /* bump past the vlan tag */
2614                                 wqe_fragment_length++;
2615                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2616                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2617                                                         wqe_fragment_index * 2]);
2618                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2619                                                         wqe_fragment_index * 2])) << 32;
2620                                         bus_address = (dma_addr_t)u64temp;
2621                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2622                                                 pci_unmap_single(nesdev->pcidev,
2623                                                                 bus_address,
2624                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2625                                                                 PCI_DMA_TODEVICE);
2626                                         }
2627                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2628                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2629                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2630                                                                                 wqe_fragment_index * 2]);
2631                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2632                                                                                 + wqe_fragment_index * 2])) <<32;
2633                                                         bus_address = (dma_addr_t)u64temp;
2634                                                         pci_unmap_page(nesdev->pcidev,
2635                                                                         bus_address,
2636                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2637                                                                         PCI_DMA_TODEVICE);
2638                                                 } else
2639                                                         break;
2640                                         }
2641                                 }
2642                                 if (skb)
2643                                         dev_kfree_skb_any(skb);
2644                                 nesnic->sq_tail++;
2645                                 nesnic->sq_tail &= nesnic->sq_size-1;
2646                                 if (sq_cqes > 128) {
2647                                         barrier();
2648                                 /* restart the queue if it had been stopped */
2649                                 if (netif_queue_stopped(nesvnic->netdev))
2650                                         netif_wake_queue(nesvnic->netdev);
2651                                         sq_cqes = 0;
2652                                 }
2653                         } else {
2654                                 rqes_processed ++;
2655
2656                                 cq->rx_cqes_completed++;
2657                                 cq->rx_pkts_indicated++;
2658                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2659                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2660                                 /* Get the skb */
2661                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2662                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2663                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2664                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2665                                 pci_unmap_single(nesdev->pcidev, bus_address,
2666                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2667                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2668                                 /* rx_skb->len = rx_pkt_size; */
2669                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2670                                 skb_put(rx_skb, rx_pkt_size);
2671                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2672                                 nesnic->rq_tail++;
2673                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2674
2675                                 atomic_inc(&nesvnic->rx_skbs_needed);
2676                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2677                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2678                                                         cq->cq_number | (cqe_count << 16));
2679                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2680                                         nesdev->currcq_count += cqe_count;
2681                                         cqe_count = 0;
2682                                         nes_replenish_nic_rq(nesvnic);
2683                                 }
2684                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2685                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2686                                 rx_skb->ip_summed = CHECKSUM_NONE;
2687
2688                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2689                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2690                                         if ((cqe_errv &
2691                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2692                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2693                                                 if (nesvnic->rx_checksum_disabled == 0) {
2694                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2695                                                 }
2696                                         } else
2697                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2698                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2699                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2700
2701                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2702                                         if ((cqe_errv &
2703                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2704                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2705                                                 if (nesvnic->rx_checksum_disabled == 0) {
2706                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2707                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2708                                                                   nesvnic->netdev->name); */
2709                                                 }
2710                                         } else
2711                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2712                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2713                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2714                                         }
2715                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2716                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2717
2718                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2719                                         if (nes_cm_recv(rx_skb, nesvnic->netdev))
2720                                                 rx_skb = NULL;
2721                                 }
2722                                 if (rx_skb == NULL)
2723                                         goto skip_rx_indicate0;
2724
2725
2726                                 if ((cqe_misc & NES_NIC_CQE_TAG_VALID) &&
2727                                     (nesvnic->vlan_grp != NULL)) {
2728                                         vlan_tag = (u16)(le32_to_cpu(
2729                                                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2730                                                         >> 16);
2731                                         nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2732                                                         nesvnic->netdev->name, vlan_tag);
2733                                         if (nes_use_lro)
2734                                                 lro_vlan_hwaccel_receive_skb(&nesvnic->lro_mgr, rx_skb,
2735                                                                 nesvnic->vlan_grp, vlan_tag, NULL);
2736                                         else
2737                                                 nes_vlan_rx(rx_skb, nesvnic->vlan_grp, vlan_tag);
2738                                 } else {
2739                                         if (nes_use_lro)
2740                                                 lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2741                                         else
2742                                                 nes_netif_rx(rx_skb);
2743                                 }
2744
2745 skip_rx_indicate0:
2746                                 nesvnic->netdev->last_rx = jiffies;
2747                                 /* nesvnic->netstats.rx_packets++; */
2748                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2749                         }
2750
2751                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2752                         /* Accounting... */
2753                         cqe_count++;
2754                         if (++head >= cq_size)
2755                                 head = 0;
2756                         if (cqe_count == 255) {
2757                                 /* Replenish Nic CQ */
2758                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2759                                                 cq->cq_number | (cqe_count << 16));
2760                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2761                                 nesdev->currcq_count += cqe_count;
2762                                 cqe_count = 0;
2763                         }
2764
2765                         if (cq->rx_cqes_completed >= nesvnic->budget)
2766                                 break;
2767                 } else {
2768                         cq->cqes_pending = 0;
2769                         break;
2770                 }
2771
2772         } while (1);
2773
2774         if (nes_use_lro)
2775                 lro_flush_all(&nesvnic->lro_mgr);
2776         if (sq_cqes) {
2777                 barrier();
2778                 /* restart the queue if it had been stopped */
2779                 if (netif_queue_stopped(nesvnic->netdev))
2780                         netif_wake_queue(nesvnic->netdev);
2781         }
2782         cq->cq_head = head;
2783         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2784                         cq->cq_number, cqe_count, cq->cq_head); */
2785         cq->cqe_allocs_pending = cqe_count;
2786         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2787         {
2788                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2789                 nesdev->currcq_count += cqe_count;
2790                 nes_nic_tune_timer(nesdev);
2791         }
2792         if (atomic_read(&nesvnic->rx_skbs_needed))
2793                 nes_replenish_nic_rq(nesvnic);
2794 }
2795
2796
2797 /**
2798  * nes_cqp_ce_handler
2799  */
2800 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2801 {
2802         u64 u64temp;
2803         unsigned long flags;
2804         struct nes_hw_cqp *cqp = NULL;
2805         struct nes_cqp_request *cqp_request;
2806         struct nes_hw_cqp_wqe *cqp_wqe;
2807         u32 head;
2808         u32 cq_size;
2809         u32 cqe_count=0;
2810         u32 error_code;
2811         /* u32 counter; */
2812
2813         head = cq->cq_head;
2814         cq_size = cq->cq_size;
2815
2816         do {
2817                 /* process the CQE */
2818                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2819                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2820
2821                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
2822                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
2823                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
2824                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
2825                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
2826                         cqp = *((struct nes_hw_cqp **)&u64temp);
2827
2828                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2829                         if (error_code) {
2830                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2831                                                 " Major/Minor codes = 0x%04X:%04X.\n",
2832                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
2833                                                 (u16)(error_code >> 16),
2834                                                 (u16)error_code);
2835                                 nes_debug(NES_DBG_CQP, "cqp: qp_id=%u, sq_head=%u, sq_tail=%u\n",
2836                                                 cqp->qp_id, cqp->sq_head, cqp->sq_tail);
2837                         }
2838
2839                         u64temp = (((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2840                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX]))) << 32) |
2841                                         ((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2842                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX])));
2843                         cqp_request = *((struct nes_cqp_request **)&u64temp);
2844                         if (cqp_request) {
2845                                 if (cqp_request->waiting) {
2846                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
2847                                         cqp_request->major_code = (u16)(error_code >> 16);
2848                                         cqp_request->minor_code = (u16)error_code;
2849                                         barrier();
2850                                         cqp_request->request_done = 1;
2851                                         wake_up(&cqp_request->waitq);
2852                                         nes_put_cqp_request(nesdev, cqp_request);
2853                                 } else {
2854                                         if (cqp_request->callback)
2855                                                 cqp_request->cqp_callback(nesdev, cqp_request);
2856                                         nes_free_cqp_request(nesdev, cqp_request);
2857                                 }
2858                         } else {
2859                                 wake_up(&nesdev->cqp.waitq);
2860                         }
2861
2862                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
2863                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
2864                         if (++cqp->sq_tail >= cqp->sq_size)
2865                                 cqp->sq_tail = 0;
2866
2867                         /* Accounting... */
2868                         cqe_count++;
2869                         if (++head >= cq_size)
2870                                 head = 0;
2871                 } else {
2872                         break;
2873                 }
2874         } while (1);
2875         cq->cq_head = head;
2876
2877         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2878         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
2879                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
2880                         (nesdev->cqp.sq_size - 1)) != 1)) {
2881                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
2882                                 struct nes_cqp_request, list);
2883                 list_del_init(&cqp_request->list);
2884                 head = nesdev->cqp.sq_head++;
2885                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
2886                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
2887                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
2888                 barrier();
2889                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] =
2890                         cpu_to_le32((u32)((unsigned long)cqp_request));
2891                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] =
2892                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
2893                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
2894                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
2895                 /* Ring doorbell (1 WQEs) */
2896                 barrier();
2897                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
2898         }
2899         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2900
2901         /* Arm the CCQ */
2902         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
2903                         cq->cq_number);
2904         nes_read32(nesdev->regs+NES_CQE_ALLOC);
2905 }
2906
2907
2908 static u8 *locate_mpa(u8 *pkt, u32 aeq_info)
2909 {
2910         u16 pkt_len;
2911
2912         if (aeq_info & NES_AEQE_Q2_DATA_ETHERNET) {
2913                 /* skip over ethernet header */
2914                 pkt_len = be16_to_cpu(*(u16 *)(pkt + ETH_HLEN - 2));
2915                 pkt += ETH_HLEN;
2916
2917                 /* Skip over IP and TCP headers */
2918                 pkt += 4 * (pkt[0] & 0x0f);
2919                 pkt += 4 * ((pkt[12] >> 4) & 0x0f);
2920         }
2921         return pkt;
2922 }
2923
2924 /* Determine if incoming error pkt is rdma layer */
2925 static u32 iwarp_opcode(struct nes_qp *nesqp, u32 aeq_info)
2926 {
2927         u8 *pkt;
2928         u16 *mpa;
2929         u32 opcode = 0xffffffff;
2930
2931         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
2932                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
2933                 mpa = (u16 *)locate_mpa(pkt, aeq_info);
2934                 opcode = be16_to_cpu(mpa[1]) & 0xf;
2935         }
2936
2937         return opcode;
2938 }
2939
2940 /* Build iWARP terminate header */
2941 static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 aeq_info)
2942 {
2943         u8 *pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
2944         u16 ddp_seg_len;
2945         int copy_len = 0;
2946         u8 is_tagged = 0;
2947         struct nes_terminate_hdr *termhdr;
2948
2949         termhdr = (struct nes_terminate_hdr *)nesqp->hwqp.q2_vbase;
2950         memset(termhdr, 0, 64);
2951
2952         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
2953
2954                 /* Use data from offending packet to fill in ddp & rdma hdrs */
2955                 pkt = locate_mpa(pkt, aeq_info);
2956                 ddp_seg_len = be16_to_cpu(*(u16 *)pkt);
2957                 if (ddp_seg_len) {
2958                         copy_len = 2;
2959                         termhdr->hdrct = DDP_LEN_FLAG;
2960                         if (pkt[2] & 0x80) {
2961                                 is_tagged = 1;
2962                                 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
2963                                         copy_len += TERM_DDP_LEN_TAGGED;
2964                                         termhdr->hdrct |= DDP_HDR_FLAG;
2965                                 }
2966                         } else {
2967                                 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
2968                                         copy_len += TERM_DDP_LEN_UNTAGGED;
2969                                         termhdr->hdrct |= DDP_HDR_FLAG;
2970                                 }
2971
2972                                 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {
2973                                         if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {
2974                                                 copy_len += TERM_RDMA_LEN;
2975                                                 termhdr->hdrct |= RDMA_HDR_FLAG;
2976                                         }
2977                                 }
2978                         }
2979                 }
2980         }
2981
2982         switch (async_event_id) {
2983         case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
2984                 switch (iwarp_opcode(nesqp, aeq_info)) {
2985                 case IWARP_OPCODE_WRITE:
2986                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
2987                         termhdr->error_code = DDP_TAGGED_INV_STAG;
2988                         break;
2989                 default:
2990                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
2991                         termhdr->error_code = RDMAP_INV_STAG;
2992                 }
2993                 break;
2994         case NES_AEQE_AEID_AMP_INVALID_STAG:
2995                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
2996                 termhdr->error_code = RDMAP_INV_STAG;
2997                 break;
2998         case NES_AEQE_AEID_AMP_BAD_QP:
2999                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3000                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3001                 break;
3002         case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3003         case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3004                 switch (iwarp_opcode(nesqp, aeq_info)) {
3005                 case IWARP_OPCODE_SEND_INV:
3006                 case IWARP_OPCODE_SEND_SE_INV:
3007                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3008                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3009                         break;
3010                 default:
3011                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3012                         termhdr->error_code = RDMAP_INV_STAG;
3013                 }
3014                 break;
3015         case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3016                 if (aeq_info & (NES_AEQE_Q2_DATA_ETHERNET | NES_AEQE_Q2_DATA_MPA)) {
3017                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3018                         termhdr->error_code = DDP_TAGGED_BOUNDS;
3019                 } else {
3020                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3021                         termhdr->error_code = RDMAP_INV_BOUNDS;
3022                 }
3023                 break;
3024         case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3025         case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3026         case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3027                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3028                 termhdr->error_code = RDMAP_ACCESS;
3029                 break;
3030         case NES_AEQE_AEID_AMP_TO_WRAP:
3031                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3032                 termhdr->error_code = RDMAP_TO_WRAP;
3033                 break;
3034         case NES_AEQE_AEID_AMP_BAD_PD:
3035                 switch (iwarp_opcode(nesqp, aeq_info)) {
3036                 case IWARP_OPCODE_WRITE:
3037                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3038                         termhdr->error_code = DDP_TAGGED_UNASSOC_STAG;
3039                         break;
3040                 case IWARP_OPCODE_SEND_INV:
3041                 case IWARP_OPCODE_SEND_SE_INV:
3042                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3043                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3044                         break;
3045                 default:
3046                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3047                         termhdr->error_code = RDMAP_UNASSOC_STAG;
3048                 }
3049                 break;
3050         case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3051                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3052                 termhdr->error_code = MPA_MARKER;
3053                 break;
3054         case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3055                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3056                 termhdr->error_code = MPA_CRC;
3057                 break;
3058         case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3059         case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3060                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3061                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3062                 break;
3063         case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3064         case NES_AEQE_AEID_DDP_NO_L_BIT:
3065                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3066                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3067                 break;
3068         case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3069         case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3070                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3071                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_RANGE;
3072                 break;
3073         case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3074                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3075                 termhdr->error_code = DDP_UNTAGGED_INV_TOO_LONG;
3076                 break;
3077         case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3078                 if (is_tagged) {
3079                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3080                         termhdr->error_code = DDP_TAGGED_INV_DDP_VER;
3081                 } else {
3082                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3083                         termhdr->error_code = DDP_UNTAGGED_INV_DDP_VER;
3084                 }
3085                 break;
3086         case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3087                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3088                 termhdr->error_code = DDP_UNTAGGED_INV_MO;
3089                 break;
3090         case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3091                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3092                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_NO_BUF;
3093                 break;
3094         case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3095                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3096                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3097                 break;
3098         case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3099                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3100                 termhdr->error_code = RDMAP_INV_RDMAP_VER;
3101                 break;
3102         case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3103                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3104                 termhdr->error_code = RDMAP_UNEXPECTED_OP;
3105                 break;
3106         default:
3107                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3108                 termhdr->error_code = RDMAP_UNSPECIFIED;
3109                 break;
3110         }
3111
3112         if (copy_len)
3113                 memcpy(termhdr + 1, pkt, copy_len);
3114
3115         return sizeof(struct nes_terminate_hdr) + copy_len;
3116 }
3117
3118 static void nes_terminate_connection(struct nes_device *nesdev, struct nes_qp *nesqp,
3119                  struct nes_hw_aeqe *aeqe, enum ib_event_type eventtype)
3120 {
3121         u64 context;
3122         unsigned long flags;
3123         u32 aeq_info;
3124         u16 async_event_id;
3125         u8 tcp_state;
3126         u8 iwarp_state;
3127         u32 termlen = 0;
3128         u32 mod_qp_flags = NES_CQP_QP_IWARP_STATE_TERMINATE |
3129                            NES_CQP_QP_TERM_DONT_SEND_FIN;
3130         struct nes_adapter *nesadapter = nesdev->nesadapter;
3131
3132         if (nesqp->term_flags & NES_TERM_SENT)
3133                 return; /* Sanity check */
3134
3135         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3136         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3137         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3138         async_event_id = (u16)aeq_info;
3139
3140         context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3141                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3142         if (!context) {
3143                 WARN_ON(!context);
3144                 return;
3145         }
3146
3147         nesqp = (struct nes_qp *)(unsigned long)context;
3148         spin_lock_irqsave(&nesqp->lock, flags);
3149         nesqp->hw_iwarp_state = iwarp_state;
3150         nesqp->hw_tcp_state = tcp_state;
3151         nesqp->last_aeq = async_event_id;
3152         nesqp->terminate_eventtype = eventtype;
3153         spin_unlock_irqrestore(&nesqp->lock, flags);
3154
3155         if (nesadapter->send_term_ok)
3156                 termlen = nes_bld_terminate_hdr(nesqp, async_event_id, aeq_info);
3157         else
3158                 mod_qp_flags |= NES_CQP_QP_TERM_DONT_SEND_TERM_MSG;
3159
3160         nes_terminate_start_timer(nesqp);
3161         nesqp->term_flags |= NES_TERM_SENT;
3162         nes_hw_modify_qp(nesdev, nesqp, mod_qp_flags, termlen, 0);
3163 }
3164
3165 static void nes_terminate_send_fin(struct nes_device *nesdev,
3166                           struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3167 {
3168         u32 aeq_info;
3169         u16 async_event_id;
3170         u8 tcp_state;
3171         u8 iwarp_state;
3172         unsigned long flags;
3173
3174         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3175         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3176         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3177         async_event_id = (u16)aeq_info;
3178
3179         spin_lock_irqsave(&nesqp->lock, flags);
3180         nesqp->hw_iwarp_state = iwarp_state;
3181         nesqp->hw_tcp_state = tcp_state;
3182         nesqp->last_aeq = async_event_id;
3183         spin_unlock_irqrestore(&nesqp->lock, flags);
3184
3185         /* Send the fin only */
3186         nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_TERMINATE |
3187                 NES_CQP_QP_TERM_DONT_SEND_TERM_MSG, 0, 0);
3188 }
3189
3190 /* Cleanup after a terminate sent or received */
3191 static void nes_terminate_done(struct nes_qp *nesqp, int timeout_occurred)
3192 {
3193         u32 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR;
3194         unsigned long flags;
3195         struct nes_vnic *nesvnic = to_nesvnic(nesqp->ibqp.device);
3196         struct nes_device *nesdev = nesvnic->nesdev;
3197         u8 first_time = 0;
3198
3199         spin_lock_irqsave(&nesqp->lock, flags);
3200         if (nesqp->hte_added) {
3201                 nesqp->hte_added = 0;
3202                 next_iwarp_state |= NES_CQP_QP_DEL_HTE;
3203         }
3204
3205         first_time = (nesqp->term_flags & NES_TERM_DONE) == 0;
3206         nesqp->term_flags |= NES_TERM_DONE;
3207         spin_unlock_irqrestore(&nesqp->lock, flags);
3208
3209         /* Make sure we go through this only once */
3210         if (first_time) {
3211                 if (timeout_occurred == 0)
3212                         del_timer(&nesqp->terminate_timer);
3213                 else
3214                         next_iwarp_state |= NES_CQP_QP_RESET;
3215
3216                 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3217                 nes_cm_disconn(nesqp);
3218         }
3219 }
3220
3221 static void nes_terminate_received(struct nes_device *nesdev,
3222                                 struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3223 {
3224         u32 aeq_info;
3225         u8 *pkt;
3226         u32 *mpa;
3227         u8 ddp_ctl;
3228         u8 rdma_ctl;
3229         u16 aeq_id = 0;
3230
3231         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3232         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3233                 /* Terminate is not a performance path so the silicon */
3234                 /* did not validate the frame - do it now */
3235                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3236                 mpa = (u32 *)locate_mpa(pkt, aeq_info);
3237                 ddp_ctl = (be32_to_cpu(mpa[0]) >> 8) & 0xff;
3238                 rdma_ctl = be32_to_cpu(mpa[0]) & 0xff;
3239                 if ((ddp_ctl & 0xc0) != 0x40)
3240                         aeq_id = NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC;
3241                 else if ((ddp_ctl & 0x03) != 1)
3242                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION;
3243                 else if (be32_to_cpu(mpa[2]) != 2)
3244                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_QN;
3245                 else if (be32_to_cpu(mpa[3]) != 1)
3246                         aeq_id = NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN;
3247                 else if (be32_to_cpu(mpa[4]) != 0)
3248                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_MO;
3249                 else if ((rdma_ctl & 0xc0) != 0x40)
3250                         aeq_id = NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION;
3251
3252                 if (aeq_id) {
3253                         /* Bad terminate recvd - send back a terminate */
3254                         aeq_info = (aeq_info & 0xffff0000) | aeq_id;
3255                         aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3256                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3257                         return;
3258                 }
3259         }
3260
3261         nesqp->term_flags |= NES_TERM_RCVD;
3262         nesqp->terminate_eventtype = IB_EVENT_QP_FATAL;
3263         nes_terminate_start_timer(nesqp);
3264         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3265 }
3266
3267 /* Timeout routine in case terminate fails to complete */
3268 static void nes_terminate_timeout(unsigned long context)
3269 {
3270         struct nes_qp *nesqp = (struct nes_qp *)(unsigned long)context;
3271
3272         nes_terminate_done(nesqp, 1);
3273 }
3274
3275 /* Set a timer in case hw cannot complete the terminate sequence */
3276 static void nes_terminate_start_timer(struct nes_qp *nesqp)
3277 {
3278         init_timer(&nesqp->terminate_timer);
3279         nesqp->terminate_timer.function = nes_terminate_timeout;
3280         nesqp->terminate_timer.expires = jiffies + HZ;
3281         nesqp->terminate_timer.data = (unsigned long)nesqp;
3282         add_timer(&nesqp->terminate_timer);
3283 }
3284
3285 /**
3286  * nes_process_iwarp_aeqe
3287  */
3288 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
3289                                    struct nes_hw_aeqe *aeqe)
3290 {
3291         u64 context;
3292         unsigned long flags;
3293         struct nes_qp *nesqp;
3294         struct nes_hw_cq *hw_cq;
3295         struct nes_cq *nescq;
3296         int resource_allocated;
3297         struct nes_adapter *nesadapter = nesdev->nesadapter;
3298         u32 aeq_info;
3299         u32 next_iwarp_state = 0;
3300         u16 async_event_id;
3301         u8 tcp_state;
3302         u8 iwarp_state;
3303         int must_disconn = 1;
3304         int must_terminate = 0;
3305         struct ib_event ibevent;
3306
3307         nes_debug(NES_DBG_AEQ, "\n");
3308         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3309         if ((NES_AEQE_INBOUND_RDMA & aeq_info) || (!(NES_AEQE_QP & aeq_info))) {
3310                 context  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
3311                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
3312         } else {
3313                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3314                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3315                 BUG_ON(!context);
3316         }
3317
3318         async_event_id = (u16)aeq_info;
3319         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3320         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3321         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
3322                         " Tcp state = %s, iWARP state = %s\n",
3323                         async_event_id,
3324                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
3325                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
3326
3327         switch (async_event_id) {
3328                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
3329                         nesqp = (struct nes_qp *)(unsigned long)context;
3330
3331                         if (nesqp->term_flags)
3332                                 return; /* Ignore it, wait for close complete */
3333
3334                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
3335                                 nesqp->cm_id->add_ref(nesqp->cm_id);
3336                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
3337                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
3338                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
3339                                                 " need ae to finish up, original_last_aeq = 0x%04X."
3340                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
3341                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
3342                                                 async_event_id, nesqp->last_aeq, tcp_state);
3343                         }
3344
3345                         if ((tcp_state != NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
3346                                         (nesqp->ibqp_state != IB_QPS_RTS)) {
3347                                 /* FIN Received but tcp state or IB state moved on,
3348                                                 should expect a close complete */
3349                                 return;
3350                         }
3351
3352                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
3353                         nesqp = (struct nes_qp *)(unsigned long)context;
3354                         if (nesqp->term_flags) {
3355                                 nes_terminate_done(nesqp, 0);
3356                                 return;
3357                         }
3358
3359                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
3360                 case NES_AEQE_AEID_RESET_SENT:
3361                         nesqp = (struct nes_qp *)(unsigned long)context;
3362                         if (async_event_id == NES_AEQE_AEID_RESET_SENT) {
3363                                 tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3364                         }
3365                         spin_lock_irqsave(&nesqp->lock, flags);
3366                         nesqp->hw_iwarp_state = iwarp_state;
3367                         nesqp->hw_tcp_state = tcp_state;
3368                         nesqp->last_aeq = async_event_id;
3369
3370                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSED) ||
3371                                         (tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT)) {
3372                                 nesqp->hte_added = 0;
3373                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE;
3374                         }
3375
3376                         if ((nesqp->ibqp_state == IB_QPS_RTS) &&
3377                                         ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
3378                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
3379                                 switch (nesqp->hw_iwarp_state) {
3380                                         case NES_AEQE_IWARP_STATE_RTS:
3381                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
3382                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
3383                                                 break;
3384                                         case NES_AEQE_IWARP_STATE_TERMINATE:
3385                                                 must_disconn = 0; /* terminate path takes care of disconn */
3386                                                 if (nesqp->term_flags == 0)
3387                                                         must_terminate = 1;
3388                                                 break;
3389                                 }
3390                         } else {
3391                                 if (async_event_id ==  NES_AEQE_AEID_LLP_FIN_RECEIVED) {
3392                                         /* FIN Received but ib state not RTS,
3393                                                         close complete will be on its way */
3394                                         must_disconn = 0;
3395                                 }
3396                         }
3397                         spin_unlock_irqrestore(&nesqp->lock, flags);
3398
3399                         if (must_terminate)
3400                                 nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3401                         else if (must_disconn) {
3402                                 if (next_iwarp_state) {
3403                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X\n",
3404                                                   nesqp->hwqp.qp_id, next_iwarp_state);
3405                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3406                                 }
3407                                 nes_cm_disconn(nesqp);
3408                         }
3409                         break;
3410
3411                 case NES_AEQE_AEID_TERMINATE_SENT:
3412                         nesqp = (struct nes_qp *)(unsigned long)context;
3413                         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3414                         break;
3415
3416                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3417                         nesqp = (struct nes_qp *)(unsigned long)context;
3418                         nes_terminate_received(nesdev, nesqp, aeqe);
3419                         break;
3420
3421                 case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3422                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3423                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3424                 case NES_AEQE_AEID_AMP_INVALID_STAG:
3425                 case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3426                 case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3427                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3428                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3429                 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3430                 case NES_AEQE_AEID_AMP_TO_WRAP:
3431                         nesqp = (struct nes_qp *)(unsigned long)context;
3432                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_ACCESS_ERR);
3433                         break;
3434
3435                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3436                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3437                 case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3438                 case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3439                         nesqp = (struct nes_qp *)(unsigned long)context;
3440                         if (iwarp_opcode(nesqp, aeq_info) > IWARP_OPCODE_TERM) {
3441                                 aeq_info &= 0xffff0000;
3442                                 aeq_info |= NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE;
3443                                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3444                         }
3445
3446                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
3447                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3448                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3449                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3450                 case NES_AEQE_AEID_AMP_BAD_QP:
3451                 case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3452                 case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3453                 case NES_AEQE_AEID_DDP_NO_L_BIT:
3454                 case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3455                 case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3456                 case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3457                 case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3458                 case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3459                 case NES_AEQE_AEID_AMP_BAD_PD:
3460                 case NES_AEQE_AEID_AMP_FASTREG_SHARED:
3461                 case NES_AEQE_AEID_AMP_FASTREG_VALID_STAG:
3462                 case NES_AEQE_AEID_AMP_FASTREG_MW_STAG:
3463                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_RIGHTS:
3464                 case NES_AEQE_AEID_AMP_FASTREG_PBL_TABLE_OVERFLOW:
3465                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_LENGTH:
3466                 case NES_AEQE_AEID_AMP_INVALIDATE_SHARED:
3467                 case NES_AEQE_AEID_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS:
3468                 case NES_AEQE_AEID_AMP_MWBIND_VALID_STAG:
3469                 case NES_AEQE_AEID_AMP_MWBIND_OF_MR_STAG:
3470                 case NES_AEQE_AEID_AMP_MWBIND_TO_ZERO_BASED_STAG:
3471                 case NES_AEQE_AEID_AMP_MWBIND_TO_MW_STAG:
3472                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_RIGHTS:
3473                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_BOUNDS:
3474                 case NES_AEQE_AEID_AMP_MWBIND_TO_INVALID_PARENT:
3475                 case NES_AEQE_AEID_AMP_MWBIND_BIND_DISABLED:
3476                 case NES_AEQE_AEID_BAD_CLOSE:
3477                 case NES_AEQE_AEID_RDMA_READ_WHILE_ORD_ZERO:
3478                 case NES_AEQE_AEID_STAG_ZERO_INVALID:
3479                 case NES_AEQE_AEID_ROE_INVALID_RDMA_READ_REQUEST:
3480                 case NES_AEQE_AEID_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
3481                         nesqp = (struct nes_qp *)(unsigned long)context;
3482                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3483                         break;
3484
3485                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3486                         context <<= 1;
3487                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3488                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3489                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3490                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3491                         if (resource_allocated) {
3492                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3493                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3494                                 hw_cq = (struct nes_hw_cq *)(unsigned long)context;
3495                                 if (hw_cq) {
3496                                         nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3497                                         if (nescq->ibcq.event_handler) {
3498                                                 ibevent.device = nescq->ibcq.device;
3499                                                 ibevent.event = IB_EVENT_CQ_ERR;
3500                                                 ibevent.element.cq = &nescq->ibcq;
3501                                                 nescq->ibcq.event_handler(&ibevent, nescq->ibcq.cq_context);
3502                                         }
3503                                 }
3504                         }
3505                         break;
3506
3507                 default:
3508                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3509                                         async_event_id);
3510                         break;
3511         }
3512
3513 }
3514
3515 /**
3516  * nes_iwarp_ce_handler
3517  */
3518 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3519 {
3520         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3521
3522         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3523                         nescq->hw_cq.cq_number); */
3524         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3525
3526         if (nescq->ibcq.comp_handler)
3527                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3528
3529         return;
3530 }
3531
3532
3533 /**
3534  * nes_manage_apbvt()
3535  */
3536 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3537                 u32 nic_index, u32 add_port)
3538 {
3539         struct nes_device *nesdev = nesvnic->nesdev;
3540         struct nes_hw_cqp_wqe *cqp_wqe;
3541         struct nes_cqp_request *cqp_request;
3542         int ret = 0;
3543         u16 major_code;
3544
3545         /* Send manage APBVT request to CQP */
3546         cqp_request = nes_get_cqp_request(nesdev);
3547         if (cqp_request == NULL) {
3548                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3549                 return -ENOMEM;
3550         }
3551         cqp_request->waiting = 1;
3552         cqp_wqe = &cqp_request->cqp_wqe;
3553
3554         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3555                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3556                         accel_local_port, accel_local_port, nic_index);
3557
3558         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3559         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3560                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3561         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3562                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3563
3564         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3565
3566         atomic_set(&cqp_request->refcount, 2);
3567         nes_post_cqp_request(nesdev, cqp_request);
3568
3569         if (add_port == NES_MANAGE_APBVT_ADD)
3570                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3571                                 NES_EVENT_TIMEOUT);
3572         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3573                         ret, cqp_request->major_code, cqp_request->minor_code);
3574         major_code = cqp_request->major_code;
3575
3576         nes_put_cqp_request(nesdev, cqp_request);
3577
3578         if (!ret)
3579                 return -ETIME;
3580         else if (major_code)
3581                 return -EIO;
3582         else
3583                 return 0;
3584 }
3585
3586
3587 /**
3588  * nes_manage_arp_cache
3589  */
3590 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3591                 u32 ip_addr, u32 action)
3592 {
3593         struct nes_hw_cqp_wqe *cqp_wqe;
3594         struct nes_vnic *nesvnic = netdev_priv(netdev);
3595         struct nes_device *nesdev;
3596         struct nes_cqp_request *cqp_request;
3597         int arp_index;
3598
3599         nesdev = nesvnic->nesdev;
3600         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3601         if (arp_index == -1) {
3602                 return;
3603         }
3604
3605         /* update the ARP entry */
3606         cqp_request = nes_get_cqp_request(nesdev);
3607         if (cqp_request == NULL) {
3608                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3609                 return;
3610         }
3611         cqp_request->waiting = 0;
3612         cqp_wqe = &cqp_request->cqp_wqe;
3613         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3614
3615         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3616                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3617         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3618                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3619         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3620
3621         if (action == NES_ARP_ADD) {
3622                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3623                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3624                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3625                                 (((u32)mac_addr[4]) << 8)  | (u32)mac_addr[5]);
3626                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3627                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3628         } else {
3629                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3630                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3631         }
3632
3633         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3634                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3635
3636         atomic_set(&cqp_request->refcount, 1);
3637         nes_post_cqp_request(nesdev, cqp_request);
3638 }
3639
3640
3641 /**
3642  * flush_wqes
3643  */
3644 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3645                 u32 which_wq, u32 wait_completion)
3646 {
3647         struct nes_cqp_request *cqp_request;
3648         struct nes_hw_cqp_wqe *cqp_wqe;
3649         int ret;
3650
3651         cqp_request = nes_get_cqp_request(nesdev);
3652         if (cqp_request == NULL) {
3653                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3654                 return;
3655         }
3656         if (wait_completion) {
3657                 cqp_request->waiting = 1;
3658                 atomic_set(&cqp_request->refcount, 2);
3659         } else {
3660                 cqp_request->waiting = 0;
3661         }
3662         cqp_wqe = &cqp_request->cqp_wqe;
3663         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3664
3665         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3666                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3667         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3668
3669         nes_post_cqp_request(nesdev, cqp_request);
3670
3671         if (wait_completion) {
3672                 /* Wait for CQP */
3673                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3674                                 NES_EVENT_TIMEOUT);
3675                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3676                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3677                                 ret, cqp_request->major_code, cqp_request->minor_code);
3678                 nes_put_cqp_request(nesdev, cqp_request);
3679         }
3680 }