drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c

   1 /*
   2  * This file is part of the Chelsio T4 Ethernet driver for Linux.
   3  *
   4  * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved.
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenIB.org BSD license below:
  11  *
  12  *     Redistribution and use in source and binary forms, with or
  13  *     without modification, are permitted provided that the following
  14  *     conditions are met:
  15  *
  16  *      - Redistributions of source code must retain the above
  17  *        copyright notice, this list of conditions and the following
  18  *        disclaimer.
  19  *
  20  *      - Redistributions in binary form must reproduce the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer in the documentation and/or other materials
  23  *        provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  */
  34
  35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  36
  37 #include <linux/bitmap.h>
  38 #include <linux/crc32.h>
  39 #include <linux/ctype.h>
  40 #include <linux/debugfs.h>
  41 #include <linux/err.h>
  42 #include <linux/etherdevice.h>
  43 #include <linux/firmware.h>
  44 #include <linux/if.h>
  45 #include <linux/if_vlan.h>
  46 #include <linux/init.h>
  47 #include <linux/log2.h>
  48 #include <linux/mdio.h>
  49 #include <linux/module.h>
  50 #include <linux/moduleparam.h>
  51 #include <linux/mutex.h>
  52 #include <linux/netdevice.h>
  53 #include <linux/pci.h>
  54 #include <linux/aer.h>
  55 #include <linux/rtnetlink.h>
  56 #include <linux/sched.h>
  57 #include <linux/seq_file.h>
  58 #include <linux/sockios.h>
  59 #include <linux/vmalloc.h>
  60 #include <linux/workqueue.h>
  61 #include <net/neighbour.h>
  62 #include <net/netevent.h>
  63 #include <net/addrconf.h>
  64 #include <asm/uaccess.h>
  65
  66 #include "cxgb4.h"
  67 #include "t4_regs.h"
  68 #include "t4_msg.h"
  69 #include "t4fw_api.h"
  70 #include "l2t.h"
  71
  72 #include <../drivers/net/bonding/bonding.h>
  73
  74 #ifdef DRV_VERSION
  75 #undef DRV_VERSION
  76 #endif
  77 #define DRV_VERSION "2.0.0-ko"
  78 #define DRV_DESC "Chelsio T4/T5 Network Driver"
  79
  80 /*
  81  * Max interrupt hold-off timer value in us.  Queues fall back to this value
  82  * under extreme memory pressure so it's largish to give the system time to
  83  * recover.
  84  */
  85 #define MAX_SGE_TIMERVAL 200U
  86
  87 enum {
  88         /*
  89          * Physical Function provisioning constants.
  90          */
  91         PFRES_NVI = 4,                  /* # of Virtual Interfaces */
  92         PFRES_NETHCTRL = 128,           /* # of EQs used for ETH or CTRL Qs */
  93         PFRES_NIQFLINT = 128,           /* # of ingress Qs/w Free List(s)/intr
  94                                          */
  95         PFRES_NEQ = 256,                /* # of egress queues */
  96         PFRES_NIQ = 0,                  /* # of ingress queues */
  97         PFRES_TC = 0,                   /* PCI-E traffic class */
  98         PFRES_NEXACTF = 128,            /* # of exact MPS filters */
  99
 100         PFRES_R_CAPS = FW_CMD_CAP_PF,
 101         PFRES_WX_CAPS = FW_CMD_CAP_PF,
 102
 103 #ifdef CONFIG_PCI_IOV
 104         /*
 105          * Virtual Function provisioning constants.  We need two extra Ingress
 106          * Queues with Interrupt capability to serve as the VF's Firmware
 107          * Event Queue and Forwarded Interrupt Queue (when using MSI mode) --
 108          * neither will have Free Lists associated with them).  For each
 109          * Ethernet/Control Egress Queue and for each Free List, we need an
 110          * Egress Context.
 111          */
 112         VFRES_NPORTS = 1,               /* # of "ports" per VF */
 113         VFRES_NQSETS = 2,               /* # of "Queue Sets" per VF */
 114
 115         VFRES_NVI = VFRES_NPORTS,       /* # of Virtual Interfaces */
 116         VFRES_NETHCTRL = VFRES_NQSETS,  /* # of EQs used for ETH or CTRL Qs */
 117         VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */
 118         VFRES_NEQ = VFRES_NQSETS*2,     /* # of egress queues */
 119         VFRES_NIQ = 0,                  /* # of non-fl/int ingress queues */
 120         VFRES_TC = 0,                   /* PCI-E traffic class */
 121         VFRES_NEXACTF = 16,             /* # of exact MPS filters */
 122
 123         VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT,
 124         VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF,
 125 #endif
 126 };
 127
 128 /*
 129  * Provide a Port Access Rights Mask for the specified PF/VF.  This is very
 130  * static and likely not to be useful in the long run.  We really need to
 131  * implement some form of persistent configuration which the firmware
 132  * controls.
 133  */
 134 static unsigned int pfvfres_pmask(struct adapter *adapter,
 135                                   unsigned int pf, unsigned int vf)
 136 {
 137         unsigned int portn, portvec;
 138
 139         /*
 140          * Give PF's access to all of the ports.
 141          */
 142         if (vf == 0)
 143                 return FW_PFVF_CMD_PMASK_MASK;
 144
 145         /*
 146          * For VFs, we'll assign them access to the ports based purely on the
 147          * PF.  We assign active ports in order, wrapping around if there are
 148          * fewer active ports than PFs: e.g. active port[pf % nports].
 149          * Unfortunately the adapter's port_info structs haven't been
 150          * initialized yet so we have to compute this.
 151          */
 152         if (adapter->params.nports == 0)
 153                 return 0;
 154
 155         portn = pf % adapter->params.nports;
 156         portvec = adapter->params.portvec;
 157         for (;;) {
 158                 /*
 159                  * Isolate the lowest set bit in the port vector.  If we're at
 160                  * the port number that we want, return that as the pmask.
 161                  * otherwise mask that bit out of the port vector and
 162                  * decrement our port number ...
 163                  */
 164                 unsigned int pmask = portvec ^ (portvec & (portvec-1));
 165                 if (portn == 0)
 166                         return pmask;
 167                 portn--;
 168                 portvec &= ~pmask;
 169         }
 170         /*NOTREACHED*/
 171 }
 172
 173 enum {
 174         MAX_TXQ_ENTRIES      = 16384,
 175         MAX_CTRL_TXQ_ENTRIES = 1024,
 176         MAX_RSPQ_ENTRIES     = 16384,
 177         MAX_RX_BUFFERS       = 16384,
 178         MIN_TXQ_ENTRIES      = 32,
 179         MIN_CTRL_TXQ_ENTRIES = 32,
 180         MIN_RSPQ_ENTRIES     = 128,
 181         MIN_FL_ENTRIES       = 16
 182 };
 183
 184 /* Host shadow copy of ingress filter entry.  This is in host native format
 185  * and doesn't match the ordering or bit order, etc. of the hardware of the
 186  * firmware command.  The use of bit-field structure elements is purely to
 187  * remind ourselves of the field size limitations and save memory in the case
 188  * where the filter table is large.
 189  */
 190 struct filter_entry {
 191         /* Administrative fields for filter.
 192          */
 193         u32 valid:1;            /* filter allocated and valid */
 194         u32 locked:1;           /* filter is administratively locked */
 195
 196         u32 pending:1;          /* filter action is pending firmware reply */
 197         u32 smtidx:8;           /* Source MAC Table index for smac */
 198         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 199
 200         /* The filter itself.  Most of this is a straight copy of information
 201          * provided by the extended ioctl().  Some fields are translated to
 202          * internal forms -- for instance the Ingress Queue ID passed in from
 203          * the ioctl() is translated into the Absolute Ingress Queue ID.
 204          */
 205         struct ch_filter_specification fs;
 206 };
 207
 208 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
 209                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
 210                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
 211
 212 #define CH_DEVICE(devid, data) { PCI_VDEVICE(CHELSIO, devid), (data) }
 213
 214 static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl) = {
 215         CH_DEVICE(0xa000, 0),  /* PE10K */
 216         CH_DEVICE(0x4001, -1),
 217         CH_DEVICE(0x4002, -1),
 218         CH_DEVICE(0x4003, -1),
 219         CH_DEVICE(0x4004, -1),
 220         CH_DEVICE(0x4005, -1),
 221         CH_DEVICE(0x4006, -1),
 222         CH_DEVICE(0x4007, -1),
 223         CH_DEVICE(0x4008, -1),
 224         CH_DEVICE(0x4009, -1),
 225         CH_DEVICE(0x400a, -1),
 226         CH_DEVICE(0x4401, 4),
 227         CH_DEVICE(0x4402, 4),
 228         CH_DEVICE(0x4403, 4),
 229         CH_DEVICE(0x4404, 4),
 230         CH_DEVICE(0x4405, 4),
 231         CH_DEVICE(0x4406, 4),
 232         CH_DEVICE(0x4407, 4),
 233         CH_DEVICE(0x4408, 4),
 234         CH_DEVICE(0x4409, 4),
 235         CH_DEVICE(0x440a, 4),
 236         CH_DEVICE(0x440d, 4),
 237         CH_DEVICE(0x440e, 4),
 238         CH_DEVICE(0x5001, 4),
 239         CH_DEVICE(0x5002, 4),
 240         CH_DEVICE(0x5003, 4),
 241         CH_DEVICE(0x5004, 4),
 242         CH_DEVICE(0x5005, 4),
 243         CH_DEVICE(0x5006, 4),
 244         CH_DEVICE(0x5007, 4),
 245         CH_DEVICE(0x5008, 4),
 246         CH_DEVICE(0x5009, 4),
 247         CH_DEVICE(0x500A, 4),
 248         CH_DEVICE(0x500B, 4),
 249         CH_DEVICE(0x500C, 4),
 250         CH_DEVICE(0x500D, 4),
 251         CH_DEVICE(0x500E, 4),
 252         CH_DEVICE(0x500F, 4),
 253         CH_DEVICE(0x5010, 4),
 254         CH_DEVICE(0x5011, 4),
 255         CH_DEVICE(0x5012, 4),
 256         CH_DEVICE(0x5013, 4),
 257         CH_DEVICE(0x5014, 4),
 258         CH_DEVICE(0x5015, 4),
 259         CH_DEVICE(0x5080, 4),
 260         CH_DEVICE(0x5081, 4),
 261         CH_DEVICE(0x5082, 4),
 262         CH_DEVICE(0x5083, 4),
 263         CH_DEVICE(0x5084, 4),
 264         CH_DEVICE(0x5085, 4),
 265         CH_DEVICE(0x5401, 4),
 266         CH_DEVICE(0x5402, 4),
 267         CH_DEVICE(0x5403, 4),
 268         CH_DEVICE(0x5404, 4),
 269         CH_DEVICE(0x5405, 4),
 270         CH_DEVICE(0x5406, 4),
 271         CH_DEVICE(0x5407, 4),
 272         CH_DEVICE(0x5408, 4),
 273         CH_DEVICE(0x5409, 4),
 274         CH_DEVICE(0x540A, 4),
 275         CH_DEVICE(0x540B, 4),
 276         CH_DEVICE(0x540C, 4),
 277         CH_DEVICE(0x540D, 4),
 278         CH_DEVICE(0x540E, 4),
 279         CH_DEVICE(0x540F, 4),
 280         CH_DEVICE(0x5410, 4),
 281         CH_DEVICE(0x5411, 4),
 282         CH_DEVICE(0x5412, 4),
 283         CH_DEVICE(0x5413, 4),
 284         CH_DEVICE(0x5414, 4),
 285         CH_DEVICE(0x5415, 4),
 286         CH_DEVICE(0x5480, 4),
 287         CH_DEVICE(0x5481, 4),
 288         CH_DEVICE(0x5482, 4),
 289         CH_DEVICE(0x5483, 4),
 290         CH_DEVICE(0x5484, 4),
 291         CH_DEVICE(0x5485, 4),
 292         { 0, }
 293 };
 294
 295 #define FW4_FNAME "cxgb4/t4fw.bin"
 296 #define FW5_FNAME "cxgb4/t5fw.bin"
 297 #define FW4_CFNAME "cxgb4/t4-config.txt"
 298 #define FW5_CFNAME "cxgb4/t5-config.txt"
 299
 300 MODULE_DESCRIPTION(DRV_DESC);
 301 MODULE_AUTHOR("Chelsio Communications");
 302 MODULE_LICENSE("Dual BSD/GPL");
 303 MODULE_VERSION(DRV_VERSION);
 304 MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
 305 MODULE_FIRMWARE(FW4_FNAME);
 306 MODULE_FIRMWARE(FW5_FNAME);
 307
 308 /*
 309  * Normally we're willing to become the firmware's Master PF but will be happy
 310  * if another PF has already become the Master and initialized the adapter.
 311  * Setting "force_init" will cause this driver to forcibly establish itself as
 312  * the Master PF and initialize the adapter.
 313  */
 314 static uint force_init;
 315
 316 module_param(force_init, uint, 0644);
 317 MODULE_PARM_DESC(force_init, "Forcibly become Master PF and initialize adapter");
 318
 319 /*
 320  * Normally if the firmware we connect to has Configuration File support, we
 321  * use that and only fall back to the old Driver-based initialization if the
 322  * Configuration File fails for some reason.  If force_old_init is set, then
 323  * we'll always use the old Driver-based initialization sequence.
 324  */
 325 static uint force_old_init;
 326
 327 module_param(force_old_init, uint, 0644);
 328 MODULE_PARM_DESC(force_old_init, "Force old initialization sequence");
 329
 330 static int dflt_msg_enable = DFLT_MSG_ENABLE;
 331
 332 module_param(dflt_msg_enable, int, 0644);
 333 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
 334
 335 /*
 336  * The driver uses the best interrupt scheme available on a platform in the
 337  * order MSI-X, MSI, legacy INTx interrupts.  This parameter determines which
 338  * of these schemes the driver may consider as follows:
 339  *
 340  * msi = 2: choose from among all three options
 341  * msi = 1: only consider MSI and INTx interrupts
 342  * msi = 0: force INTx interrupts
 343  */
 344 static int msi = 2;
 345
 346 module_param(msi, int, 0644);
 347 MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
 348
 349 /*
 350  * Queue interrupt hold-off timer values.  Queues default to the first of these
 351  * upon creation.
 352  */
 353 static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
 354
 355 module_param_array(intr_holdoff, uint, NULL, 0644);
 356 MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
 357                  "0..4 in microseconds");
 358
 359 static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
 360
 361 module_param_array(intr_cnt, uint, NULL, 0644);
 362 MODULE_PARM_DESC(intr_cnt,
 363                  "thresholds 1..3 for queue interrupt packet counters");
 364
 365 /*
 366  * Normally we tell the chip to deliver Ingress Packets into our DMA buffers
 367  * offset by 2 bytes in order to have the IP headers line up on 4-byte
 368  * boundaries.  This is a requirement for many architectures which will throw
 369  * a machine check fault if an attempt is made to access one of the 4-byte IP
 370  * header fields on a non-4-byte boundary.  And it's a major performance issue
 371  * even on some architectures which allow it like some implementations of the
 372  * x86 ISA.  However, some architectures don't mind this and for some very
 373  * edge-case performance sensitive applications (like forwarding large volumes
 374  * of small packets), setting this DMA offset to 0 will decrease the number of
 375  * PCI-E Bus transfers enough to measurably affect performance.
 376  */
 377 static int rx_dma_offset = 2;
 378
 379 static bool vf_acls;
 380
 381 #ifdef CONFIG_PCI_IOV
 382 module_param(vf_acls, bool, 0644);
 383 MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");
 384
 385 /* Configure the number of PCI-E Virtual Function which are to be instantiated
 386  * on SR-IOV Capable Physical Functions.
 387  */
 388 static unsigned int num_vf[NUM_OF_PF_WITH_SRIOV];
 389
 390 module_param_array(num_vf, uint, NULL, 0644);
 391 MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
 392 #endif
 393
 394 /*
 395  * The filter TCAM has a fixed portion and a variable portion.  The fixed
 396  * portion can match on source/destination IP IPv4/IPv6 addresses and TCP/UDP
 397  * ports.  The variable portion is 36 bits which can include things like Exact
 398  * Match MAC Index (9 bits), Ether Type (16 bits), IP Protocol (8 bits),
 399  * [Inner] VLAN Tag (17 bits), etc. which, if all were somehow selected, would
 400  * far exceed the 36-bit budget for this "compressed" header portion of the
 401  * filter.  Thus, we have a scarce resource which must be carefully managed.
 402  *
 403  * By default we set this up to mostly match the set of filter matching
 404  * capabilities of T3 but with accommodations for some of T4's more
 405  * interesting features:
 406  *
 407  *   { IP Fragment (1), MPS Match Type (3), IP Protocol (8),
 408  *     [Inner] VLAN (17), Port (3), FCoE (1) }
 409  */
 410 enum {
 411         TP_VLAN_PRI_MAP_DEFAULT = HW_TPL_FR_MT_PR_IV_P_FC,
 412         TP_VLAN_PRI_MAP_FIRST = FCOE_SHIFT,
 413         TP_VLAN_PRI_MAP_LAST = FRAGMENTATION_SHIFT,
 414 };
 415
 416 static unsigned int tp_vlan_pri_map = TP_VLAN_PRI_MAP_DEFAULT;
 417
 418 module_param(tp_vlan_pri_map, uint, 0644);
 419 MODULE_PARM_DESC(tp_vlan_pri_map, "global compressed filter configuration");
 420
 421 static struct dentry *cxgb4_debugfs_root;
 422
 423 static LIST_HEAD(adapter_list);
 424 static DEFINE_MUTEX(uld_mutex);
 425 /* Adapter list to be accessed from atomic context */
 426 static LIST_HEAD(adap_rcu_list);
 427 static DEFINE_SPINLOCK(adap_rcu_lock);
 428 static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
 429 static const char *uld_str[] = { "RDMA", "iSCSI" };
 430
 431 static void link_report(struct net_device *dev)
 432 {
 433         if (!netif_carrier_ok(dev))
 434                 netdev_info(dev, "link down\n");
 435         else {
 436                 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
 437
 438                 const char *s = "10Mbps";
 439                 const struct port_info *p = netdev_priv(dev);
 440
 441                 switch (p->link_cfg.speed) {
 442                 case 10000:
 443                         s = "10Gbps";
 444                         break;
 445                 case 1000:
 446                         s = "1000Mbps";
 447                         break;
 448                 case 100:
 449                         s = "100Mbps";
 450                         break;
 451                 case 40000:
 452                         s = "40Gbps";
 453                         break;
 454                 }
 455
 456                 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
 457                             fc[p->link_cfg.fc]);
 458         }
 459 }
 460
 461 void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
 462 {
 463         struct net_device *dev = adapter->port[port_id];
 464
 465         /* Skip changes from disabled ports. */
 466         if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
 467                 if (link_stat)
 468                         netif_carrier_on(dev);
 469                 else
 470                         netif_carrier_off(dev);
 471
 472                 link_report(dev);
 473         }
 474 }
 475
 476 void t4_os_portmod_changed(const struct adapter *adap, int port_id)
 477 {
 478         static const char *mod_str[] = {
 479                 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
 480         };
 481
 482         const struct net_device *dev = adap->port[port_id];
 483         const struct port_info *pi = netdev_priv(dev);
 484
 485         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
 486                 netdev_info(dev, "port module unplugged\n");
 487         else if (pi->mod_type < ARRAY_SIZE(mod_str))
 488                 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
 489 }
 490
 491 /*
 492  * Configure the exact and hash address filters to handle a port's multicast
 493  * and secondary unicast MAC addresses.
 494  */
 495 static int set_addr_filters(const struct net_device *dev, bool sleep)
 496 {
 497         u64 mhash = 0;
 498         u64 uhash = 0;
 499         bool free = true;
 500         u16 filt_idx[7];
 501         const u8 *addr[7];
 502         int ret, naddr = 0;
 503         const struct netdev_hw_addr *ha;
 504         int uc_cnt = netdev_uc_count(dev);
 505         int mc_cnt = netdev_mc_count(dev);
 506         const struct port_info *pi = netdev_priv(dev);
 507         unsigned int mb = pi->adapter->fn;
 508
 509         /* first do the secondary unicast addresses */
 510         netdev_for_each_uc_addr(ha, dev) {
 511                 addr[naddr++] = ha->addr;
 512                 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
 513                         ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
 514                                         naddr, addr, filt_idx, &uhash, sleep);
 515                         if (ret < 0)
 516                                 return ret;
 517
 518                         free = false;
 519                         naddr = 0;
 520                 }
 521         }
 522
 523         /* next set up the multicast addresses */
 524         netdev_for_each_mc_addr(ha, dev) {
 525                 addr[naddr++] = ha->addr;
 526                 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
 527                         ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
 528                                         naddr, addr, filt_idx, &mhash, sleep);
 529                         if (ret < 0)
 530                                 return ret;
 531
 532                         free = false;
 533                         naddr = 0;
 534                 }
 535         }
 536
 537         return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
 538                                 uhash | mhash, sleep);
 539 }
 540
 541 int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */
 542 module_param(dbfifo_int_thresh, int, 0644);
 543 MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold");
 544
 545 /*
 546  * usecs to sleep while draining the dbfifo
 547  */
 548 static int dbfifo_drain_delay = 1000;
 549 module_param(dbfifo_drain_delay, int, 0644);
 550 MODULE_PARM_DESC(dbfifo_drain_delay,
 551                  "usecs to sleep while draining the dbfifo");
 552
 553 /*
 554  * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
 555  * If @mtu is -1 it is left unchanged.
 556  */
 557 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
 558 {
 559         int ret;
 560         struct port_info *pi = netdev_priv(dev);
 561
 562         ret = set_addr_filters(dev, sleep_ok);
 563         if (ret == 0)
 564                 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
 565                                     (dev->flags & IFF_PROMISC) ? 1 : 0,
 566                                     (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
 567                                     sleep_ok);
 568         return ret;
 569 }
 570
 571 static struct workqueue_struct *workq;
 572
 573 /**
 574  *      link_start - enable a port
 575  *      @dev: the port to enable
 576  *
 577  *      Performs the MAC and PHY actions needed to enable a port.
 578  */
 579 static int link_start(struct net_device *dev)
 580 {
 581         int ret;
 582         struct port_info *pi = netdev_priv(dev);
 583         unsigned int mb = pi->adapter->fn;
 584
 585         /*
 586          * We do not set address filters and promiscuity here, the stack does
 587          * that step explicitly.
 588          */
 589         ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
 590                             !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true);
 591         if (ret == 0) {
 592                 ret = t4_change_mac(pi->adapter, mb, pi->viid,
 593                                     pi->xact_addr_filt, dev->dev_addr, true,
 594                                     true);
 595                 if (ret >= 0) {
 596                         pi->xact_addr_filt = ret;
 597                         ret = 0;
 598                 }
 599         }
 600         if (ret == 0)
 601                 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
 602                                     &pi->link_cfg);
 603         if (ret == 0)
 604                 ret = t4_enable_vi(pi->adapter, mb, pi->viid, true, true);
 605         return ret;
 606 }
 607
 608 /* Clear a filter and release any of its resources that we own.  This also
 609  * clears the filter's "pending" status.
 610  */
 611 static void clear_filter(struct adapter *adap, struct filter_entry *f)
 612 {
 613         /* If the new or old filter have loopback rewriteing rules then we'll
 614          * need to free any existing Layer Two Table (L2T) entries of the old
 615          * filter rule.  The firmware will handle freeing up any Source MAC
 616          * Table (SMT) entries used for rewriting Source MAC Addresses in
 617          * loopback rules.
 618          */
 619         if (f->l2t)
 620                 cxgb4_l2t_release(f->l2t);
 621
 622         /* The zeroing of the filter rule below clears the filter valid,
 623          * pending, locked flags, l2t pointer, etc. so it's all we need for
 624          * this operation.
 625          */
 626         memset(f, 0, sizeof(*f));
 627 }
 628
 629 /* Handle a filter write/deletion reply.
 630  */
 631 static void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl)
 632 {
 633         unsigned int idx = GET_TID(rpl);
 634         unsigned int nidx = idx - adap->tids.ftid_base;
 635         unsigned int ret;
 636         struct filter_entry *f;
 637
 638         if (idx >= adap->tids.ftid_base && nidx <
 639            (adap->tids.nftids + adap->tids.nsftids)) {
 640                 idx = nidx;
 641                 ret = GET_TCB_COOKIE(rpl->cookie);
 642                 f = &adap->tids.ftid_tab[idx];
 643
 644                 if (ret == FW_FILTER_WR_FLT_DELETED) {
 645                         /* Clear the filter when we get confirmation from the
 646                          * hardware that the filter has been deleted.
 647                          */
 648                         clear_filter(adap, f);
 649                 } else if (ret == FW_FILTER_WR_SMT_TBL_FULL) {
 650                         dev_err(adap->pdev_dev, "filter %u setup failed due to full SMT\n",
 651                                 idx);
 652                         clear_filter(adap, f);
 653                 } else if (ret == FW_FILTER_WR_FLT_ADDED) {
 654                         f->smtidx = (be64_to_cpu(rpl->oldval) >> 24) & 0xff;
 655                         f->pending = 0;  /* asynchronous setup completed */
 656                         f->valid = 1;
 657                 } else {
 658                         /* Something went wrong.  Issue a warning about the
 659                          * problem and clear everything out.
 660                          */
 661                         dev_err(adap->pdev_dev, "filter %u setup failed with error %u\n",
 662                                 idx, ret);
 663                         clear_filter(adap, f);
 664                 }
 665         }
 666 }
 667
 668 /* Response queue handler for the FW event queue.
 669  */
 670 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
 671                           const struct pkt_gl *gl)
 672 {
 673         u8 opcode = ((const struct rss_header *)rsp)->opcode;
 674
 675         rsp++;                                          /* skip RSS header */
 676
 677         /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
 678          */
 679         if (unlikely(opcode == CPL_FW4_MSG &&
 680            ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) {
 681                 rsp++;
 682                 opcode = ((const struct rss_header *)rsp)->opcode;
 683                 rsp++;
 684                 if (opcode != CPL_SGE_EGR_UPDATE) {
 685                         dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n"
 686                                 , opcode);
 687                         goto out;
 688                 }
 689         }
 690
 691         if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
 692                 const struct cpl_sge_egr_update *p = (void *)rsp;
 693                 unsigned int qid = EGR_QID(ntohl(p->opcode_qid));
 694                 struct sge_txq *txq;
 695
 696                 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
 697                 txq->restarts++;
 698                 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
 699                         struct sge_eth_txq *eq;
 700
 701                         eq = container_of(txq, struct sge_eth_txq, q);
 702                         netif_tx_wake_queue(eq->txq);
 703                 } else {
 704                         struct sge_ofld_txq *oq;
 705
 706                         oq = container_of(txq, struct sge_ofld_txq, q);
 707                         tasklet_schedule(&oq->qresume_tsk);
 708                 }
 709         } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
 710                 const struct cpl_fw6_msg *p = (void *)rsp;
 711
 712                 if (p->type == 0)
 713                         t4_handle_fw_rpl(q->adap, p->data);
 714         } else if (opcode == CPL_L2T_WRITE_RPL) {
 715                 const struct cpl_l2t_write_rpl *p = (void *)rsp;
 716
 717                 do_l2t_write_rpl(q->adap, p);
 718         } else if (opcode == CPL_SET_TCB_RPL) {
 719                 const struct cpl_set_tcb_rpl *p = (void *)rsp;
 720
 721                 filter_rpl(q->adap, p);
 722         } else
 723                 dev_err(q->adap->pdev_dev,
 724                         "unexpected CPL %#x on FW event queue\n", opcode);
 725 out:
 726         return 0;
 727 }
 728
 729 /**
 730  *      uldrx_handler - response queue handler for ULD queues
 731  *      @q: the response queue that received the packet
 732  *      @rsp: the response queue descriptor holding the offload message
 733  *      @gl: the gather list of packet fragments
 734  *
 735  *      Deliver an ingress offload packet to a ULD.  All processing is done by
 736  *      the ULD, we just maintain statistics.
 737  */
 738 static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
 739                          const struct pkt_gl *gl)
 740 {
 741         struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
 742
 743         /* FW can send CPLs encapsulated in a CPL_FW4_MSG.
 744          */
 745         if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG &&
 746             ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL)
 747                 rsp += 2;
 748
 749         if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
 750                 rxq->stats.nomem++;
 751                 return -1;
 752         }
 753         if (gl == NULL)
 754                 rxq->stats.imm++;
 755         else if (gl == CXGB4_MSG_AN)
 756                 rxq->stats.an++;
 757         else
 758                 rxq->stats.pkts++;
 759         return 0;
 760 }
 761
 762 static void disable_msi(struct adapter *adapter)
 763 {
 764         if (adapter->flags & USING_MSIX) {
 765                 pci_disable_msix(adapter->pdev);
 766                 adapter->flags &= ~USING_MSIX;
 767         } else if (adapter->flags & USING_MSI) {
 768                 pci_disable_msi(adapter->pdev);
 769                 adapter->flags &= ~USING_MSI;
 770         }
 771 }
 772
 773 /*
 774  * Interrupt handler for non-data events used with MSI-X.
 775  */
 776 static irqreturn_t t4_nondata_intr(int irq, void *cookie)
 777 {
 778         struct adapter *adap = cookie;
 779
 780         u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE));
 781         if (v & PFSW) {
 782                 adap->swintr = 1;
 783                 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v);
 784         }
 785         t4_slow_intr_handler(adap);
 786         return IRQ_HANDLED;
 787 }
 788
 789 /*
 790  * Name the MSI-X interrupts.
 791  */
 792 static void name_msix_vecs(struct adapter *adap)
 793 {
 794         int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);
 795
 796         /* non-data interrupts */
 797         snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);
 798
 799         /* FW events */
 800         snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",
 801                  adap->port[0]->name);
 802
 803         /* Ethernet queues */
 804         for_each_port(adap, j) {
 805                 struct net_device *d = adap->port[j];
 806                 const struct port_info *pi = netdev_priv(d);
 807
 808                 for (i = 0; i < pi->nqsets; i++, msi_idx++)
 809                         snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
 810                                  d->name, i);
 811         }
 812
 813         /* offload queues */
 814         for_each_ofldrxq(&adap->sge, i)
 815                 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d",
 816                          adap->port[0]->name, i);
 817
 818         for_each_rdmarxq(&adap->sge, i)
 819                 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d",
 820                          adap->port[0]->name, i);
 821 }
 822
 823 static int request_msix_queue_irqs(struct adapter *adap)
 824 {
 825         struct sge *s = &adap->sge;
 826         int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, msi_index = 2;
 827
 828         err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
 829                           adap->msix_info[1].desc, &s->fw_evtq);
 830         if (err)
 831                 return err;
 832
 833         for_each_ethrxq(s, ethqidx) {
 834                 err = request_irq(adap->msix_info[msi_index].vec,
 835                                   t4_sge_intr_msix, 0,
 836                                   adap->msix_info[msi_index].desc,
 837                                   &s->ethrxq[ethqidx].rspq);
 838                 if (err)
 839                         goto unwind;
 840                 msi_index++;
 841         }
 842         for_each_ofldrxq(s, ofldqidx) {
 843                 err = request_irq(adap->msix_info[msi_index].vec,
 844                                   t4_sge_intr_msix, 0,
 845                                   adap->msix_info[msi_index].desc,
 846                                   &s->ofldrxq[ofldqidx].rspq);
 847                 if (err)
 848                         goto unwind;
 849                 msi_index++;
 850         }
 851         for_each_rdmarxq(s, rdmaqidx) {
 852                 err = request_irq(adap->msix_info[msi_index].vec,
 853                                   t4_sge_intr_msix, 0,
 854                                   adap->msix_info[msi_index].desc,
 855                                   &s->rdmarxq[rdmaqidx].rspq);
 856                 if (err)
 857                         goto unwind;
 858                 msi_index++;
 859         }
 860         return 0;
 861
 862 unwind:
 863         while (--rdmaqidx >= 0)
 864                 free_irq(adap->msix_info[--msi_index].vec,
 865                          &s->rdmarxq[rdmaqidx].rspq);
 866         while (--ofldqidx >= 0)
 867                 free_irq(adap->msix_info[--msi_index].vec,
 868                          &s->ofldrxq[ofldqidx].rspq);
 869         while (--ethqidx >= 0)
 870                 free_irq(adap->msix_info[--msi_index].vec,
 871                          &s->ethrxq[ethqidx].rspq);
 872         free_irq(adap->msix_info[1].vec, &s->fw_evtq);
 873         return err;
 874 }
 875
 876 static void free_msix_queue_irqs(struct adapter *adap)
 877 {
 878         int i, msi_index = 2;
 879         struct sge *s = &adap->sge;
 880
 881         free_irq(adap->msix_info[1].vec, &s->fw_evtq);
 882         for_each_ethrxq(s, i)
 883                 free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq);
 884         for_each_ofldrxq(s, i)
 885                 free_irq(adap->msix_info[msi_index++].vec, &s->ofldrxq[i].rspq);
 886         for_each_rdmarxq(s, i)
 887                 free_irq(adap->msix_info[msi_index++].vec, &s->rdmarxq[i].rspq);
 888 }
 889
 890 /**
 891  *      write_rss - write the RSS table for a given port
 892  *      @pi: the port
 893  *      @queues: array of queue indices for RSS
 894  *
 895  *      Sets up the portion of the HW RSS table for the port's VI to distribute
 896  *      packets to the Rx queues in @queues.
 897  */
 898 static int write_rss(const struct port_info *pi, const u16 *queues)
 899 {
 900         u16 *rss;
 901         int i, err;
 902         const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
 903
 904         rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
 905         if (!rss)
 906                 return -ENOMEM;
 907
 908         /* map the queue indices to queue ids */
 909         for (i = 0; i < pi->rss_size; i++, queues++)
 910                 rss[i] = q[*queues].rspq.abs_id;
 911
 912         err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
 913                                   pi->rss_size, rss, pi->rss_size);
 914         kfree(rss);
 915         return err;
 916 }
 917
 918 /**
 919  *      setup_rss - configure RSS
 920  *      @adap: the adapter
 921  *
 922  *      Sets up RSS for each port.
 923  */
 924 static int setup_rss(struct adapter *adap)
 925 {
 926         int i, err;
 927
 928         for_each_port(adap, i) {
 929                 const struct port_info *pi = adap2pinfo(adap, i);
 930
 931                 err = write_rss(pi, pi->rss);
 932                 if (err)
 933                         return err;
 934         }
 935         return 0;
 936 }
 937
 938 /*
 939  * Return the channel of the ingress queue with the given qid.
 940  */
 941 static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
 942 {
 943         qid -= p->ingr_start;
 944         return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
 945 }
 946
 947 /*
 948  * Wait until all NAPI handlers are descheduled.
 949  */
 950 static void quiesce_rx(struct adapter *adap)
 951 {
 952         int i;
 953
 954         for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
 955                 struct sge_rspq *q = adap->sge.ingr_map[i];
 956
 957                 if (q && q->handler)
 958                         napi_disable(&q->napi);
 959         }
 960 }
 961
 962 /*
 963  * Enable NAPI scheduling and interrupt generation for all Rx queues.
 964  */
 965 static void enable_rx(struct adapter *adap)
 966 {
 967         int i;
 968
 969         for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
 970                 struct sge_rspq *q = adap->sge.ingr_map[i];
 971
 972                 if (!q)
 973                         continue;
 974                 if (q->handler)
 975                         napi_enable(&q->napi);
 976                 /* 0-increment GTS to start the timer and enable interrupts */
 977                 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS),
 978                              SEINTARM(q->intr_params) |
 979                              INGRESSQID(q->cntxt_id));
 980         }
 981 }
 982
 983 /**
 984  *      setup_sge_queues - configure SGE Tx/Rx/response queues
 985  *      @adap: the adapter
 986  *
 987  *      Determines how many sets of SGE queues to use and initializes them.
 988  *      We support multiple queue sets per port if we have MSI-X, otherwise
 989  *      just one queue set per port.
 990  */
 991 static int setup_sge_queues(struct adapter *adap)
 992 {
 993         int err, msi_idx, i, j;
 994         struct sge *s = &adap->sge;
 995
 996         bitmap_zero(s->starving_fl, MAX_EGRQ);
 997         bitmap_zero(s->txq_maperr, MAX_EGRQ);
 998
 999         if (adap->flags & USING_MSIX)
1000                 msi_idx = 1;         /* vector 0 is for non-queue interrupts */
1001         else {
1002                 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
1003                                        NULL, NULL);
1004                 if (err)
1005                         return err;
1006                 msi_idx = -((int)s->intrq.abs_id + 1);
1007         }
1008
1009         err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
1010                                msi_idx, NULL, fwevtq_handler);
1011         if (err) {
1012 freeout:        t4_free_sge_resources(adap);
1013                 return err;
1014         }
1015
1016         for_each_port(adap, i) {
1017                 struct net_device *dev = adap->port[i];
1018                 struct port_info *pi = netdev_priv(dev);
1019                 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
1020                 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
1021
1022                 for (j = 0; j < pi->nqsets; j++, q++) {
1023                         if (msi_idx > 0)
1024                                 msi_idx++;
1025                         err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
1026                                                msi_idx, &q->fl,
1027                                                t4_ethrx_handler);
1028                         if (err)
1029                                 goto freeout;
1030                         q->rspq.idx = j;
1031                         memset(&q->stats, 0, sizeof(q->stats));
1032                 }
1033                 for (j = 0; j < pi->nqsets; j++, t++) {
1034                         err = t4_sge_alloc_eth_txq(adap, t, dev,
1035                                         netdev_get_tx_queue(dev, j),
1036                                         s->fw_evtq.cntxt_id);
1037                         if (err)
1038                                 goto freeout;
1039                 }
1040         }
1041
1042         j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
1043         for_each_ofldrxq(s, i) {
1044                 struct sge_ofld_rxq *q = &s->ofldrxq[i];
1045                 struct net_device *dev = adap->port[i / j];
1046
1047                 if (msi_idx > 0)
1048                         msi_idx++;
1049                 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
1050                                        &q->fl, uldrx_handler);
1051                 if (err)
1052                         goto freeout;
1053                 memset(&q->stats, 0, sizeof(q->stats));
1054                 s->ofld_rxq[i] = q->rspq.abs_id;
1055                 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
1056                                             s->fw_evtq.cntxt_id);
1057                 if (err)
1058                         goto freeout;
1059         }
1060
1061         for_each_rdmarxq(s, i) {
1062                 struct sge_ofld_rxq *q = &s->rdmarxq[i];
1063
1064                 if (msi_idx > 0)
1065                         msi_idx++;
1066                 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
1067                                        msi_idx, &q->fl, uldrx_handler);
1068                 if (err)
1069                         goto freeout;
1070                 memset(&q->stats, 0, sizeof(q->stats));
1071                 s->rdma_rxq[i] = q->rspq.abs_id;
1072         }
1073
1074         for_each_port(adap, i) {
1075                 /*
1076                  * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
1077                  * have RDMA queues, and that's the right value.
1078                  */
1079                 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
1080                                             s->fw_evtq.cntxt_id,
1081                                             s->rdmarxq[i].rspq.cntxt_id);
1082                 if (err)
1083                         goto freeout;
1084         }
1085
1086         t4_write_reg(adap, MPS_TRC_RSS_CONTROL,
1087                      RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) |
1088                      QUEUENUMBER(s->ethrxq[0].rspq.abs_id));
1089         return 0;
1090 }
1091
1092 /*
1093  * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
1094  * The allocated memory is cleared.
1095  */
1096 void *t4_alloc_mem(size_t size)
1097 {
1098         void *p = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
1099
1100         if (!p)
1101                 p = vzalloc(size);
1102         return p;
1103 }
1104
1105 /*
1106  * Free memory allocated through alloc_mem().
1107  */
1108 static void t4_free_mem(void *addr)
1109 {
1110         if (is_vmalloc_addr(addr))
1111                 vfree(addr);
1112         else
1113                 kfree(addr);
1114 }
1115
1116 /* Send a Work Request to write the filter at a specified index.  We construct
1117  * a Firmware Filter Work Request to have the work done and put the indicated
1118  * filter into "pending" mode which will prevent any further actions against
1119  * it till we get a reply from the firmware on the completion status of the
1120  * request.
1121  */
1122 static int set_filter_wr(struct adapter *adapter, int fidx)
1123 {
1124         struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1125         struct sk_buff *skb;
1126         struct fw_filter_wr *fwr;
1127         unsigned int ftid;
1128
1129         /* If the new filter requires loopback Destination MAC and/or VLAN
1130          * rewriting then we need to allocate a Layer 2 Table (L2T) entry for
1131          * the filter.
1132          */
1133         if (f->fs.newdmac || f->fs.newvlan) {
1134                 /* allocate L2T entry for new filter */
1135                 f->l2t = t4_l2t_alloc_switching(adapter->l2t);
1136                 if (f->l2t == NULL)
1137                         return -EAGAIN;
1138                 if (t4_l2t_set_switching(adapter, f->l2t, f->fs.vlan,
1139                                         f->fs.eport, f->fs.dmac)) {
1140                         cxgb4_l2t_release(f->l2t);
1141                         f->l2t = NULL;
1142                         return -ENOMEM;
1143                 }
1144         }
1145
1146         ftid = adapter->tids.ftid_base + fidx;
1147
1148         skb = alloc_skb(sizeof(*fwr), GFP_KERNEL | __GFP_NOFAIL);
1149         fwr = (struct fw_filter_wr *)__skb_put(skb, sizeof(*fwr));
1150         memset(fwr, 0, sizeof(*fwr));
1151
1152         /* It would be nice to put most of the following in t4_hw.c but most
1153          * of the work is translating the cxgbtool ch_filter_specification
1154          * into the Work Request and the definition of that structure is
1155          * currently in cxgbtool.h which isn't appropriate to pull into the
1156          * common code.  We may eventually try to come up with a more neutral
1157          * filter specification structure but for now it's easiest to simply
1158          * put this fairly direct code in line ...
1159          */
1160         fwr->op_pkd = htonl(FW_WR_OP(FW_FILTER_WR));
1161         fwr->len16_pkd = htonl(FW_WR_LEN16(sizeof(*fwr)/16));
1162         fwr->tid_to_iq =
1163                 htonl(V_FW_FILTER_WR_TID(ftid) |
1164                       V_FW_FILTER_WR_RQTYPE(f->fs.type) |
1165                       V_FW_FILTER_WR_NOREPLY(0) |
1166                       V_FW_FILTER_WR_IQ(f->fs.iq));
1167         fwr->del_filter_to_l2tix =
1168                 htonl(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
1169                       V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
1170                       V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
1171                       V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
1172                       V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
1173                       V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
1174                       V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
1175                       V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
1176                       V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
1177                                              f->fs.newvlan == VLAN_REWRITE) |
1178                       V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
1179                                             f->fs.newvlan == VLAN_REWRITE) |
1180                       V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
1181                       V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
1182                       V_FW_FILTER_WR_PRIO(f->fs.prio) |
1183                       V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
1184         fwr->ethtype = htons(f->fs.val.ethtype);
1185         fwr->ethtypem = htons(f->fs.mask.ethtype);
1186         fwr->frag_to_ovlan_vldm =
1187                 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
1188                  V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
1189                  V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.ivlan_vld) |
1190                  V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.ovlan_vld) |
1191                  V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.ivlan_vld) |
1192                  V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.ovlan_vld));
1193         fwr->smac_sel = 0;
1194         fwr->rx_chan_rx_rpl_iq =
1195                 htons(V_FW_FILTER_WR_RX_CHAN(0) |
1196                       V_FW_FILTER_WR_RX_RPL_IQ(adapter->sge.fw_evtq.abs_id));
1197         fwr->maci_to_matchtypem =
1198                 htonl(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
1199                       V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
1200                       V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
1201                       V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
1202                       V_FW_FILTER_WR_PORT(f->fs.val.iport) |
1203                       V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
1204                       V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
1205                       V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
1206         fwr->ptcl = f->fs.val.proto;
1207         fwr->ptclm = f->fs.mask.proto;
1208         fwr->ttyp = f->fs.val.tos;
1209         fwr->ttypm = f->fs.mask.tos;
1210         fwr->ivlan = htons(f->fs.val.ivlan);
1211         fwr->ivlanm = htons(f->fs.mask.ivlan);
1212         fwr->ovlan = htons(f->fs.val.ovlan);
1213         fwr->ovlanm = htons(f->fs.mask.ovlan);
1214         memcpy(fwr->lip, f->fs.val.lip, sizeof(fwr->lip));
1215         memcpy(fwr->lipm, f->fs.mask.lip, sizeof(fwr->lipm));
1216         memcpy(fwr->fip, f->fs.val.fip, sizeof(fwr->fip));
1217         memcpy(fwr->fipm, f->fs.mask.fip, sizeof(fwr->fipm));
1218         fwr->lp = htons(f->fs.val.lport);
1219         fwr->lpm = htons(f->fs.mask.lport);
1220         fwr->fp = htons(f->fs.val.fport);
1221         fwr->fpm = htons(f->fs.mask.fport);
1222         if (f->fs.newsmac)
1223                 memcpy(fwr->sma, f->fs.smac, sizeof(fwr->sma));
1224
1225         /* Mark the filter as "pending" and ship off the Filter Work Request.
1226          * When we get the Work Request Reply we'll clear the pending status.
1227          */
1228         f->pending = 1;
1229         set_wr_txq(skb, CPL_PRIORITY_CONTROL, f->fs.val.iport & 0x3);
1230         t4_ofld_send(adapter, skb);
1231         return 0;
1232 }
1233
1234 /* Delete the filter at a specified index.
1235  */
1236 static int del_filter_wr(struct adapter *adapter, int fidx)
1237 {
1238         struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1239         struct sk_buff *skb;
1240         struct fw_filter_wr *fwr;
1241         unsigned int len, ftid;
1242
1243         len = sizeof(*fwr);
1244         ftid = adapter->tids.ftid_base + fidx;
1245
1246         skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
1247         fwr = (struct fw_filter_wr *)__skb_put(skb, len);
1248         t4_mk_filtdelwr(ftid, fwr, adapter->sge.fw_evtq.abs_id);
1249
1250         /* Mark the filter as "pending" and ship off the Filter Work Request.
1251          * When we get the Work Request Reply we'll clear the pending status.
1252          */
1253         f->pending = 1;
1254         t4_mgmt_tx(adapter, skb);
1255         return 0;
1256 }
1257
1258 static inline int is_offload(const struct adapter *adap)
1259 {
1260         return adap->params.offload;
1261 }
1262
1263 /*
1264  * Implementation of ethtool operations.
1265  */
1266
1267 static u32 get_msglevel(struct net_device *dev)
1268 {
1269         return netdev2adap(dev)->msg_enable;
1270 }
1271
1272 static void set_msglevel(struct net_device *dev, u32 val)
1273 {
1274         netdev2adap(dev)->msg_enable = val;
1275 }
1276
1277 static char stats_strings[][ETH_GSTRING_LEN] = {
1278         "TxOctetsOK         ",
1279         "TxFramesOK         ",
1280         "TxBroadcastFrames  ",
1281         "TxMulticastFrames  ",
1282         "TxUnicastFrames    ",
1283         "TxErrorFrames      ",
1284
1285         "TxFrames64         ",
1286         "TxFrames65To127    ",
1287         "TxFrames128To255   ",
1288         "TxFrames256To511   ",
1289         "TxFrames512To1023  ",
1290         "TxFrames1024To1518 ",
1291         "TxFrames1519ToMax  ",
1292
1293         "TxFramesDropped    ",
1294         "TxPauseFrames      ",
1295         "TxPPP0Frames       ",
1296         "TxPPP1Frames       ",
1297         "TxPPP2Frames       ",
1298         "TxPPP3Frames       ",
1299         "TxPPP4Frames       ",
1300         "TxPPP5Frames       ",
1301         "TxPPP6Frames       ",
1302         "TxPPP7Frames       ",
1303
1304         "RxOctetsOK         ",
1305         "RxFramesOK         ",
1306         "RxBroadcastFrames  ",
1307         "RxMulticastFrames  ",
1308         "RxUnicastFrames    ",
1309
1310         "RxFramesTooLong    ",
1311         "RxJabberErrors     ",
1312         "RxFCSErrors        ",
1313         "RxLengthErrors     ",
1314         "RxSymbolErrors     ",
1315         "RxRuntFrames       ",
1316
1317         "RxFrames64         ",
1318         "RxFrames65To127    ",
1319         "RxFrames128To255   ",
1320         "RxFrames256To511   ",
1321         "RxFrames512To1023  ",
1322         "RxFrames1024To1518 ",
1323         "RxFrames1519ToMax  ",
1324
1325         "RxPauseFrames      ",
1326         "RxPPP0Frames       ",
1327         "RxPPP1Frames       ",
1328         "RxPPP2Frames       ",
1329         "RxPPP3Frames       ",
1330         "RxPPP4Frames       ",
1331         "RxPPP5Frames       ",
1332         "RxPPP6Frames       ",
1333         "RxPPP7Frames       ",
1334
1335         "RxBG0FramesDropped ",
1336         "RxBG1FramesDropped ",
1337         "RxBG2FramesDropped ",
1338         "RxBG3FramesDropped ",
1339         "RxBG0FramesTrunc   ",
1340         "RxBG1FramesTrunc   ",
1341         "RxBG2FramesTrunc   ",
1342         "RxBG3FramesTrunc   ",
1343
1344         "TSO                ",
1345         "TxCsumOffload      ",
1346         "RxCsumGood         ",
1347         "VLANextractions    ",
1348         "VLANinsertions     ",
1349         "GROpackets         ",
1350         "GROmerged          ",
1351         "WriteCoalSuccess   ",
1352         "WriteCoalFail      ",
1353 };
1354
1355 static int get_sset_count(struct net_device *dev, int sset)
1356 {
1357         switch (sset) {
1358         case ETH_SS_STATS:
1359                 return ARRAY_SIZE(stats_strings);
1360         default:
1361                 return -EOPNOTSUPP;
1362         }
1363 }
1364
1365 #define T4_REGMAP_SIZE (160 * 1024)
1366 #define T5_REGMAP_SIZE (332 * 1024)
1367
1368 static int get_regs_len(struct net_device *dev)
1369 {
1370         struct adapter *adap = netdev2adap(dev);
1371         if (is_t4(adap->params.chip))
1372                 return T4_REGMAP_SIZE;
1373         else
1374                 return T5_REGMAP_SIZE;
1375 }
1376
1377 static int get_eeprom_len(struct net_device *dev)
1378 {
1379         return EEPROMSIZE;
1380 }
1381
1382 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1383 {
1384         struct adapter *adapter = netdev2adap(dev);
1385
1386         strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1387         strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1388         strlcpy(info->bus_info, pci_name(adapter->pdev),
1389                 sizeof(info->bus_info));
1390
1391         if (adapter->params.fw_vers)
1392                 snprintf(info->fw_version, sizeof(info->fw_version),
1393                         "%u.%u.%u.%u, TP %u.%u.%u.%u",
1394                         FW_HDR_FW_VER_MAJOR_GET(adapter->params.fw_vers),
1395                         FW_HDR_FW_VER_MINOR_GET(adapter->params.fw_vers),
1396                         FW_HDR_FW_VER_MICRO_GET(adapter->params.fw_vers),
1397                         FW_HDR_FW_VER_BUILD_GET(adapter->params.fw_vers),
1398                         FW_HDR_FW_VER_MAJOR_GET(adapter->params.tp_vers),
1399                         FW_HDR_FW_VER_MINOR_GET(adapter->params.tp_vers),
1400                         FW_HDR_FW_VER_MICRO_GET(adapter->params.tp_vers),
1401                         FW_HDR_FW_VER_BUILD_GET(adapter->params.tp_vers));
1402 }
1403
1404 static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1405 {
1406         if (stringset == ETH_SS_STATS)
1407                 memcpy(data, stats_strings, sizeof(stats_strings));
1408 }
1409
1410 /*
1411  * port stats maintained per queue of the port.  They should be in the same
1412  * order as in stats_strings above.
1413  */
1414 struct queue_port_stats {
1415         u64 tso;
1416         u64 tx_csum;
1417         u64 rx_csum;
1418         u64 vlan_ex;
1419         u64 vlan_ins;
1420         u64 gro_pkts;
1421         u64 gro_merged;
1422 };
1423
1424 static void collect_sge_port_stats(const struct adapter *adap,
1425                 const struct port_info *p, struct queue_port_stats *s)
1426 {
1427         int i;
1428         const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1429         const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1430
1431         memset(s, 0, sizeof(*s));
1432         for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1433                 s->tso += tx->tso;
1434                 s->tx_csum += tx->tx_cso;
1435                 s->rx_csum += rx->stats.rx_cso;
1436                 s->vlan_ex += rx->stats.vlan_ex;
1437                 s->vlan_ins += tx->vlan_ins;
1438                 s->gro_pkts += rx->stats.lro_pkts;
1439                 s->gro_merged += rx->stats.lro_merged;
1440         }
1441 }
1442
1443 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1444                       u64 *data)
1445 {
1446         struct port_info *pi = netdev_priv(dev);
1447         struct adapter *adapter = pi->adapter;
1448         u32 val1, val2;
1449
1450         t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1451
1452         data += sizeof(struct port_stats) / sizeof(u64);
1453         collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1454         data += sizeof(struct queue_port_stats) / sizeof(u64);
1455         if (!is_t4(adapter->params.chip)) {
1456                 t4_write_reg(adapter, SGE_STAT_CFG, STATSOURCE_T5(7));
1457                 val1 = t4_read_reg(adapter, SGE_STAT_TOTAL);
1458                 val2 = t4_read_reg(adapter, SGE_STAT_MATCH);
1459                 *data = val1 - val2;
1460                 data++;
1461                 *data = val2;
1462                 data++;
1463         } else {
1464                 memset(data, 0, 2 * sizeof(u64));
1465                 *data += 2;
1466         }
1467 }
1468
1469 /*
1470  * Return a version number to identify the type of adapter.  The scheme is:
1471  * - bits 0..9: chip version
1472  * - bits 10..15: chip revision
1473  * - bits 16..23: register dump version
1474  */
1475 static inline unsigned int mk_adap_vers(const struct adapter *ap)
1476 {
1477         return CHELSIO_CHIP_VERSION(ap->params.chip) |
1478                 (CHELSIO_CHIP_RELEASE(ap->params.chip) << 10) | (1 << 16);
1479 }
1480
1481 static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1482                            unsigned int end)
1483 {
1484         u32 *p = buf + start;
1485
1486         for ( ; start <= end; start += sizeof(u32))
1487                 *p++ = t4_read_reg(ap, start);
1488 }
1489
1490 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1491                      void *buf)
1492 {
1493         static const unsigned int t4_reg_ranges[] = {
1494                 0x1008, 0x1108,
1495                 0x1180, 0x11b4,
1496                 0x11fc, 0x123c,
1497                 0x1300, 0x173c,
1498                 0x1800, 0x18fc,
1499                 0x3000, 0x30d8,
1500                 0x30e0, 0x5924,
1501                 0x5960, 0x59d4,
1502                 0x5a00, 0x5af8,
1503                 0x6000, 0x6098,
1504                 0x6100, 0x6150,
1505                 0x6200, 0x6208,
1506                 0x6240, 0x6248,
1507                 0x6280, 0x6338,
1508                 0x6370, 0x638c,
1509                 0x6400, 0x643c,
1510                 0x6500, 0x6524,
1511                 0x6a00, 0x6a38,
1512                 0x6a60, 0x6a78,
1513                 0x6b00, 0x6b84,
1514                 0x6bf0, 0x6c84,
1515                 0x6cf0, 0x6d84,
1516                 0x6df0, 0x6e84,
1517                 0x6ef0, 0x6f84,
1518                 0x6ff0, 0x7084,
1519                 0x70f0, 0x7184,
1520                 0x71f0, 0x7284,
1521                 0x72f0, 0x7384,
1522                 0x73f0, 0x7450,
1523                 0x7500, 0x7530,
1524                 0x7600, 0x761c,
1525                 0x7680, 0x76cc,
1526                 0x7700, 0x7798,
1527                 0x77c0, 0x77fc,
1528                 0x7900, 0x79fc,
1529                 0x7b00, 0x7c38,
1530                 0x7d00, 0x7efc,
1531                 0x8dc0, 0x8e1c,
1532                 0x8e30, 0x8e78,
1533                 0x8ea0, 0x8f6c,
1534                 0x8fc0, 0x9074,
1535                 0x90fc, 0x90fc,
1536                 0x9400, 0x9458,
1537                 0x9600, 0x96bc,
1538                 0x9800, 0x9808,
1539                 0x9820, 0x983c,
1540                 0x9850, 0x9864,
1541                 0x9c00, 0x9c6c,
1542                 0x9c80, 0x9cec,
1543                 0x9d00, 0x9d6c,
1544                 0x9d80, 0x9dec,
1545                 0x9e00, 0x9e6c,
1546                 0x9e80, 0x9eec,
1547                 0x9f00, 0x9f6c,
1548                 0x9f80, 0x9fec,
1549                 0xd004, 0xd03c,
1550                 0xdfc0, 0xdfe0,
1551                 0xe000, 0xea7c,
1552                 0xf000, 0x11190,
1553                 0x19040, 0x1906c,
1554                 0x19078, 0x19080,
1555                 0x1908c, 0x19124,
1556                 0x19150, 0x191b0,
1557                 0x191d0, 0x191e8,
1558                 0x19238, 0x1924c,
1559                 0x193f8, 0x19474,
1560                 0x19490, 0x194f8,
1561                 0x19800, 0x19f30,
1562                 0x1a000, 0x1a06c,
1563                 0x1a0b0, 0x1a120,
1564                 0x1a128, 0x1a138,
1565                 0x1a190, 0x1a1c4,
1566                 0x1a1fc, 0x1a1fc,
1567                 0x1e040, 0x1e04c,
1568                 0x1e284, 0x1e28c,
1569                 0x1e2c0, 0x1e2c0,
1570                 0x1e2e0, 0x1e2e0,
1571                 0x1e300, 0x1e384,
1572                 0x1e3c0, 0x1e3c8,
1573                 0x1e440, 0x1e44c,
1574                 0x1e684, 0x1e68c,
1575                 0x1e6c0, 0x1e6c0,
1576                 0x1e6e0, 0x1e6e0,
1577                 0x1e700, 0x1e784,
1578                 0x1e7c0, 0x1e7c8,
1579                 0x1e840, 0x1e84c,
1580                 0x1ea84, 0x1ea8c,
1581                 0x1eac0, 0x1eac0,
1582                 0x1eae0, 0x1eae0,
1583                 0x1eb00, 0x1eb84,
1584                 0x1ebc0, 0x1ebc8,
1585                 0x1ec40, 0x1ec4c,
1586                 0x1ee84, 0x1ee8c,
1587                 0x1eec0, 0x1eec0,
1588                 0x1eee0, 0x1eee0,
1589                 0x1ef00, 0x1ef84,
1590                 0x1efc0, 0x1efc8,
1591                 0x1f040, 0x1f04c,
1592                 0x1f284, 0x1f28c,
1593                 0x1f2c0, 0x1f2c0,
1594                 0x1f2e0, 0x1f2e0,
1595                 0x1f300, 0x1f384,
1596                 0x1f3c0, 0x1f3c8,
1597                 0x1f440, 0x1f44c,
1598                 0x1f684, 0x1f68c,
1599                 0x1f6c0, 0x1f6c0,
1600                 0x1f6e0, 0x1f6e0,
1601                 0x1f700, 0x1f784,
1602                 0x1f7c0, 0x1f7c8,
1603                 0x1f840, 0x1f84c,
1604                 0x1fa84, 0x1fa8c,
1605                 0x1fac0, 0x1fac0,
1606                 0x1fae0, 0x1fae0,
1607                 0x1fb00, 0x1fb84,
1608                 0x1fbc0, 0x1fbc8,
1609                 0x1fc40, 0x1fc4c,
1610                 0x1fe84, 0x1fe8c,
1611                 0x1fec0, 0x1fec0,
1612                 0x1fee0, 0x1fee0,
1613                 0x1ff00, 0x1ff84,
1614                 0x1ffc0, 0x1ffc8,
1615                 0x20000, 0x2002c,
1616                 0x20100, 0x2013c,
1617                 0x20190, 0x201c8,
1618                 0x20200, 0x20318,
1619                 0x20400, 0x20528,
1620                 0x20540, 0x20614,
1621                 0x21000, 0x21040,
1622                 0x2104c, 0x21060,
1623                 0x210c0, 0x210ec,
1624                 0x21200, 0x21268,
1625                 0x21270, 0x21284,
1626                 0x212fc, 0x21388,
1627                 0x21400, 0x21404,
1628                 0x21500, 0x21518,
1629                 0x2152c, 0x2153c,
1630                 0x21550, 0x21554,
1631                 0x21600, 0x21600,
1632                 0x21608, 0x21628,
1633                 0x21630, 0x2163c,
1634                 0x21700, 0x2171c,
1635                 0x21780, 0x2178c,
1636                 0x21800, 0x21c38,
1637                 0x21c80, 0x21d7c,
1638                 0x21e00, 0x21e04,
1639                 0x22000, 0x2202c,
1640                 0x22100, 0x2213c,
1641                 0x22190, 0x221c8,
1642                 0x22200, 0x22318,
1643                 0x22400, 0x22528,
1644                 0x22540, 0x22614,
1645                 0x23000, 0x23040,
1646                 0x2304c, 0x23060,
1647                 0x230c0, 0x230ec,
1648                 0x23200, 0x23268,
1649                 0x23270, 0x23284,
1650                 0x232fc, 0x23388,
1651                 0x23400, 0x23404,
1652                 0x23500, 0x23518,
1653                 0x2352c, 0x2353c,
1654                 0x23550, 0x23554,
1655                 0x23600, 0x23600,
1656                 0x23608, 0x23628,
1657                 0x23630, 0x2363c,
1658                 0x23700, 0x2371c,
1659                 0x23780, 0x2378c,
1660                 0x23800, 0x23c38,
1661                 0x23c80, 0x23d7c,
1662                 0x23e00, 0x23e04,
1663                 0x24000, 0x2402c,
1664                 0x24100, 0x2413c,
1665                 0x24190, 0x241c8,
1666                 0x24200, 0x24318,
1667                 0x24400, 0x24528,
1668                 0x24540, 0x24614,
1669                 0x25000, 0x25040,
1670                 0x2504c, 0x25060,
1671                 0x250c0, 0x250ec,
1672                 0x25200, 0x25268,
1673                 0x25270, 0x25284,
1674                 0x252fc, 0x25388,
1675                 0x25400, 0x25404,
1676                 0x25500, 0x25518,
1677                 0x2552c, 0x2553c,
1678                 0x25550, 0x25554,
1679                 0x25600, 0x25600,
1680                 0x25608, 0x25628,
1681                 0x25630, 0x2563c,
1682                 0x25700, 0x2571c,
1683                 0x25780, 0x2578c,
1684                 0x25800, 0x25c38,
1685                 0x25c80, 0x25d7c,
1686                 0x25e00, 0x25e04,
1687                 0x26000, 0x2602c,
1688                 0x26100, 0x2613c,
1689                 0x26190, 0x261c8,
1690                 0x26200, 0x26318,
1691                 0x26400, 0x26528,
1692                 0x26540, 0x26614,
1693                 0x27000, 0x27040,
1694                 0x2704c, 0x27060,
1695                 0x270c0, 0x270ec,
1696                 0x27200, 0x27268,
1697                 0x27270, 0x27284,
1698                 0x272fc, 0x27388,
1699                 0x27400, 0x27404,
1700                 0x27500, 0x27518,
1701                 0x2752c, 0x2753c,
1702                 0x27550, 0x27554,
1703                 0x27600, 0x27600,
1704                 0x27608, 0x27628,
1705                 0x27630, 0x2763c,
1706                 0x27700, 0x2771c,
1707                 0x27780, 0x2778c,
1708                 0x27800, 0x27c38,
1709                 0x27c80, 0x27d7c,
1710                 0x27e00, 0x27e04
1711         };
1712
1713         static const unsigned int t5_reg_ranges[] = {
1714                 0x1008, 0x1148,
1715                 0x1180, 0x11b4,
1716                 0x11fc, 0x123c,
1717                 0x1280, 0x173c,
1718                 0x1800, 0x18fc,
1719                 0x3000, 0x3028,
1720                 0x3060, 0x30d8,
1721                 0x30e0, 0x30fc,
1722                 0x3140, 0x357c,
1723                 0x35a8, 0x35cc,
1724                 0x35ec, 0x35ec,
1725                 0x3600, 0x5624,
1726                 0x56cc, 0x575c,
1727                 0x580c, 0x5814,
1728                 0x5890, 0x58bc,
1729                 0x5940, 0x59dc,
1730                 0x59fc, 0x5a18,
1731                 0x5a60, 0x5a9c,
1732                 0x5b9c, 0x5bfc,
1733                 0x6000, 0x6040,
1734                 0x6058, 0x614c,
1735                 0x7700, 0x7798,
1736                 0x77c0, 0x78fc,
1737                 0x7b00, 0x7c54,
1738                 0x7d00, 0x7efc,
1739                 0x8dc0, 0x8de0,
1740                 0x8df8, 0x8e84,
1741                 0x8ea0, 0x8f84,
1742                 0x8fc0, 0x90f8,
1743                 0x9400, 0x9470,
1744                 0x9600, 0x96f4,
1745                 0x9800, 0x9808,
1746                 0x9820, 0x983c,
1747                 0x9850, 0x9864,
1748                 0x9c00, 0x9c6c,
1749                 0x9c80, 0x9cec,
1750                 0x9d00, 0x9d6c,
1751                 0x9d80, 0x9dec,
1752                 0x9e00, 0x9e6c,
1753                 0x9e80, 0x9eec,
1754                 0x9f00, 0x9f6c,
1755                 0x9f80, 0xa020,
1756                 0xd004, 0xd03c,
1757                 0xdfc0, 0xdfe0,
1758                 0xe000, 0x11088,
1759                 0x1109c, 0x1117c,
1760                 0x11190, 0x11204,
1761                 0x19040, 0x1906c,
1762                 0x19078, 0x19080,
1763                 0x1908c, 0x19124,
1764                 0x19150, 0x191b0,
1765                 0x191d0, 0x191e8,
1766                 0x19238, 0x19290,
1767                 0x193f8, 0x19474,
1768                 0x19490, 0x194cc,
1769                 0x194f0, 0x194f8,
1770                 0x19c00, 0x19c60,
1771                 0x19c94, 0x19e10,
1772                 0x19e50, 0x19f34,
1773                 0x19f40, 0x19f50,
1774                 0x19f90, 0x19fe4,
1775                 0x1a000, 0x1a06c,
1776                 0x1a0b0, 0x1a120,
1777                 0x1a128, 0x1a138,
1778                 0x1a190, 0x1a1c4,
1779                 0x1a1fc, 0x1a1fc,
1780                 0x1e008, 0x1e00c,
1781                 0x1e040, 0x1e04c,
1782                 0x1e284, 0x1e290,
1783                 0x1e2c0, 0x1e2c0,
1784                 0x1e2e0, 0x1e2e0,
1785                 0x1e300, 0x1e384,
1786                 0x1e3c0, 0x1e3c8,
1787                 0x1e408, 0x1e40c,
1788                 0x1e440, 0x1e44c,
1789                 0x1e684, 0x1e690,
1790                 0x1e6c0, 0x1e6c0,
1791                 0x1e6e0, 0x1e6e0,
1792                 0x1e700, 0x1e784,
1793                 0x1e7c0, 0x1e7c8,
1794                 0x1e808, 0x1e80c,
1795                 0x1e840, 0x1e84c,
1796                 0x1ea84, 0x1ea90,
1797                 0x1eac0, 0x1eac0,
1798                 0x1eae0, 0x1eae0,
1799                 0x1eb00, 0x1eb84,
1800                 0x1ebc0, 0x1ebc8,
1801                 0x1ec08, 0x1ec0c,
1802                 0x1ec40, 0x1ec4c,
1803                 0x1ee84, 0x1ee90,
1804                 0x1eec0, 0x1eec0,
1805                 0x1eee0, 0x1eee0,
1806                 0x1ef00, 0x1ef84,
1807                 0x1efc0, 0x1efc8,
1808                 0x1f008, 0x1f00c,
1809                 0x1f040, 0x1f04c,
1810                 0x1f284, 0x1f290,
1811                 0x1f2c0, 0x1f2c0,
1812                 0x1f2e0, 0x1f2e0,
1813                 0x1f300, 0x1f384,
1814                 0x1f3c0, 0x1f3c8,
1815                 0x1f408, 0x1f40c,
1816                 0x1f440, 0x1f44c,
1817                 0x1f684, 0x1f690,
1818                 0x1f6c0, 0x1f6c0,
1819                 0x1f6e0, 0x1f6e0,
1820                 0x1f700, 0x1f784,
1821                 0x1f7c0, 0x1f7c8,
1822                 0x1f808, 0x1f80c,
1823                 0x1f840, 0x1f84c,
1824                 0x1fa84, 0x1fa90,
1825                 0x1fac0, 0x1fac0,
1826                 0x1fae0, 0x1fae0,
1827                 0x1fb00, 0x1fb84,
1828                 0x1fbc0, 0x1fbc8,
1829                 0x1fc08, 0x1fc0c,
1830                 0x1fc40, 0x1fc4c,
1831                 0x1fe84, 0x1fe90,
1832                 0x1fec0, 0x1fec0,
1833                 0x1fee0, 0x1fee0,
1834                 0x1ff00, 0x1ff84,
1835                 0x1ffc0, 0x1ffc8,
1836                 0x30000, 0x30030,
1837                 0x30100, 0x30144,
1838                 0x30190, 0x301d0,
1839                 0x30200, 0x30318,
1840                 0x30400, 0x3052c,
1841                 0x30540, 0x3061c,
1842                 0x30800, 0x30834,
1843                 0x308c0, 0x30908,
1844                 0x30910, 0x309ac,
1845                 0x30a00, 0x30a04,
1846                 0x30a0c, 0x30a2c,
1847                 0x30a44, 0x30a50,
1848                 0x30a74, 0x30c24,
1849                 0x30d08, 0x30d14,
1850                 0x30d1c, 0x30d20,
1851                 0x30d3c, 0x30d50,
1852                 0x31200, 0x3120c,
1853                 0x31220, 0x31220,
1854                 0x31240, 0x31240,
1855                 0x31600, 0x31600,
1856                 0x31608, 0x3160c,
1857                 0x31a00, 0x31a1c,
1858                 0x31e04, 0x31e20,
1859                 0x31e38, 0x31e3c,
1860                 0x31e80, 0x31e80,
1861                 0x31e88, 0x31ea8,
1862                 0x31eb0, 0x31eb4,
1863                 0x31ec8, 0x31ed4,
1864                 0x31fb8, 0x32004,
1865                 0x32208, 0x3223c,
1866                 0x32600, 0x32630,
1867                 0x32a00, 0x32abc,
1868                 0x32b00, 0x32b70,
1869                 0x33000, 0x33048,
1870                 0x33060, 0x3309c,
1871                 0x330f0, 0x33148,
1872                 0x33160, 0x3319c,
1873                 0x331f0, 0x332e4,
1874                 0x332f8, 0x333e4,
1875                 0x333f8, 0x33448,
1876                 0x33460, 0x3349c,
1877                 0x334f0, 0x33548,
1878                 0x33560, 0x3359c,
1879                 0x335f0, 0x336e4,
1880                 0x336f8, 0x337e4,
1881                 0x337f8, 0x337fc,
1882                 0x33814, 0x33814,
1883                 0x3382c, 0x3382c,
1884                 0x33880, 0x3388c,
1885                 0x338e8, 0x338ec,
1886                 0x33900, 0x33948,
1887                 0x33960, 0x3399c,
1888                 0x339f0, 0x33ae4,
1889                 0x33af8, 0x33b10,
1890                 0x33b28, 0x33b28,
1891                 0x33b3c, 0x33b50,
1892                 0x33bf0, 0x33c10,
1893                 0x33c28, 0x33c28,
1894                 0x33c3c, 0x33c50,
1895                 0x33cf0, 0x33cfc,
1896                 0x34000, 0x34030,
1897                 0x34100, 0x34144,
1898                 0x34190, 0x341d0,
1899                 0x34200, 0x34318,
1900                 0x34400, 0x3452c,
1901                 0x34540, 0x3461c,
1902                 0x34800, 0x34834,
1903                 0x348c0, 0x34908,
1904                 0x34910, 0x349ac,
1905                 0x34a00, 0x34a04,
1906                 0x34a0c, 0x34a2c,
1907                 0x34a44, 0x34a50,
1908                 0x34a74, 0x34c24,
1909                 0x34d08, 0x34d14,
1910                 0x34d1c, 0x34d20,
1911                 0x34d3c, 0x34d50,
1912                 0x35200, 0x3520c,
1913                 0x35220, 0x35220,
1914                 0x35240, 0x35240,
1915                 0x35600, 0x35600,
1916                 0x35608, 0x3560c,
1917                 0x35a00, 0x35a1c,
1918                 0x35e04, 0x35e20,
1919                 0x35e38, 0x35e3c,
1920                 0x35e80, 0x35e80,
1921                 0x35e88, 0x35ea8,
1922                 0x35eb0, 0x35eb4,
1923                 0x35ec8, 0x35ed4,
1924                 0x35fb8, 0x36004,
1925                 0x36208, 0x3623c,
1926                 0x36600, 0x36630,
1927                 0x36a00, 0x36abc,
1928                 0x36b00, 0x36b70,
1929                 0x37000, 0x37048,
1930                 0x37060, 0x3709c,
1931                 0x370f0, 0x37148,
1932                 0x37160, 0x3719c,
1933                 0x371f0, 0x372e4,
1934                 0x372f8, 0x373e4,
1935                 0x373f8, 0x37448,
1936                 0x37460, 0x3749c,
1937                 0x374f0, 0x37548,
1938                 0x37560, 0x3759c,
1939                 0x375f0, 0x376e4,
1940                 0x376f8, 0x377e4,
1941                 0x377f8, 0x377fc,
1942                 0x37814, 0x37814,
1943                 0x3782c, 0x3782c,
1944                 0x37880, 0x3788c,
1945                 0x378e8, 0x378ec,
1946                 0x37900, 0x37948,
1947                 0x37960, 0x3799c,
1948                 0x379f0, 0x37ae4,
1949                 0x37af8, 0x37b10,
1950                 0x37b28, 0x37b28,
1951                 0x37b3c, 0x37b50,
1952                 0x37bf0, 0x37c10,
1953                 0x37c28, 0x37c28,
1954                 0x37c3c, 0x37c50,
1955                 0x37cf0, 0x37cfc,
1956                 0x38000, 0x38030,
1957                 0x38100, 0x38144,
1958                 0x38190, 0x381d0,
1959                 0x38200, 0x38318,
1960                 0x38400, 0x3852c,
1961                 0x38540, 0x3861c,
1962                 0x38800, 0x38834,
1963                 0x388c0, 0x38908,
1964                 0x38910, 0x389ac,
1965                 0x38a00, 0x38a04,
1966                 0x38a0c, 0x38a2c,
1967                 0x38a44, 0x38a50,
1968                 0x38a74, 0x38c24,
1969                 0x38d08, 0x38d14,
1970                 0x38d1c, 0x38d20,
1971                 0x38d3c, 0x38d50,
1972                 0x39200, 0x3920c,
1973                 0x39220, 0x39220,
1974                 0x39240, 0x39240,
1975                 0x39600, 0x39600,
1976                 0x39608, 0x3960c,
1977                 0x39a00, 0x39a1c,
1978                 0x39e04, 0x39e20,
1979                 0x39e38, 0x39e3c,
1980                 0x39e80, 0x39e80,
1981                 0x39e88, 0x39ea8,
1982                 0x39eb0, 0x39eb4,
1983                 0x39ec8, 0x39ed4,
1984                 0x39fb8, 0x3a004,
1985                 0x3a208, 0x3a23c,
1986                 0x3a600, 0x3a630,
1987                 0x3aa00, 0x3aabc,
1988                 0x3ab00, 0x3ab70,
1989                 0x3b000, 0x3b048,
1990                 0x3b060, 0x3b09c,
1991                 0x3b0f0, 0x3b148,
1992                 0x3b160, 0x3b19c,
1993                 0x3b1f0, 0x3b2e4,
1994                 0x3b2f8, 0x3b3e4,
1995                 0x3b3f8, 0x3b448,
1996                 0x3b460, 0x3b49c,
1997                 0x3b4f0, 0x3b548,
1998                 0x3b560, 0x3b59c,
1999                 0x3b5f0, 0x3b6e4,
2000                 0x3b6f8, 0x3b7e4,
2001                 0x3b7f8, 0x3b7fc,
2002                 0x3b814, 0x3b814,
2003                 0x3b82c, 0x3b82c,
2004                 0x3b880, 0x3b88c,
2005                 0x3b8e8, 0x3b8ec,
2006                 0x3b900, 0x3b948,
2007                 0x3b960, 0x3b99c,
2008                 0x3b9f0, 0x3bae4,
2009                 0x3baf8, 0x3bb10,
2010                 0x3bb28, 0x3bb28,
2011                 0x3bb3c, 0x3bb50,
2012                 0x3bbf0, 0x3bc10,
2013                 0x3bc28, 0x3bc28,
2014                 0x3bc3c, 0x3bc50,
2015                 0x3bcf0, 0x3bcfc,
2016                 0x3c000, 0x3c030,
2017                 0x3c100, 0x3c144,
2018                 0x3c190, 0x3c1d0,
2019                 0x3c200, 0x3c318,
2020                 0x3c400, 0x3c52c,
2021                 0x3c540, 0x3c61c,
2022                 0x3c800, 0x3c834,
2023                 0x3c8c0, 0x3c908,
2024                 0x3c910, 0x3c9ac,
2025                 0x3ca00, 0x3ca04,
2026                 0x3ca0c, 0x3ca2c,
2027                 0x3ca44, 0x3ca50,
2028                 0x3ca74, 0x3cc24,
2029                 0x3cd08, 0x3cd14,
2030                 0x3cd1c, 0x3cd20,
2031                 0x3cd3c, 0x3cd50,
2032                 0x3d200, 0x3d20c,
2033                 0x3d220, 0x3d220,
2034                 0x3d240, 0x3d240,
2035                 0x3d600, 0x3d600,
2036                 0x3d608, 0x3d60c,
2037                 0x3da00, 0x3da1c,
2038                 0x3de04, 0x3de20,
2039                 0x3de38, 0x3de3c,
2040                 0x3de80, 0x3de80,
2041                 0x3de88, 0x3dea8,
2042                 0x3deb0, 0x3deb4,
2043                 0x3dec8, 0x3ded4,
2044                 0x3dfb8, 0x3e004,
2045                 0x3e208, 0x3e23c,
2046                 0x3e600, 0x3e630,
2047                 0x3ea00, 0x3eabc,
2048                 0x3eb00, 0x3eb70,
2049                 0x3f000, 0x3f048,
2050                 0x3f060, 0x3f09c,
2051                 0x3f0f0, 0x3f148,
2052                 0x3f160, 0x3f19c,
2053                 0x3f1f0, 0x3f2e4,
2054                 0x3f2f8, 0x3f3e4,
2055                 0x3f3f8, 0x3f448,
2056                 0x3f460, 0x3f49c,
2057                 0x3f4f0, 0x3f548,
2058                 0x3f560, 0x3f59c,
2059                 0x3f5f0, 0x3f6e4,
2060                 0x3f6f8, 0x3f7e4,
2061                 0x3f7f8, 0x3f7fc,
2062                 0x3f814, 0x3f814,
2063                 0x3f82c, 0x3f82c,
2064                 0x3f880, 0x3f88c,
2065                 0x3f8e8, 0x3f8ec,
2066                 0x3f900, 0x3f948,
2067                 0x3f960, 0x3f99c,
2068                 0x3f9f0, 0x3fae4,
2069                 0x3faf8, 0x3fb10,
2070                 0x3fb28, 0x3fb28,
2071                 0x3fb3c, 0x3fb50,
2072                 0x3fbf0, 0x3fc10,
2073                 0x3fc28, 0x3fc28,
2074                 0x3fc3c, 0x3fc50,
2075                 0x3fcf0, 0x3fcfc,
2076                 0x40000, 0x4000c,
2077                 0x40040, 0x40068,
2078                 0x40080, 0x40144,
2079                 0x40180, 0x4018c,
2080                 0x40200, 0x40298,
2081                 0x402ac, 0x4033c,
2082                 0x403f8, 0x403fc,
2083                 0x41304, 0x413c4,
2084                 0x41400, 0x4141c,
2085                 0x41480, 0x414d0,
2086                 0x44000, 0x44078,
2087                 0x440c0, 0x44278,
2088                 0x442c0, 0x44478,
2089                 0x444c0, 0x44678,
2090                 0x446c0, 0x44878,
2091                 0x448c0, 0x449fc,
2092                 0x45000, 0x45068,
2093                 0x45080, 0x45084,
2094                 0x450a0, 0x450b0,
2095                 0x45200, 0x45268,
2096                 0x45280, 0x45284,
2097                 0x452a0, 0x452b0,
2098                 0x460c0, 0x460e4,
2099                 0x47000, 0x4708c,
2100                 0x47200, 0x47250,
2101                 0x47400, 0x47420,
2102                 0x47600, 0x47618,
2103                 0x47800, 0x47814,
2104                 0x48000, 0x4800c,
2105                 0x48040, 0x48068,
2106                 0x48080, 0x48144,
2107                 0x48180, 0x4818c,
2108                 0x48200, 0x48298,
2109                 0x482ac, 0x4833c,
2110                 0x483f8, 0x483fc,
2111                 0x49304, 0x493c4,
2112                 0x49400, 0x4941c,
2113                 0x49480, 0x494d0,
2114                 0x4c000, 0x4c078,
2115                 0x4c0c0, 0x4c278,
2116                 0x4c2c0, 0x4c478,
2117                 0x4c4c0, 0x4c678,
2118                 0x4c6c0, 0x4c878,
2119                 0x4c8c0, 0x4c9fc,
2120                 0x4d000, 0x4d068,
2121                 0x4d080, 0x4d084,
2122                 0x4d0a0, 0x4d0b0,
2123                 0x4d200, 0x4d268,
2124                 0x4d280, 0x4d284,
2125                 0x4d2a0, 0x4d2b0,
2126                 0x4e0c0, 0x4e0e4,
2127                 0x4f000, 0x4f08c,
2128                 0x4f200, 0x4f250,
2129                 0x4f400, 0x4f420,
2130                 0x4f600, 0x4f618,
2131                 0x4f800, 0x4f814,
2132                 0x50000, 0x500cc,
2133                 0x50400, 0x50400,
2134                 0x50800, 0x508cc,
2135                 0x50c00, 0x50c00,
2136                 0x51000, 0x5101c,
2137                 0x51300, 0x51308,
2138         };
2139
2140         int i;
2141         struct adapter *ap = netdev2adap(dev);
2142         static const unsigned int *reg_ranges;
2143         int arr_size = 0, buf_size = 0;
2144
2145         if (is_t4(ap->params.chip)) {
2146                 reg_ranges = &t4_reg_ranges[0];
2147                 arr_size = ARRAY_SIZE(t4_reg_ranges);
2148                 buf_size = T4_REGMAP_SIZE;
2149         } else {
2150                 reg_ranges = &t5_reg_ranges[0];
2151                 arr_size = ARRAY_SIZE(t5_reg_ranges);
2152                 buf_size = T5_REGMAP_SIZE;
2153         }
2154
2155         regs->version = mk_adap_vers(ap);
2156
2157         memset(buf, 0, buf_size);
2158         for (i = 0; i < arr_size; i += 2)
2159                 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
2160 }
2161
2162 static int restart_autoneg(struct net_device *dev)
2163 {
2164         struct port_info *p = netdev_priv(dev);
2165
2166         if (!netif_running(dev))
2167                 return -EAGAIN;
2168         if (p->link_cfg.autoneg != AUTONEG_ENABLE)
2169                 return -EINVAL;
2170         t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
2171         return 0;
2172 }
2173
2174 static int identify_port(struct net_device *dev,
2175                          enum ethtool_phys_id_state state)
2176 {
2177         unsigned int val;
2178         struct adapter *adap = netdev2adap(dev);
2179
2180         if (state == ETHTOOL_ID_ACTIVE)
2181                 val = 0xffff;
2182         else if (state == ETHTOOL_ID_INACTIVE)
2183                 val = 0;
2184         else
2185                 return -EINVAL;
2186
2187         return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val);
2188 }
2189
2190 static unsigned int from_fw_linkcaps(unsigned int type, unsigned int caps)
2191 {
2192         unsigned int v = 0;
2193
2194         if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
2195             type == FW_PORT_TYPE_BT_XAUI) {
2196                 v |= SUPPORTED_TP;
2197                 if (caps & FW_PORT_CAP_SPEED_100M)
2198                         v |= SUPPORTED_100baseT_Full;
2199                 if (caps & FW_PORT_CAP_SPEED_1G)
2200                         v |= SUPPORTED_1000baseT_Full;
2201                 if (caps & FW_PORT_CAP_SPEED_10G)
2202                         v |= SUPPORTED_10000baseT_Full;
2203         } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
2204                 v |= SUPPORTED_Backplane;
2205                 if (caps & FW_PORT_CAP_SPEED_1G)
2206                         v |= SUPPORTED_1000baseKX_Full;
2207                 if (caps & FW_PORT_CAP_SPEED_10G)
2208                         v |= SUPPORTED_10000baseKX4_Full;
2209         } else if (type == FW_PORT_TYPE_KR)
2210                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
2211         else if (type == FW_PORT_TYPE_BP_AP)
2212                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2213                      SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full;
2214         else if (type == FW_PORT_TYPE_BP4_AP)
2215                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2216                      SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full |
2217                      SUPPORTED_10000baseKX4_Full;
2218         else if (type == FW_PORT_TYPE_FIBER_XFI ||
2219                  type == FW_PORT_TYPE_FIBER_XAUI || type == FW_PORT_TYPE_SFP)
2220                 v |= SUPPORTED_FIBRE;
2221         else if (type == FW_PORT_TYPE_BP40_BA)
2222                 v |= SUPPORTED_40000baseSR4_Full;
2223
2224         if (caps & FW_PORT_CAP_ANEG)
2225                 v |= SUPPORTED_Autoneg;
2226         return v;
2227 }
2228
2229 static unsigned int to_fw_linkcaps(unsigned int caps)
2230 {
2231         unsigned int v = 0;
2232
2233         if (caps & ADVERTISED_100baseT_Full)
2234                 v |= FW_PORT_CAP_SPEED_100M;
2235         if (caps & ADVERTISED_1000baseT_Full)
2236                 v |= FW_PORT_CAP_SPEED_1G;
2237         if (caps & ADVERTISED_10000baseT_Full)
2238                 v |= FW_PORT_CAP_SPEED_10G;
2239         if (caps & ADVERTISED_40000baseSR4_Full)
2240                 v |= FW_PORT_CAP_SPEED_40G;
2241         return v;
2242 }
2243
2244 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2245 {
2246         const struct port_info *p = netdev_priv(dev);
2247
2248         if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
2249             p->port_type == FW_PORT_TYPE_BT_XFI ||
2250             p->port_type == FW_PORT_TYPE_BT_XAUI)
2251                 cmd->port = PORT_TP;
2252         else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
2253                  p->port_type == FW_PORT_TYPE_FIBER_XAUI)
2254                 cmd->port = PORT_FIBRE;
2255         else if (p->port_type == FW_PORT_TYPE_SFP ||
2256                  p->port_type == FW_PORT_TYPE_QSFP_10G ||
2257                  p->port_type == FW_PORT_TYPE_QSFP) {
2258                 if (p->mod_type == FW_PORT_MOD_TYPE_LR ||
2259                     p->mod_type == FW_PORT_MOD_TYPE_SR ||
2260                     p->mod_type == FW_PORT_MOD_TYPE_ER ||
2261                     p->mod_type == FW_PORT_MOD_TYPE_LRM)
2262                         cmd->port = PORT_FIBRE;
2263                 else if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
2264                          p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
2265                         cmd->port = PORT_DA;
2266                 else
2267                         cmd->port = PORT_OTHER;
2268         } else
2269                 cmd->port = PORT_OTHER;
2270
2271         if (p->mdio_addr >= 0) {
2272                 cmd->phy_address = p->mdio_addr;
2273                 cmd->transceiver = XCVR_EXTERNAL;
2274                 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
2275                         MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
2276         } else {
2277                 cmd->phy_address = 0;  /* not really, but no better option */
2278                 cmd->transceiver = XCVR_INTERNAL;
2279                 cmd->mdio_support = 0;
2280         }
2281
2282         cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
2283         cmd->advertising = from_fw_linkcaps(p->port_type,
2284                                             p->link_cfg.advertising);
2285         ethtool_cmd_speed_set(cmd,
2286                               netif_carrier_ok(dev) ? p->link_cfg.speed : 0);
2287         cmd->duplex = DUPLEX_FULL;
2288         cmd->autoneg = p->link_cfg.autoneg;
2289         cmd->maxtxpkt = 0;
2290         cmd->maxrxpkt = 0;
2291         return 0;
2292 }
2293
2294 static unsigned int speed_to_caps(int speed)
2295 {
2296         if (speed == 100)
2297                 return FW_PORT_CAP_SPEED_100M;
2298         if (speed == 1000)
2299                 return FW_PORT_CAP_SPEED_1G;
2300         if (speed == 10000)
2301                 return FW_PORT_CAP_SPEED_10G;
2302         if (speed == 40000)
2303                 return FW_PORT_CAP_SPEED_40G;
2304         return 0;
2305 }
2306
2307 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2308 {
2309         unsigned int cap;
2310         struct port_info *p = netdev_priv(dev);
2311         struct link_config *lc = &p->link_cfg;
2312         u32 speed = ethtool_cmd_speed(cmd);
2313
2314         if (cmd->duplex != DUPLEX_FULL)     /* only full-duplex supported */
2315                 return -EINVAL;
2316
2317         if (!(lc->supported & FW_PORT_CAP_ANEG)) {
2318                 /*
2319                  * PHY offers a single speed.  See if that's what's
2320                  * being requested.
2321                  */
2322                 if (cmd->autoneg == AUTONEG_DISABLE &&
2323                     (lc->supported & speed_to_caps(speed)))
2324                         return 0;
2325                 return -EINVAL;
2326         }
2327
2328         if (cmd->autoneg == AUTONEG_DISABLE) {
2329                 cap = speed_to_caps(speed);
2330
2331                 if (!(lc->supported & cap) ||
2332                     (speed == 1000) ||
2333                     (speed == 10000) ||
2334                     (speed == 40000))
2335                         return -EINVAL;
2336                 lc->requested_speed = cap;
2337                 lc->advertising = 0;
2338         } else {
2339                 cap = to_fw_linkcaps(cmd->advertising);
2340                 if (!(lc->supported & cap))
2341                         return -EINVAL;
2342                 lc->requested_speed = 0;
2343                 lc->advertising = cap | FW_PORT_CAP_ANEG;
2344         }
2345         lc->autoneg = cmd->autoneg;
2346
2347         if (netif_running(dev))
2348                 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2349                                      lc);
2350         return 0;
2351 }
2352
2353 static void get_pauseparam(struct net_device *dev,
2354                            struct ethtool_pauseparam *epause)
2355 {
2356         struct port_info *p = netdev_priv(dev);
2357
2358         epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
2359         epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
2360         epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
2361 }
2362
2363 static int set_pauseparam(struct net_device *dev,
2364                           struct ethtool_pauseparam *epause)
2365 {
2366         struct port_info *p = netdev_priv(dev);
2367         struct link_config *lc = &p->link_cfg;
2368
2369         if (epause->autoneg == AUTONEG_DISABLE)
2370                 lc->requested_fc = 0;
2371         else if (lc->supported & FW_PORT_CAP_ANEG)
2372                 lc->requested_fc = PAUSE_AUTONEG;
2373         else
2374                 return -EINVAL;
2375
2376         if (epause->rx_pause)
2377                 lc->requested_fc |= PAUSE_RX;
2378         if (epause->tx_pause)
2379                 lc->requested_fc |= PAUSE_TX;
2380         if (netif_running(dev))
2381                 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2382                                      lc);
2383         return 0;
2384 }
2385
2386 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2387 {
2388         const struct port_info *pi = netdev_priv(dev);
2389         const struct sge *s = &pi->adapter->sge;
2390
2391         e->rx_max_pending = MAX_RX_BUFFERS;
2392         e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
2393         e->rx_jumbo_max_pending = 0;
2394         e->tx_max_pending = MAX_TXQ_ENTRIES;
2395
2396         e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
2397         e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
2398         e->rx_jumbo_pending = 0;
2399         e->tx_pending = s->ethtxq[pi->first_qset].q.size;
2400 }
2401
2402 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2403 {
2404         int i;
2405         const struct port_info *pi = netdev_priv(dev);
2406         struct adapter *adapter = pi->adapter;
2407         struct sge *s = &adapter->sge;
2408
2409         if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
2410             e->tx_pending > MAX_TXQ_ENTRIES ||
2411             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
2412             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
2413             e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
2414                 return -EINVAL;
2415
2416         if (adapter->flags & FULL_INIT_DONE)
2417                 return -EBUSY;
2418
2419         for (i = 0; i < pi->nqsets; ++i) {
2420                 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
2421                 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
2422                 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
2423         }
2424         return 0;
2425 }
2426
2427 static int closest_timer(const struct sge *s, int time)
2428 {
2429         int i, delta, match = 0, min_delta = INT_MAX;
2430
2431         for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
2432                 delta = time - s->timer_val[i];
2433                 if (delta < 0)
2434                         delta = -delta;
2435                 if (delta < min_delta) {
2436                         min_delta = delta;
2437                         match = i;
2438                 }
2439         }
2440         return match;
2441 }
2442
2443 static int closest_thres(const struct sge *s, int thres)
2444 {
2445         int i, delta, match = 0, min_delta = INT_MAX;
2446
2447         for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
2448                 delta = thres - s->counter_val[i];
2449                 if (delta < 0)
2450                         delta = -delta;
2451                 if (delta < min_delta) {
2452                         min_delta = delta;
2453                         match = i;
2454                 }
2455         }
2456         return match;
2457 }
2458
2459 /*
2460  * Return a queue's interrupt hold-off time in us.  0 means no timer.
2461  */
2462 static unsigned int qtimer_val(const struct adapter *adap,
2463                                const struct sge_rspq *q)
2464 {
2465         unsigned int idx = q->intr_params >> 1;
2466
2467         return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
2468 }
2469
2470 /**
2471  *      set_rxq_intr_params - set a queue's interrupt holdoff parameters
2472  *      @adap: the adapter
2473  *      @q: the Rx queue
2474  *      @us: the hold-off time in us, or 0 to disable timer
2475  *      @cnt: the hold-off packet count, or 0 to disable counter
2476  *
2477  *      Sets an Rx queue's interrupt hold-off time and packet count.  At least
2478  *      one of the two needs to be enabled for the queue to generate interrupts.
2479  */
2480 static int set_rxq_intr_params(struct adapter *adap, struct sge_rspq *q,
2481                                unsigned int us, unsigned int cnt)
2482 {
2483         if ((us | cnt) == 0)
2484                 cnt = 1;
2485
2486         if (cnt) {
2487                 int err;
2488                 u32 v, new_idx;
2489
2490                 new_idx = closest_thres(&adap->sge, cnt);
2491                 if (q->desc && q->pktcnt_idx != new_idx) {
2492                         /* the queue has already been created, update it */
2493                         v = FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
2494                             FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
2495                             FW_PARAMS_PARAM_YZ(q->cntxt_id);
2496                         err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
2497                                             &new_idx);
2498                         if (err)
2499                                 return err;
2500                 }
2501                 q->pktcnt_idx = new_idx;
2502         }
2503
2504         us = us == 0 ? 6 : closest_timer(&adap->sge, us);
2505         q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
2506         return 0;
2507 }
2508
2509 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2510 {
2511         const struct port_info *pi = netdev_priv(dev);
2512         struct adapter *adap = pi->adapter;
2513         struct sge_rspq *q;
2514         int i;
2515         int r = 0;
2516
2517         for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++) {
2518                 q = &adap->sge.ethrxq[i].rspq;
2519                 r = set_rxq_intr_params(adap, q, c->rx_coalesce_usecs,
2520                         c->rx_max_coalesced_frames);
2521                 if (r) {
2522                         dev_err(&dev->dev, "failed to set coalesce %d\n", r);
2523                         break;
2524                 }
2525         }
2526         return r;
2527 }
2528
2529 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2530 {
2531         const struct port_info *pi = netdev_priv(dev);
2532         const struct adapter *adap = pi->adapter;
2533         const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
2534
2535         c->rx_coalesce_usecs = qtimer_val(adap, rq);
2536         c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
2537                 adap->sge.counter_val[rq->pktcnt_idx] : 0;
2538         return 0;
2539 }
2540
2541 /**
2542  *      eeprom_ptov - translate a physical EEPROM address to virtual
2543  *      @phys_addr: the physical EEPROM address
2544  *      @fn: the PCI function number
2545  *      @sz: size of function-specific area
2546  *
2547  *      Translate a physical EEPROM address to virtual.  The first 1K is
2548  *      accessed through virtual addresses starting at 31K, the rest is
2549  *      accessed through virtual addresses starting at 0.
2550  *
2551  *      The mapping is as follows:
2552  *      [0..1K) -> [31K..32K)
2553  *      [1K..1K+A) -> [31K-A..31K)
2554  *      [1K+A..ES) -> [0..ES-A-1K)
2555  *
2556  *      where A = @fn * @sz, and ES = EEPROM size.
2557  */
2558 static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
2559 {
2560         fn *= sz;
2561         if (phys_addr < 1024)
2562                 return phys_addr + (31 << 10);
2563         if (phys_addr < 1024 + fn)
2564                 return 31744 - fn + phys_addr - 1024;
2565         if (phys_addr < EEPROMSIZE)
2566                 return phys_addr - 1024 - fn;
2567         return -EINVAL;
2568 }
2569
2570 /*
2571  * The next two routines implement eeprom read/write from physical addresses.
2572  */
2573 static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
2574 {
2575         int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
2576
2577         if (vaddr >= 0)
2578                 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
2579         return vaddr < 0 ? vaddr : 0;
2580 }
2581
2582 static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
2583 {
2584         int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
2585
2586         if (vaddr >= 0)
2587                 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
2588         return vaddr < 0 ? vaddr : 0;
2589 }
2590
2591 #define EEPROM_MAGIC 0x38E2F10C
2592
2593 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
2594                       u8 *data)
2595 {
2596         int i, err = 0;
2597         struct adapter *adapter = netdev2adap(dev);
2598
2599         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
2600         if (!buf)
2601                 return -ENOMEM;
2602
2603         e->magic = EEPROM_MAGIC;
2604         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
2605                 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
2606
2607         if (!err)
2608                 memcpy(data, buf + e->offset, e->len);
2609         kfree(buf);
2610         return err;
2611 }
2612
2613 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
2614                       u8 *data)
2615 {
2616         u8 *buf;
2617         int err = 0;
2618         u32 aligned_offset, aligned_len, *p;
2619         struct adapter *adapter = netdev2adap(dev);
2620
2621         if (eeprom->magic != EEPROM_MAGIC)
2622                 return -EINVAL;
2623
2624         aligned_offset = eeprom->offset & ~3;
2625         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2626
2627         if (adapter->fn > 0) {
2628                 u32 start = 1024 + adapter->fn * EEPROMPFSIZE;
2629
2630                 if (aligned_offset < start ||
2631                     aligned_offset + aligned_len > start + EEPROMPFSIZE)
2632                         return -EPERM;
2633         }
2634
2635         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2636                 /*
2637                  * RMW possibly needed for first or last words.
2638                  */
2639                 buf = kmalloc(aligned_len, GFP_KERNEL);
2640                 if (!buf)
2641                         return -ENOMEM;
2642                 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
2643                 if (!err && aligned_len > 4)
2644                         err = eeprom_rd_phys(adapter,
2645                                              aligned_offset + aligned_len - 4,
2646                                              (u32 *)&buf[aligned_len - 4]);
2647                 if (err)
2648                         goto out;
2649                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2650         } else
2651                 buf = data;
2652
2653         err = t4_seeprom_wp(adapter, false);
2654         if (err)
2655                 goto out;
2656
2657         for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2658                 err = eeprom_wr_phys(adapter, aligned_offset, *p);
2659                 aligned_offset += 4;
2660         }
2661
2662         if (!err)
2663                 err = t4_seeprom_wp(adapter, true);
2664 out:
2665         if (buf != data)
2666                 kfree(buf);
2667         return err;
2668 }
2669
2670 static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
2671 {
2672         int ret;
2673         const struct firmware *fw;
2674         struct adapter *adap = netdev2adap(netdev);
2675
2676         ef->data[sizeof(ef->data) - 1] = '\0';
2677         ret = request_firmware(&fw, ef->data, adap->pdev_dev);
2678         if (ret < 0)
2679                 return ret;
2680
2681         ret = t4_load_fw(adap, fw->data, fw->size);
2682         release_firmware(fw);
2683         if (!ret)
2684                 dev_info(adap->pdev_dev, "loaded firmware %s\n", ef->data);
2685         return ret;
2686 }
2687
2688 #define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
2689 #define BCAST_CRC 0xa0ccc1a6
2690
2691 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2692 {
2693         wol->supported = WAKE_BCAST | WAKE_MAGIC;
2694         wol->wolopts = netdev2adap(dev)->wol;
2695         memset(&wol->sopass, 0, sizeof(wol->sopass));
2696 }
2697
2698 static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2699 {
2700         int err = 0;
2701         struct port_info *pi = netdev_priv(dev);
2702
2703         if (wol->wolopts & ~WOL_SUPPORTED)
2704                 return -EINVAL;
2705         t4_wol_magic_enable(pi->adapter, pi->tx_chan,
2706                             (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
2707         if (wol->wolopts & WAKE_BCAST) {
2708                 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
2709                                         ~0ULL, 0, false);
2710                 if (!err)
2711                         err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
2712                                                 ~6ULL, ~0ULL, BCAST_CRC, true);
2713         } else
2714                 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
2715         return err;
2716 }
2717
2718 static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
2719 {
2720         const struct port_info *pi = netdev_priv(dev);
2721         netdev_features_t changed = dev->features ^ features;
2722         int err;
2723
2724         if (!(changed & NETIF_F_HW_VLAN_CTAG_RX))
2725                 return 0;
2726
2727         err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1,
2728                             -1, -1, -1,
2729                             !!(features & NETIF_F_HW_VLAN_CTAG_RX), true);
2730         if (unlikely(err))
2731                 dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX;
2732         return err;
2733 }
2734
2735 static u32 get_rss_table_size(struct net_device *dev)
2736 {
2737         const struct port_info *pi = netdev_priv(dev);
2738
2739         return pi->rss_size;
2740 }
2741
2742 static int get_rss_table(struct net_device *dev, u32 *p)
2743 {
2744         const struct port_info *pi = netdev_priv(dev);
2745         unsigned int n = pi->rss_size;
2746
2747         while (n--)
2748                 p[n] = pi->rss[n];
2749         return 0;
2750 }
2751
2752 static int set_rss_table(struct net_device *dev, const u32 *p)
2753 {
2754         unsigned int i;
2755         struct port_info *pi = netdev_priv(dev);
2756
2757         for (i = 0; i < pi->rss_size; i++)
2758                 pi->rss[i] = p[i];
2759         if (pi->adapter->flags & FULL_INIT_DONE)
2760                 return write_rss(pi, pi->rss);
2761         return 0;
2762 }
2763
2764 static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
2765                      u32 *rules)
2766 {
2767         const struct port_info *pi = netdev_priv(dev);
2768
2769         switch (info->cmd) {
2770         case ETHTOOL_GRXFH: {
2771                 unsigned int v = pi->rss_mode;
2772
2773                 info->data = 0;
2774                 switch (info->flow_type) {
2775                 case TCP_V4_FLOW:
2776                         if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
2777                                 info->data = RXH_IP_SRC | RXH_IP_DST |
2778                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
2779                         else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
2780                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2781                         break;
2782                 case UDP_V4_FLOW:
2783                         if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) &&
2784                             (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
2785                                 info->data = RXH_IP_SRC | RXH_IP_DST |
2786                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
2787                         else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
2788                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2789                         break;
2790                 case SCTP_V4_FLOW:
2791                 case AH_ESP_V4_FLOW:
2792                 case IPV4_FLOW:
2793                         if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
2794                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2795                         break;
2796                 case TCP_V6_FLOW:
2797                         if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
2798                                 info->data = RXH_IP_SRC | RXH_IP_DST |
2799                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
2800                         else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
2801                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2802                         break;
2803                 case UDP_V6_FLOW:
2804                         if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) &&
2805                             (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
2806                                 info->data = RXH_IP_SRC | RXH_IP_DST |
2807                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
2808                         else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
2809                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2810                         break;
2811                 case SCTP_V6_FLOW:
2812                 case AH_ESP_V6_FLOW:
2813                 case IPV6_FLOW:
2814                         if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
2815                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2816                         break;
2817                 }
2818                 return 0;
2819         }
2820         case ETHTOOL_GRXRINGS:
2821                 info->data = pi->nqsets;
2822                 return 0;
2823         }
2824         return -EOPNOTSUPP;
2825 }
2826
2827 static const struct ethtool_ops cxgb_ethtool_ops = {
2828         .get_settings      = get_settings,
2829         .set_settings      = set_settings,
2830         .get_drvinfo       = get_drvinfo,
2831         .get_msglevel      = get_msglevel,
2832         .set_msglevel      = set_msglevel,
2833         .get_ringparam     = get_sge_param,
2834         .set_ringparam     = set_sge_param,
2835         .get_coalesce      = get_coalesce,
2836         .set_coalesce      = set_coalesce,
2837         .get_eeprom_len    = get_eeprom_len,
2838         .get_eeprom        = get_eeprom,
2839         .set_eeprom        = set_eeprom,
2840         .get_pauseparam    = get_pauseparam,
2841         .set_pauseparam    = set_pauseparam,
2842         .get_link          = ethtool_op_get_link,
2843         .get_strings       = get_strings,
2844         .set_phys_id       = identify_port,
2845         .nway_reset        = restart_autoneg,
2846         .get_sset_count    = get_sset_count,
2847         .get_ethtool_stats = get_stats,
2848         .get_regs_len      = get_regs_len,
2849         .get_regs          = get_regs,
2850         .get_wol           = get_wol,
2851         .set_wol           = set_wol,
2852         .get_rxnfc         = get_rxnfc,
2853         .get_rxfh_indir_size = get_rss_table_size,
2854         .get_rxfh_indir    = get_rss_table,
2855         .set_rxfh_indir    = set_rss_table,
2856         .flash_device      = set_flash,
2857 };
2858
2859 /*
2860  * debugfs support
2861  */
2862 static ssize_t mem_read(struct file *file, char __user *buf, size_t count,
2863                         loff_t *ppos)
2864 {
2865         loff_t pos = *ppos;
2866         loff_t avail = file_inode(file)->i_size;
2867         unsigned int mem = (uintptr_t)file->private_data & 3;
2868         struct adapter *adap = file->private_data - mem;
2869
2870         if (pos < 0)
2871                 return -EINVAL;
2872         if (pos >= avail)
2873                 return 0;
2874         if (count > avail - pos)
2875                 count = avail - pos;
2876
2877         while (count) {
2878                 size_t len;
2879                 int ret, ofst;
2880                 __be32 data[16];
2881
2882                 if ((mem == MEM_MC) || (mem == MEM_MC1))
2883                         ret = t4_mc_read(adap, mem % MEM_MC, pos, data, NULL);
2884                 else
2885                         ret = t4_edc_read(adap, mem, pos, data, NULL);
2886                 if (ret)
2887                         return ret;
2888
2889                 ofst = pos % sizeof(data);
2890                 len = min(count, sizeof(data) - ofst);
2891                 if (copy_to_user(buf, (u8 *)data + ofst, len))
2892                         return -EFAULT;
2893
2894                 buf += len;
2895                 pos += len;
2896                 count -= len;
2897         }
2898         count = pos - *ppos;
2899         *ppos = pos;
2900         return count;
2901 }
2902
2903 static const struct file_operations mem_debugfs_fops = {
2904         .owner   = THIS_MODULE,
2905         .open    = simple_open,
2906         .read    = mem_read,
2907         .llseek  = default_llseek,
2908 };
2909
2910 static void add_debugfs_mem(struct adapter *adap, const char *name,
2911                             unsigned int idx, unsigned int size_mb)
2912 {
2913         struct dentry *de;
2914
2915         de = debugfs_create_file(name, S_IRUSR, adap->debugfs_root,
2916                                  (void *)adap + idx, &mem_debugfs_fops);
2917         if (de && de->d_inode)
2918                 de->d_inode->i_size = size_mb << 20;
2919 }
2920
2921 static int setup_debugfs(struct adapter *adap)
2922 {
2923         int i;
2924         u32 size;
2925
2926         if (IS_ERR_OR_NULL(adap->debugfs_root))
2927                 return -1;
2928
2929         i = t4_read_reg(adap, MA_TARGET_MEM_ENABLE);
2930         if (i & EDRAM0_ENABLE) {
2931                 size = t4_read_reg(adap, MA_EDRAM0_BAR);
2932                 add_debugfs_mem(adap, "edc0", MEM_EDC0, EDRAM_SIZE_GET(size));
2933         }
2934         if (i & EDRAM1_ENABLE) {
2935                 size = t4_read_reg(adap, MA_EDRAM1_BAR);
2936                 add_debugfs_mem(adap, "edc1", MEM_EDC1, EDRAM_SIZE_GET(size));
2937         }
2938         if (is_t4(adap->params.chip)) {
2939                 size = t4_read_reg(adap, MA_EXT_MEMORY_BAR);
2940                 if (i & EXT_MEM_ENABLE)
2941                         add_debugfs_mem(adap, "mc", MEM_MC,
2942                                         EXT_MEM_SIZE_GET(size));
2943         } else {
2944                 if (i & EXT_MEM_ENABLE) {
2945                         size = t4_read_reg(adap, MA_EXT_MEMORY_BAR);
2946                         add_debugfs_mem(adap, "mc0", MEM_MC0,
2947                                         EXT_MEM_SIZE_GET(size));
2948                 }
2949                 if (i & EXT_MEM1_ENABLE) {
2950                         size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR);
2951                         add_debugfs_mem(adap, "mc1", MEM_MC1,
2952                                         EXT_MEM_SIZE_GET(size));
2953                 }
2954         }
2955         if (adap->l2t)
2956                 debugfs_create_file("l2t", S_IRUSR, adap->debugfs_root, adap,
2957                                     &t4_l2t_fops);
2958         return 0;
2959 }
2960
2961 /*
2962  * upper-layer driver support
2963  */
2964
2965 /*
2966  * Allocate an active-open TID and set it to the supplied value.
2967  */
2968 int cxgb4_alloc_atid(struct tid_info *t, void *data)
2969 {
2970         int atid = -1;
2971
2972         spin_lock_bh(&t->atid_lock);
2973         if (t->afree) {
2974                 union aopen_entry *p = t->afree;
2975
2976                 atid = (p - t->atid_tab) + t->atid_base;
2977                 t->afree = p->next;
2978                 p->data = data;
2979                 t->atids_in_use++;
2980         }
2981         spin_unlock_bh(&t->atid_lock);
2982         return atid;
2983 }
2984 EXPORT_SYMBOL(cxgb4_alloc_atid);
2985
2986 /*
2987  * Release an active-open TID.
2988  */
2989 void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
2990 {
2991         union aopen_entry *p = &t->atid_tab[atid - t->atid_base];
2992
2993         spin_lock_bh(&t->atid_lock);
2994         p->next = t->afree;
2995         t->afree = p;
2996         t->atids_in_use--;
2997         spin_unlock_bh(&t->atid_lock);
2998 }
2999 EXPORT_SYMBOL(cxgb4_free_atid);
3000
3001 /*
3002  * Allocate a server TID and set it to the supplied value.
3003  */
3004 int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
3005 {
3006         int stid;
3007
3008         spin_lock_bh(&t->stid_lock);
3009         if (family == PF_INET) {
3010                 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
3011                 if (stid < t->nstids)
3012                         __set_bit(stid, t->stid_bmap);
3013                 else
3014                         stid = -1;
3015         } else {
3016                 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
3017                 if (stid < 0)
3018                         stid = -1;
3019         }
3020         if (stid >= 0) {
3021                 t->stid_tab[stid].data = data;
3022                 stid += t->stid_base;
3023                 /* IPv6 requires max of 520 bits or 16 cells in TCAM
3024                  * This is equivalent to 4 TIDs. With CLIP enabled it
3025                  * needs 2 TIDs.
3026                  */
3027                 if (family == PF_INET)
3028                         t->stids_in_use++;
3029                 else
3030                         t->stids_in_use += 4;
3031         }
3032         spin_unlock_bh(&t->stid_lock);
3033         return stid;
3034 }
3035 EXPORT_SYMBOL(cxgb4_alloc_stid);
3036
3037 /* Allocate a server filter TID and set it to the supplied value.
3038  */
3039 int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data)
3040 {
3041         int stid;
3042
3043         spin_lock_bh(&t->stid_lock);
3044         if (family == PF_INET) {
3045                 stid = find_next_zero_bit(t->stid_bmap,
3046                                 t->nstids + t->nsftids, t->nstids);
3047                 if (stid < (t->nstids + t->nsftids))
3048                         __set_bit(stid, t->stid_bmap);
3049                 else
3050                         stid = -1;
3051         } else {
3052                 stid = -1;
3053         }
3054         if (stid >= 0) {
3055                 t->stid_tab[stid].data = data;
3056                 stid -= t->nstids;
3057                 stid += t->sftid_base;
3058                 t->stids_in_use++;
3059         }
3060         spin_unlock_bh(&t->stid_lock);
3061         return stid;
3062 }
3063 EXPORT_SYMBOL(cxgb4_alloc_sftid);
3064
3065 /* Release a server TID.
3066  */
3067 void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
3068 {
3069         /* Is it a server filter TID? */
3070         if (t->nsftids && (stid >= t->sftid_base)) {
3071                 stid -= t->sftid_base;
3072                 stid += t->nstids;
3073         } else {
3074                 stid -= t->stid_base;
3075         }
3076
3077         spin_lock_bh(&t->stid_lock);
3078         if (family == PF_INET)
3079                 __clear_bit(stid, t->stid_bmap);
3080         else
3081                 bitmap_release_region(t->stid_bmap, stid, 2);
3082         t->stid_tab[stid].data = NULL;
3083         if (family == PF_INET)
3084                 t->stids_in_use--;
3085         else
3086                 t->stids_in_use -= 4;
3087         spin_unlock_bh(&t->stid_lock);
3088 }
3089 EXPORT_SYMBOL(cxgb4_free_stid);
3090
3091 /*
3092  * Populate a TID_RELEASE WR.  Caller must properly size the skb.
3093  */
3094 static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
3095                            unsigned int tid)
3096 {
3097         struct cpl_tid_release *req;
3098
3099         set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
3100         req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
3101         INIT_TP_WR(req, tid);
3102         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
3103 }
3104
3105 /*
3106  * Queue a TID release request and if necessary schedule a work queue to
3107  * process it.
3108  */
3109 static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
3110                                     unsigned int tid)
3111 {
3112         void **p = &t->tid_tab[tid];
3113         struct adapter *adap = container_of(t, struct adapter, tids);
3114
3115         spin_lock_bh(&adap->tid_release_lock);
3116         *p = adap->tid_release_head;
3117         /* Low 2 bits encode the Tx channel number */
3118         adap->tid_release_head = (void **)((uintptr_t)p | chan);
3119         if (!adap->tid_release_task_busy) {
3120                 adap->tid_release_task_busy = true;
3121                 queue_work(workq, &adap->tid_release_task);
3122         }
3123         spin_unlock_bh(&adap->tid_release_lock);
3124 }
3125
3126 /*
3127  * Process the list of pending TID release requests.
3128  */
3129 static void process_tid_release_list(struct work_struct *work)
3130 {
3131         struct sk_buff *skb;
3132         struct adapter *adap;
3133
3134         adap = container_of(work, struct adapter, tid_release_task);
3135
3136         spin_lock_bh(&adap->tid_release_lock);
3137         while (adap->tid_release_head) {
3138                 void **p = adap->tid_release_head;
3139                 unsigned int chan = (uintptr_t)p & 3;
3140                 p = (void *)p - chan;
3141
3142                 adap->tid_release_head = *p;
3143                 *p = NULL;
3144                 spin_unlock_bh(&adap->tid_release_lock);
3145
3146                 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
3147                                          GFP_KERNEL)))
3148                         schedule_timeout_uninterruptible(1);
3149
3150                 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
3151                 t4_ofld_send(adap, skb);
3152                 spin_lock_bh(&adap->tid_release_lock);
3153         }
3154         adap->tid_release_task_busy = false;
3155         spin_unlock_bh(&adap->tid_release_lock);
3156 }
3157
3158 /*
3159  * Release a TID and inform HW.  If we are unable to allocate the release
3160  * message we defer to a work queue.
3161  */
3162 void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
3163 {
3164         void *old;
3165         struct sk_buff *skb;
3166         struct adapter *adap = container_of(t, struct adapter, tids);
3167
3168         old = t->tid_tab[tid];
3169         skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
3170         if (likely(skb)) {
3171                 t->tid_tab[tid] = NULL;
3172                 mk_tid_release(skb, chan, tid);
3173                 t4_ofld_send(adap, skb);
3174         } else
3175                 cxgb4_queue_tid_release(t, chan, tid);
3176         if (old)
3177                 atomic_dec(&t->tids_in_use);
3178 }
3179 EXPORT_SYMBOL(cxgb4_remove_tid);
3180
3181 /*
3182  * Allocate and initialize the TID tables.  Returns 0 on success.
3183  */
3184 static int tid_init(struct tid_info *t)
3185 {
3186         size_t size;
3187         unsigned int stid_bmap_size;
3188         unsigned int natids = t->natids;
3189         struct adapter *adap = container_of(t, struct adapter, tids);
3190
3191         stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids);
3192         size = t->ntids * sizeof(*t->tid_tab) +
3193                natids * sizeof(*t->atid_tab) +
3194                t->nstids * sizeof(*t->stid_tab) +
3195                t->nsftids * sizeof(*t->stid_tab) +
3196                stid_bmap_size * sizeof(long) +
3197                t->nftids * sizeof(*t->ftid_tab) +
3198                t->nsftids * sizeof(*t->ftid_tab);
3199
3200         t->tid_tab = t4_alloc_mem(size);
3201         if (!t->tid_tab)
3202                 return -ENOMEM;
3203
3204         t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
3205         t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
3206         t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids];
3207         t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size];
3208         spin_lock_init(&t->stid_lock);
3209         spin_lock_init(&t->atid_lock);
3210
3211         t->stids_in_use = 0;
3212         t->afree = NULL;
3213         t->atids_in_use = 0;
3214         atomic_set(&t->tids_in_use, 0);
3215
3216         /* Setup the free list for atid_tab and clear the stid bitmap. */
3217         if (natids) {
3218                 while (--natids)
3219                         t->atid_tab[natids - 1].next = &t->atid_tab[natids];
3220                 t->afree = t->atid_tab;
3221         }
3222         bitmap_zero(t->stid_bmap, t->nstids + t->nsftids);
3223         /* Reserve stid 0 for T4/T5 adapters */
3224         if (!t->stid_base &&
3225             (is_t4(adap->params.chip) || is_t5(adap->params.chip)))
3226                 __set_bit(0, t->stid_bmap);
3227
3228         return 0;
3229 }
3230
3231 static int cxgb4_clip_get(const struct net_device *dev,
3232                           const struct in6_addr *lip)
3233 {
3234         struct adapter *adap;
3235         struct fw_clip_cmd c;
3236
3237         adap = netdev2adap(dev);
3238         memset(&c, 0, sizeof(c));
3239         c.op_to_write = htonl(FW_CMD_OP(FW_CLIP_CMD) |
3240                         FW_CMD_REQUEST | FW_CMD_WRITE);
3241         c.alloc_to_len16 = htonl(F_FW_CLIP_CMD_ALLOC | FW_LEN16(c));
3242         c.ip_hi = *(__be64 *)(lip->s6_addr);
3243         c.ip_lo = *(__be64 *)(lip->s6_addr + 8);
3244         return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, false);
3245 }
3246
3247 static int cxgb4_clip_release(const struct net_device *dev,
3248                               const struct in6_addr *lip)
3249 {
3250         struct adapter *adap;
3251         struct fw_clip_cmd c;
3252
3253         adap = netdev2adap(dev);
3254         memset(&c, 0, sizeof(c));
3255         c.op_to_write = htonl(FW_CMD_OP(FW_CLIP_CMD) |
3256                         FW_CMD_REQUEST | FW_CMD_READ);
3257         c.alloc_to_len16 = htonl(F_FW_CLIP_CMD_FREE | FW_LEN16(c));
3258         c.ip_hi = *(__be64 *)(lip->s6_addr);
3259         c.ip_lo = *(__be64 *)(lip->s6_addr + 8);
3260         return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, false);
3261 }
3262
3263 /**
3264  *      cxgb4_create_server - create an IP server
3265  *      @dev: the device
3266  *      @stid: the server TID
3267  *      @sip: local IP address to bind server to
3268  *      @sport: the server's TCP port
3269  *      @queue: queue to direct messages from this server to
3270  *
3271  *      Create an IP server for the given port and address.
3272  *      Returns <0 on error and one of the %NET_XMIT_* values on success.
3273  */
3274 int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
3275                         __be32 sip, __be16 sport, __be16 vlan,
3276                         unsigned int queue)
3277 {
3278         unsigned int chan;
3279         struct sk_buff *skb;
3280         struct adapter *adap;
3281         struct cpl_pass_open_req *req;
3282         int ret;
3283
3284         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3285         if (!skb)
3286                 return -ENOMEM;
3287
3288         adap = netdev2adap(dev);
3289         req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
3290         INIT_TP_WR(req, 0);
3291         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
3292         req->local_port = sport;
3293         req->peer_port = htons(0);
3294         req->local_ip = sip;
3295         req->peer_ip = htonl(0);
3296         chan = rxq_to_chan(&adap->sge, queue);
3297         req->opt0 = cpu_to_be64(TX_CHAN(chan));
3298         req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
3299                                 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
3300         ret = t4_mgmt_tx(adap, skb);
3301         return net_xmit_eval(ret);
3302 }
3303 EXPORT_SYMBOL(cxgb4_create_server);
3304
3305 /*      cxgb4_create_server6 - create an IPv6 server
3306  *      @dev: the device
3307  *      @stid: the server TID
3308  *      @sip: local IPv6 address to bind server to
3309  *      @sport: the server's TCP port
3310  *      @queue: queue to direct messages from this server to
3311  *
3312  *      Create an IPv6 server for the given port and address.
3313  *      Returns <0 on error and one of the %NET_XMIT_* values on success.
3314  */
3315 int cxgb4_create_server6(const struct net_device *dev, unsigned int stid,
3316                          const struct in6_addr *sip, __be16 sport,
3317                          unsigned int queue)
3318 {
3319         unsigned int chan;
3320         struct sk_buff *skb;
3321         struct adapter *adap;
3322         struct cpl_pass_open_req6 *req;
3323         int ret;
3324
3325         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3326         if (!skb)
3327                 return -ENOMEM;
3328
3329         adap = netdev2adap(dev);
3330         req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req));
3331         INIT_TP_WR(req, 0);
3332         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid));
3333         req->local_port = sport;
3334         req->peer_port = htons(0);
3335         req->local_ip_hi = *(__be64 *)(sip->s6_addr);
3336         req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8);
3337         req->peer_ip_hi = cpu_to_be64(0);
3338         req->peer_ip_lo = cpu_to_be64(0);
3339         chan = rxq_to_chan(&adap->sge, queue);
3340         req->opt0 = cpu_to_be64(TX_CHAN(chan));
3341         req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
3342                                 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
3343         ret = t4_mgmt_tx(adap, skb);
3344         return net_xmit_eval(ret);
3345 }
3346 EXPORT_SYMBOL(cxgb4_create_server6);
3347
3348 int cxgb4_remove_server(const struct net_device *dev, unsigned int stid,
3349                         unsigned int queue, bool ipv6)
3350 {
3351         struct sk_buff *skb;
3352         struct adapter *adap;
3353         struct cpl_close_listsvr_req *req;
3354         int ret;
3355
3356         adap = netdev2adap(dev);
3357
3358         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3359         if (!skb)
3360                 return -ENOMEM;
3361
3362         req = (struct cpl_close_listsvr_req *)__skb_put(skb, sizeof(*req));
3363         INIT_TP_WR(req, 0);
3364         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, stid));
3365         req->reply_ctrl = htons(NO_REPLY(0) | (ipv6 ? LISTSVR_IPV6(1) :
3366                                 LISTSVR_IPV6(0)) | QUEUENO(queue));
3367         ret = t4_mgmt_tx(adap, skb);
3368         return net_xmit_eval(ret);
3369 }
3370 EXPORT_SYMBOL(cxgb4_remove_server);
3371
3372 /**
3373  *      cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
3374  *      @mtus: the HW MTU table
3375  *      @mtu: the target MTU
3376  *      @idx: index of selected entry in the MTU table
3377  *
3378  *      Returns the index and the value in the HW MTU table that is closest to
3379  *      but does not exceed @mtu, unless @mtu is smaller than any value in the
3380  *      table, in which case that smallest available value is selected.
3381  */
3382 unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
3383                             unsigned int *idx)
3384 {
3385         unsigned int i = 0;
3386
3387         while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
3388                 ++i;
3389         if (idx)
3390                 *idx = i;
3391         return mtus[i];
3392 }
3393 EXPORT_SYMBOL(cxgb4_best_mtu);
3394
3395 /**
3396  *      cxgb4_port_chan - get the HW channel of a port
3397  *      @dev: the net device for the port
3398  *
3399  *      Return the HW Tx channel of the given port.
3400  */
3401 unsigned int cxgb4_port_chan(const struct net_device *dev)
3402 {
3403         return netdev2pinfo(dev)->tx_chan;
3404 }
3405 EXPORT_SYMBOL(cxgb4_port_chan);
3406
3407 unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo)
3408 {
3409         struct adapter *adap = netdev2adap(dev);
3410         u32 v1, v2, lp_count, hp_count;
3411
3412         v1 = t4_read_reg(adap, A_SGE_DBFIFO_STATUS);
3413         v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2);
3414         if (is_t4(adap->params.chip)) {
3415                 lp_count = G_LP_COUNT(v1);
3416                 hp_count = G_HP_COUNT(v1);
3417         } else {
3418                 lp_count = G_LP_COUNT_T5(v1);
3419                 hp_count = G_HP_COUNT_T5(v2);
3420         }
3421         return lpfifo ? lp_count : hp_count;
3422 }
3423 EXPORT_SYMBOL(cxgb4_dbfifo_count);
3424
3425 /**
3426  *      cxgb4_port_viid - get the VI id of a port
3427  *      @dev: the net device for the port
3428  *
3429  *      Return the VI id of the given port.
3430  */
3431 unsigned int cxgb4_port_viid(const struct net_device *dev)
3432 {
3433         return netdev2pinfo(dev)->viid;
3434 }
3435 EXPORT_SYMBOL(cxgb4_port_viid);
3436
3437 /**
3438  *      cxgb4_port_idx - get the index of a port
3439  *      @dev: the net device for the port
3440  *
3441  *      Return the index of the given port.
3442  */
3443 unsigned int cxgb4_port_idx(const struct net_device *dev)
3444 {
3445         return netdev2pinfo(dev)->port_id;
3446 }
3447 EXPORT_SYMBOL(cxgb4_port_idx);
3448
3449 void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
3450                          struct tp_tcp_stats *v6)
3451 {
3452         struct adapter *adap = pci_get_drvdata(pdev);
3453
3454         spin_lock(&adap->stats_lock);
3455         t4_tp_get_tcp_stats(adap, v4, v6);
3456         spin_unlock(&adap->stats_lock);
3457 }
3458 EXPORT_SYMBOL(cxgb4_get_tcp_stats);
3459
3460 void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
3461                       const unsigned int *pgsz_order)
3462 {
3463         struct adapter *adap = netdev2adap(dev);
3464
3465         t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask);
3466         t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) |
3467                      HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) |
3468                      HPZ3(pgsz_order[3]));
3469 }
3470 EXPORT_SYMBOL(cxgb4_iscsi_init);
3471
3472 int cxgb4_flush_eq_cache(struct net_device *dev)
3473 {
3474         struct adapter *adap = netdev2adap(dev);
3475         int ret;
3476
3477         ret = t4_fwaddrspace_write(adap, adap->mbox,
3478                                    0xe1000000 + A_SGE_CTXT_CMD, 0x20000000);
3479         return ret;
3480 }
3481 EXPORT_SYMBOL(cxgb4_flush_eq_cache);
3482
3483 static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx)
3484 {
3485         u32 addr = t4_read_reg(adap, A_SGE_DBQ_CTXT_BADDR) + 24 * qid + 8;
3486         __be64 indices;
3487         int ret;
3488
3489         ret = t4_mem_win_read_len(adap, addr, (__be32 *)&indices, 8);
3490         if (!ret) {
3491                 *cidx = (be64_to_cpu(indices) >> 25) & 0xffff;
3492                 *pidx = (be64_to_cpu(indices) >> 9) & 0xffff;
3493         }
3494         return ret;
3495 }
3496
3497 int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx,
3498                         u16 size)
3499 {
3500         struct adapter *adap = netdev2adap(dev);
3501         u16 hw_pidx, hw_cidx;
3502         int ret;
3503
3504         ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx);
3505         if (ret)
3506                 goto out;
3507
3508         if (pidx != hw_pidx) {
3509                 u16 delta;
3510
3511                 if (pidx >= hw_pidx)
3512                         delta = pidx - hw_pidx;
3513                 else
3514                         delta = size - hw_pidx + pidx;
3515                 wmb();
3516                 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),
3517                              QID(qid) | PIDX(delta));
3518         }
3519 out:
3520         return ret;
3521 }
3522 EXPORT_SYMBOL(cxgb4_sync_txq_pidx);
3523
3524 void cxgb4_disable_db_coalescing(struct net_device *dev)
3525 {
3526         struct adapter *adap;
3527
3528         adap = netdev2adap(dev);
3529         t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_NOCOALESCE,
3530                          F_NOCOALESCE);
3531 }
3532 EXPORT_SYMBOL(cxgb4_disable_db_coalescing);
3533
3534 void cxgb4_enable_db_coalescing(struct net_device *dev)
3535 {
3536         struct adapter *adap;
3537
3538         adap = netdev2adap(dev);
3539         t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_NOCOALESCE, 0);
3540 }
3541 EXPORT_SYMBOL(cxgb4_enable_db_coalescing);
3542
3543 static struct pci_driver cxgb4_driver;
3544
3545 static void check_neigh_update(struct neighbour *neigh)
3546 {
3547         const struct device *parent;
3548         const struct net_device *netdev = neigh->dev;
3549
3550         if (netdev->priv_flags & IFF_802_1Q_VLAN)
3551                 netdev = vlan_dev_real_dev(netdev);
3552         parent = netdev->dev.parent;
3553         if (parent && parent->driver == &cxgb4_driver.driver)
3554                 t4_l2t_update(dev_get_drvdata(parent), neigh);
3555 }
3556
3557 static int netevent_cb(struct notifier_block *nb, unsigned long event,
3558                        void *data)
3559 {
3560         switch (event) {
3561         case NETEVENT_NEIGH_UPDATE:
3562                 check_neigh_update(data);
3563                 break;
3564         case NETEVENT_REDIRECT:
3565         default:
3566                 break;
3567         }
3568         return 0;
3569 }
3570
3571 static bool netevent_registered;
3572 static struct notifier_block cxgb4_netevent_nb = {
3573         .notifier_call = netevent_cb
3574 };
3575
3576 static void drain_db_fifo(struct adapter *adap, int usecs)
3577 {
3578         u32 v1, v2, lp_count, hp_count;
3579
3580         do {
3581                 v1 = t4_read_reg(adap, A_SGE_DBFIFO_STATUS);
3582                 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2);
3583                 if (is_t4(adap->params.chip)) {
3584                         lp_count = G_LP_COUNT(v1);
3585                         hp_count = G_HP_COUNT(v1);
3586                 } else {
3587                         lp_count = G_LP_COUNT_T5(v1);
3588                         hp_count = G_HP_COUNT_T5(v2);
3589                 }
3590
3591                 if (lp_count == 0 && hp_count == 0)
3592                         break;
3593                 set_current_state(TASK_UNINTERRUPTIBLE);
3594                 schedule_timeout(usecs_to_jiffies(usecs));
3595         } while (1);
3596 }
3597
3598 static void disable_txq_db(struct sge_txq *q)
3599 {
3600         unsigned long flags;
3601
3602         spin_lock_irqsave(&q->db_lock, flags);
3603         q->db_disabled = 1;
3604         spin_unlock_irqrestore(&q->db_lock, flags);
3605 }
3606
3607 static void enable_txq_db(struct adapter *adap, struct sge_txq *q)
3608 {
3609         spin_lock_irq(&q->db_lock);
3610         if (q->db_pidx_inc) {
3611                 /* Make sure that all writes to the TX descriptors
3612                  * are committed before we tell HW about them.
3613                  */
3614                 wmb();
3615                 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),
3616                              QID(q->cntxt_id) | PIDX(q->db_pidx_inc));
3617                 q->db_pidx_inc = 0;
3618         }
3619         q->db_disabled = 0;
3620         spin_unlock_irq(&q->db_lock);
3621 }
3622
3623 static void disable_dbs(struct adapter *adap)
3624 {
3625         int i;
3626
3627         for_each_ethrxq(&adap->sge, i)
3628                 disable_txq_db(&adap->sge.ethtxq[i].q);
3629         for_each_ofldrxq(&adap->sge, i)
3630                 disable_txq_db(&adap->sge.ofldtxq[i].q);
3631         for_each_port(adap, i)
3632                 disable_txq_db(&adap->sge.ctrlq[i].q);
3633 }
3634
3635 static void enable_dbs(struct adapter *adap)
3636 {
3637         int i;
3638
3639         for_each_ethrxq(&adap->sge, i)
3640                 enable_txq_db(adap, &adap->sge.ethtxq[i].q);
3641         for_each_ofldrxq(&adap->sge, i)
3642                 enable_txq_db(adap, &adap->sge.ofldtxq[i].q);
3643         for_each_port(adap, i)
3644                 enable_txq_db(adap, &adap->sge.ctrlq[i].q);
3645 }
3646
3647 static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd)
3648 {
3649         if (adap->uld_handle[CXGB4_ULD_RDMA])
3650                 ulds[CXGB4_ULD_RDMA].control(adap->uld_handle[CXGB4_ULD_RDMA],
3651                                 cmd);
3652 }
3653
3654 static void process_db_full(struct work_struct *work)
3655 {
3656         struct adapter *adap;
3657
3658         adap = container_of(work, struct adapter, db_full_task);
3659
3660         drain_db_fifo(adap, dbfifo_drain_delay);
3661         enable_dbs(adap);
3662         notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
3663         t4_set_reg_field(adap, SGE_INT_ENABLE3,
3664                          DBFIFO_HP_INT | DBFIFO_LP_INT,
3665                          DBFIFO_HP_INT | DBFIFO_LP_INT);
3666 }
3667
3668 static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q)
3669 {
3670         u16 hw_pidx, hw_cidx;
3671         int ret;
3672
3673         spin_lock_irq(&q->db_lock);
3674         ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx);
3675         if (ret)
3676                 goto out;
3677         if (q->db_pidx != hw_pidx) {
3678                 u16 delta;
3679
3680                 if (q->db_pidx >= hw_pidx)
3681                         delta = q->db_pidx - hw_pidx;
3682                 else
3683                         delta = q->size - hw_pidx + q->db_pidx;
3684                 wmb();
3685                 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),
3686                              QID(q->cntxt_id) | PIDX(delta));
3687         }
3688 out:
3689         q->db_disabled = 0;
3690         q->db_pidx_inc = 0;
3691         spin_unlock_irq(&q->db_lock);
3692         if (ret)
3693                 CH_WARN(adap, "DB drop recovery failed.\n");
3694 }
3695 static void recover_all_queues(struct adapter *adap)
3696 {
3697         int i;
3698
3699         for_each_ethrxq(&adap->sge, i)
3700                 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q);
3701         for_each_ofldrxq(&adap->sge, i)
3702                 sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q);
3703         for_each_port(adap, i)
3704                 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q);
3705 }
3706
3707 static void process_db_drop(struct work_struct *work)
3708 {
3709         struct adapter *adap;
3710
3711         adap = container_of(work, struct adapter, db_drop_task);
3712
3713         if (is_t4(adap->params.chip)) {
3714                 drain_db_fifo(adap, dbfifo_drain_delay);
3715                 notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP);
3716                 drain_db_fifo(adap, dbfifo_drain_delay);
3717                 recover_all_queues(adap);
3718                 drain_db_fifo(adap, dbfifo_drain_delay);
3719                 enable_dbs(adap);
3720                 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
3721         } else {
3722                 u32 dropped_db = t4_read_reg(adap, 0x010ac);
3723                 u16 qid = (dropped_db >> 15) & 0x1ffff;
3724                 u16 pidx_inc = dropped_db & 0x1fff;
3725                 unsigned int s_qpp;
3726                 unsigned short udb_density;
3727                 unsigned long qpshift;
3728                 int page;
3729                 u32 udb;
3730
3731                 dev_warn(adap->pdev_dev,
3732                          "Dropped DB 0x%x qid %d bar2 %d coalesce %d pidx %d\n",
3733                          dropped_db, qid,
3734                          (dropped_db >> 14) & 1,
3735                          (dropped_db >> 13) & 1,
3736                          pidx_inc);
3737
3738                 drain_db_fifo(adap, 1);
3739
3740                 s_qpp = QUEUESPERPAGEPF1 * adap->fn;
3741                 udb_density = 1 << QUEUESPERPAGEPF0_GET(t4_read_reg(adap,
3742                                 SGE_EGRESS_QUEUES_PER_PAGE_PF) >> s_qpp);
3743                 qpshift = PAGE_SHIFT - ilog2(udb_density);
3744                 udb = qid << qpshift;
3745                 udb &= PAGE_MASK;
3746                 page = udb / PAGE_SIZE;
3747                 udb += (qid - (page * udb_density)) * 128;
3748
3749                 writel(PIDX(pidx_inc),  adap->bar2 + udb + 8);
3750
3751                 /* Re-enable BAR2 WC */
3752                 t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15);
3753         }
3754
3755         t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_DROPPED_DB, 0);
3756 }
3757
3758 void t4_db_full(struct adapter *adap)
3759 {
3760         if (is_t4(adap->params.chip)) {
3761                 disable_dbs(adap);
3762                 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
3763                 t4_set_reg_field(adap, SGE_INT_ENABLE3,
3764                                  DBFIFO_HP_INT | DBFIFO_LP_INT, 0);
3765                 queue_work(workq, &adap->db_full_task);
3766         }
3767 }
3768
3769 void t4_db_dropped(struct adapter *adap)
3770 {
3771         if (is_t4(adap->params.chip)) {
3772                 disable_dbs(adap);
3773                 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
3774         }
3775         queue_work(workq, &adap->db_drop_task);
3776 }
3777
3778 static void uld_attach(struct adapter *adap, unsigned int uld)
3779 {
3780         void *handle;
3781         struct cxgb4_lld_info lli;
3782         unsigned short i;
3783
3784         lli.pdev = adap->pdev;
3785         lli.l2t = adap->l2t;
3786         lli.tids = &adap->tids;
3787         lli.ports = adap->port;
3788         lli.vr = &adap->vres;
3789         lli.mtus = adap->params.mtus;
3790         if (uld == CXGB4_ULD_RDMA) {
3791                 lli.rxq_ids = adap->sge.rdma_rxq;
3792                 lli.nrxq = adap->sge.rdmaqs;
3793         } else if (uld == CXGB4_ULD_ISCSI) {
3794                 lli.rxq_ids = adap->sge.ofld_rxq;
3795                 lli.nrxq = adap->sge.ofldqsets;
3796         }
3797         lli.ntxq = adap->sge.ofldqsets;
3798         lli.nchan = adap->params.nports;
3799         lli.nports = adap->params.nports;
3800         lli.wr_cred = adap->params.ofldq_wr_cred;
3801         lli.adapter_type = adap->params.chip;
3802         lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2));
3803         lli.udb_density = 1 << QUEUESPERPAGEPF0_GET(
3804                         t4_read_reg(adap, SGE_EGRESS_QUEUES_PER_PAGE_PF) >>
3805                         (adap->fn * 4));
3806         lli.ucq_density = 1 << QUEUESPERPAGEPF0_GET(
3807                         t4_read_reg(adap, SGE_INGRESS_QUEUES_PER_PAGE_PF) >>
3808                         (adap->fn * 4));
3809         lli.filt_mode = adap->params.tp.vlan_pri_map;
3810         /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */
3811         for (i = 0; i < NCHAN; i++)
3812                 lli.tx_modq[i] = i;
3813         lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS);
3814         lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL);
3815         lli.fw_vers = adap->params.fw_vers;
3816         lli.dbfifo_int_thresh = dbfifo_int_thresh;
3817         lli.sge_pktshift = adap->sge.pktshift;
3818         lli.enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN;
3819         lli.ulptx_memwrite_dsgl = adap->params.ulptx_memwrite_dsgl;
3820
3821         handle = ulds[uld].add(&lli);
3822         if (IS_ERR(handle)) {
3823                 dev_warn(adap->pdev_dev,
3824                          "could not attach to the %s driver, error %ld\n",
3825                          uld_str[uld], PTR_ERR(handle));
3826                 return;
3827         }
3828
3829         adap->uld_handle[uld] = handle;
3830
3831         if (!netevent_registered) {
3832                 register_netevent_notifier(&cxgb4_netevent_nb);
3833                 netevent_registered = true;
3834         }
3835
3836         if (adap->flags & FULL_INIT_DONE)
3837                 ulds[uld].state_change(handle, CXGB4_STATE_UP);
3838 }
3839
3840 static void attach_ulds(struct adapter *adap)
3841 {
3842         unsigned int i;
3843
3844         spin_lock(&adap_rcu_lock);
3845         list_add_tail_rcu(&adap->rcu_node, &adap_rcu_list);
3846         spin_unlock(&adap_rcu_lock);
3847
3848         mutex_lock(&uld_mutex);
3849         list_add_tail(&adap->list_node, &adapter_list);
3850         for (i = 0; i < CXGB4_ULD_MAX; i++)
3851                 if (ulds[i].add)
3852                         uld_attach(adap, i);
3853         mutex_unlock(&uld_mutex);
3854 }
3855
3856 static void detach_ulds(struct adapter *adap)
3857 {
3858         unsigned int i;
3859
3860         mutex_lock(&uld_mutex);
3861         list_del(&adap->list_node);
3862         for (i = 0; i < CXGB4_ULD_MAX; i++)
3863                 if (adap->uld_handle[i]) {
3864                         ulds[i].state_change(adap->uld_handle[i],
3865                                              CXGB4_STATE_DETACH);
3866                         adap->uld_handle[i] = NULL;
3867                 }
3868         if (netevent_registered && list_empty(&adapter_list)) {
3869                 unregister_netevent_notifier(&cxgb4_netevent_nb);
3870                 netevent_registered = false;
3871         }
3872         mutex_unlock(&uld_mutex);
3873
3874         spin_lock(&adap_rcu_lock);
3875         list_del_rcu(&adap->rcu_node);
3876         spin_unlock(&adap_rcu_lock);
3877 }
3878
3879 static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
3880 {
3881         unsigned int i;
3882
3883         mutex_lock(&uld_mutex);
3884         for (i = 0; i < CXGB4_ULD_MAX; i++)
3885                 if (adap->uld_handle[i])
3886                         ulds[i].state_change(adap->uld_handle[i], new_state);
3887         mutex_unlock(&uld_mutex);
3888 }
3889
3890 /**
3891  *      cxgb4_register_uld - register an upper-layer driver
3892  *      @type: the ULD type
3893  *      @p: the ULD methods
3894  *
3895  *      Registers an upper-layer driver with this driver and notifies the ULD
3896  *      about any presently available devices that support its type.  Returns
3897  *      %-EBUSY if a ULD of the same type is already registered.
3898  */
3899 int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
3900 {
3901         int ret = 0;
3902         struct adapter *adap;
3903
3904         if (type >= CXGB4_ULD_MAX)
3905                 return -EINVAL;
3906         mutex_lock(&uld_mutex);
3907         if (ulds[type].add) {
3908                 ret = -EBUSY;
3909                 goto out;
3910         }
3911         ulds[type] = *p;
3912         list_for_each_entry(adap, &adapter_list, list_node)
3913                 uld_attach(adap, type);
3914 out:    mutex_unlock(&uld_mutex);
3915         return ret;
3916 }
3917 EXPORT_SYMBOL(cxgb4_register_uld);
3918
3919 /**
3920  *      cxgb4_unregister_uld - unregister an upper-layer driver
3921  *      @type: the ULD type
3922  *
3923  *      Unregisters an existing upper-layer driver.
3924  */
3925 int cxgb4_unregister_uld(enum cxgb4_uld type)
3926 {
3927         struct adapter *adap;
3928
3929         if (type >= CXGB4_ULD_MAX)
3930                 return -EINVAL;
3931         mutex_lock(&uld_mutex);
3932         list_for_each_entry(adap, &adapter_list, list_node)
3933                 adap->uld_handle[type] = NULL;
3934         ulds[type].add = NULL;
3935         mutex_unlock(&uld_mutex);
3936         return 0;
3937 }
3938 EXPORT_SYMBOL(cxgb4_unregister_uld);
3939
3940 /* Check if netdev on which event is occured belongs to us or not. Return
3941  * suceess (1) if it belongs otherwise failure (0).
3942  */
3943 static int cxgb4_netdev(struct net_device *netdev)
3944 {
3945         struct adapter *adap;
3946         int i;
3947
3948         spin_lock(&adap_rcu_lock);
3949         list_for_each_entry_rcu(adap, &adap_rcu_list, rcu_node)
3950                 for (i = 0; i < MAX_NPORTS; i++)
3951                         if (adap->port[i] == netdev) {
3952                                 spin_unlock(&adap_rcu_lock);
3953                                 return 1;
3954                         }
3955         spin_unlock(&adap_rcu_lock);
3956         return 0;
3957 }
3958
3959 static int clip_add(struct net_device *event_dev, struct inet6_ifaddr *ifa,
3960                     unsigned long event)
3961 {
3962         int ret = NOTIFY_DONE;
3963
3964         rcu_read_lock();
3965         if (cxgb4_netdev(event_dev)) {
3966                 switch (event) {
3967                 case NETDEV_UP:
3968                         ret = cxgb4_clip_get(event_dev,
3969                                 (const struct in6_addr *)ifa->addr.s6_addr);
3970                         if (ret < 0) {
3971                                 rcu_read_unlock();
3972                                 return ret;
3973                         }
3974                         ret = NOTIFY_OK;
3975                         break;
3976                 case NETDEV_DOWN:
3977                         cxgb4_clip_release(event_dev,
3978                                 (const struct in6_addr *)ifa->addr.s6_addr);
3979                         ret = NOTIFY_OK;
3980                         break;
3981                 default:
3982                         break;
3983                 }
3984         }
3985         rcu_read_unlock();
3986         return ret;
3987 }
3988
3989 static int cxgb4_inet6addr_handler(struct notifier_block *this,
3990                 unsigned long event, void *data)
3991 {
3992         struct inet6_ifaddr *ifa = data;
3993         struct net_device *event_dev;
3994         int ret = NOTIFY_DONE;
3995         struct bonding *bond = netdev_priv(ifa->idev->dev);
3996         struct list_head *iter;
3997         struct slave *slave;
3998         struct pci_dev *first_pdev = NULL;
3999
4000         if (ifa->idev->dev->priv_flags & IFF_802_1Q_VLAN) {
4001                 event_dev = vlan_dev_real_dev(ifa->idev->dev);
4002                 ret = clip_add(event_dev, ifa, event);
4003         } else if (ifa->idev->dev->flags & IFF_MASTER) {
4004                 /* It is possible that two different adapters are bonded in one
4005                  * bond. We need to find such different adapters and add clip
4006                  * in all of them only once.
4007                  */
4008                 read_lock(&bond->lock);
4009                 bond_for_each_slave(bond, slave, iter) {
4010                         if (!first_pdev) {
4011                                 ret = clip_add(slave->dev, ifa, event);
4012                                 /* If clip_add is success then only initialize
4013                                  * first_pdev since it means it is our device
4014                                  */
4015                                 if (ret == NOTIFY_OK)
4016                                         first_pdev = to_pci_dev(
4017                                                         slave->dev->dev.parent);
4018                         } else if (first_pdev !=
4019                                    to_pci_dev(slave->dev->dev.parent))
4020                                         ret = clip_add(slave->dev, ifa, event);
4021                 }
4022                 read_unlock(&bond->lock);
4023         } else
4024                 ret = clip_add(ifa->idev->dev, ifa, event);
4025
4026         return ret;
4027 }
4028
4029 static struct notifier_block cxgb4_inet6addr_notifier = {
4030         .notifier_call = cxgb4_inet6addr_handler
4031 };
4032
4033 /* Retrieves IPv6 addresses from a root device (bond, vlan) associated with
4034  * a physical device.
4035  * The physical device reference is needed to send the actul CLIP command.
4036  */
4037 static int update_dev_clip(struct net_device *root_dev, struct net_device *dev)
4038 {
4039         struct inet6_dev *idev = NULL;
4040         struct inet6_ifaddr *ifa;
4041         int ret = 0;
4042
4043         idev = __in6_dev_get(root_dev);
4044         if (!idev)
4045                 return ret;
4046
4047         read_lock_bh(&idev->lock);
4048         list_for_each_entry(ifa, &idev->addr_list, if_list) {
4049                 ret = cxgb4_clip_get(dev,
4050                                 (const struct in6_addr *)ifa->addr.s6_addr);
4051                 if (ret < 0)
4052                         break;
4053         }
4054         read_unlock_bh(&idev->lock);
4055
4056         return ret;
4057 }
4058
4059 static int update_root_dev_clip(struct net_device *dev)
4060 {
4061         struct net_device *root_dev = NULL;
4062         int i, ret = 0;
4063
4064         /* First populate the real net device's IPv6 addresses */
4065         ret = update_dev_clip(dev, dev);
4066         if (ret)
4067                 return ret;
4068
4069         /* Parse all bond and vlan devices layered on top of the physical dev */
4070         for (i = 0; i < VLAN_N_VID; i++) {
4071                 root_dev = __vlan_find_dev_deep(dev, htons(ETH_P_8021Q), i);
4072                 if (!root_dev)
4073                         continue;
4074
4075                 ret = update_dev_clip(root_dev, dev);
4076                 if (ret)
4077                         break;
4078         }
4079         return ret;
4080 }
4081
4082 static void update_clip(const struct adapter *adap)
4083 {
4084         int i;
4085         struct net_device *dev;
4086         int ret;
4087
4088         rcu_read_lock();
4089
4090         for (i = 0; i < MAX_NPORTS; i++) {
4091                 dev = adap->port[i];
4092                 ret = 0;
4093
4094                 if (dev)
4095                         ret = update_root_dev_clip(dev);
4096
4097                 if (ret < 0)
4098                         break;
4099         }
4100         rcu_read_unlock();
4101 }
4102
4103 /**
4104  *      cxgb_up - enable the adapter
4105  *      @adap: adapter being enabled
4106  *
4107  *      Called when the first port is enabled, this function performs the
4108  *      actions necessary to make an adapter operational, such as completing
4109  *      the initialization of HW modules, and enabling interrupts.
4110  *
4111  *      Must be called with the rtnl lock held.
4112  */
4113 static int cxgb_up(struct adapter *adap)
4114 {
4115         int err;
4116
4117         err = setup_sge_queues(adap);
4118         if (err)
4119                 goto out;
4120         err = setup_rss(adap);
4121         if (err)
4122                 goto freeq;
4123
4124         if (adap->flags & USING_MSIX) {
4125                 name_msix_vecs(adap);
4126                 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
4127                                   adap->msix_info[0].desc, adap);
4128                 if (err)
4129                         goto irq_err;
4130
4131                 err = request_msix_queue_irqs(adap);
4132                 if (err) {
4133                         free_irq(adap->msix_info[0].vec, adap);
4134                         goto irq_err;
4135                 }
4136         } else {
4137                 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
4138                                   (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
4139                                   adap->port[0]->name, adap);
4140                 if (err)
4141                         goto irq_err;
4142         }
4143         enable_rx(adap);
4144         t4_sge_start(adap);
4145         t4_intr_enable(adap);
4146         adap->flags |= FULL_INIT_DONE;
4147         notify_ulds(adap, CXGB4_STATE_UP);
4148         update_clip(adap);
4149  out:
4150         return err;
4151  irq_err:
4152         dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
4153  freeq:
4154         t4_free_sge_resources(adap);
4155         goto out;
4156 }
4157
4158 static void cxgb_down(struct adapter *adapter)
4159 {
4160         t4_intr_disable(adapter);
4161         cancel_work_sync(&adapter->tid_release_task);
4162         cancel_work_sync(&adapter->db_full_task);
4163         cancel_work_sync(&adapter->db_drop_task);
4164         adapter->tid_release_task_busy = false;
4165         adapter->tid_release_head = NULL;
4166
4167         if (adapter->flags & USING_MSIX) {
4168                 free_msix_queue_irqs(adapter);
4169                 free_irq(adapter->msix_info[0].vec, adapter);
4170         } else
4171                 free_irq(adapter->pdev->irq, adapter);
4172         quiesce_rx(adapter);
4173         t4_sge_stop(adapter);
4174         t4_free_sge_resources(adapter);
4175         adapter->flags &= ~FULL_INIT_DONE;
4176 }
4177
4178 /*
4179  * net_device operations
4180  */
4181 static int cxgb_open(struct net_device *dev)
4182 {
4183         int err;
4184         struct port_info *pi = netdev_priv(dev);
4185         struct adapter *adapter = pi->adapter;
4186
4187         netif_carrier_off(dev);
4188
4189         if (!(adapter->flags & FULL_INIT_DONE)) {
4190                 err = cxgb_up(adapter);
4191                 if (err < 0)
4192                         return err;
4193         }
4194
4195         err = link_start(dev);
4196         if (!err)
4197                 netif_tx_start_all_queues(dev);
4198         return err;
4199 }
4200
4201 static int cxgb_close(struct net_device *dev)
4202 {
4203         struct port_info *pi = netdev_priv(dev);
4204         struct adapter *adapter = pi->adapter;
4205
4206         netif_tx_stop_all_queues(dev);
4207         netif_carrier_off(dev);
4208         return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
4209 }
4210
4211 /* Return an error number if the indicated filter isn't writable ...
4212  */
4213 static int writable_filter(struct filter_entry *f)
4214 {
4215         if (f->locked)
4216                 return -EPERM;
4217         if (f->pending)
4218                 return -EBUSY;
4219
4220         return 0;
4221 }
4222
4223 /* Delete the filter at the specified index (if valid).  The checks for all
4224  * the common problems with doing this like the filter being locked, currently
4225  * pending in another operation, etc.
4226  */
4227 static int delete_filter(struct adapter *adapter, unsigned int fidx)
4228 {
4229         struct filter_entry *f;
4230         int ret;
4231
4232         if (fidx >= adapter->tids.nftids + adapter->tids.nsftids)
4233                 return -EINVAL;
4234
4235         f = &adapter->tids.ftid_tab[fidx];
4236         ret = writable_filter(f);
4237         if (ret)
4238                 return ret;
4239         if (f->valid)
4240                 return del_filter_wr(adapter, fidx);
4241
4242         return 0;
4243 }
4244
4245 int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid,
4246                 __be32 sip, __be16 sport, __be16 vlan,
4247                 unsigned int queue, unsigned char port, unsigned char mask)
4248 {
4249         int ret;
4250         struct filter_entry *f;
4251         struct adapter *adap;
4252         int i;
4253         u8 *val;
4254
4255         adap = netdev2adap(dev);
4256
4257         /* Adjust stid to correct filter index */
4258         stid -= adap->tids.sftid_base;
4259         stid += adap->tids.nftids;
4260
4261         /* Check to make sure the filter requested is writable ...
4262          */
4263         f = &adap->tids.ftid_tab[stid];
4264         ret = writable_filter(f);
4265         if (ret)
4266                 return ret;
4267
4268         /* Clear out any old resources being used by the filter before
4269          * we start constructing the new filter.
4270          */
4271         if (f->valid)
4272                 clear_filter(adap, f);
4273
4274         /* Clear out filter specifications */
4275         memset(&f->fs, 0, sizeof(struct ch_filter_specification));
4276         f->fs.val.lport = cpu_to_be16(sport);
4277         f->fs.mask.lport  = ~0;
4278         val = (u8 *)&sip;
4279         if ((val[0] | val[1] | val[2] | val[3]) != 0) {
4280                 for (i = 0; i < 4; i++) {
4281                         f->fs.val.lip[i] = val[i];
4282                         f->fs.mask.lip[i] = ~0;
4283                 }
4284                 if (adap->params.tp.vlan_pri_map & F_PORT) {
4285                         f->fs.val.iport = port;
4286                         f->fs.mask.iport = mask;
4287                 }
4288         }
4289
4290         if (adap->params.tp.vlan_pri_map & F_PROTOCOL) {
4291                 f->fs.val.proto = IPPROTO_TCP;
4292                 f->fs.mask.proto = ~0;
4293         }
4294
4295         f->fs.dirsteer = 1;
4296         f->fs.iq = queue;
4297         /* Mark filter as locked */
4298         f->locked = 1;
4299         f->fs.rpttid = 1;
4300
4301         ret = set_filter_wr(adap, stid);
4302         if (ret) {
4303                 clear_filter(adap, f);
4304                 return ret;
4305         }
4306
4307         return 0;
4308 }
4309 EXPORT_SYMBOL(cxgb4_create_server_filter);
4310
4311 int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid,
4312                 unsigned int queue, bool ipv6)
4313 {
4314         int ret;
4315         struct filter_entry *f;
4316         struct adapter *adap;
4317
4318         adap = netdev2adap(dev);
4319
4320         /* Adjust stid to correct filter index */
4321         stid -= adap->tids.sftid_base;
4322         stid += adap->tids.nftids;
4323
4324         f = &adap->tids.ftid_tab[stid];
4325         /* Unlock the filter */
4326         f->locked = 0;
4327
4328         ret = delete_filter(adap, stid);
4329         if (ret)
4330                 return ret;
4331
4332         return 0;
4333 }
4334 EXPORT_SYMBOL(cxgb4_remove_server_filter);
4335
4336 static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
4337                                                 struct rtnl_link_stats64 *ns)
4338 {
4339         struct port_stats stats;
4340         struct port_info *p = netdev_priv(dev);
4341         struct adapter *adapter = p->adapter;
4342
4343         /* Block retrieving statistics during EEH error
4344          * recovery. Otherwise, the recovery might fail
4345          * and the PCI device will be removed permanently
4346          */
4347         spin_lock(&adapter->stats_lock);
4348         if (!netif_device_present(dev)) {
4349                 spin_unlock(&adapter->stats_lock);
4350                 return ns;
4351         }
4352         t4_get_port_stats(adapter, p->tx_chan, &stats);
4353         spin_unlock(&adapter->stats_lock);
4354
4355         ns->tx_bytes   = stats.tx_octets;
4356         ns->tx_packets = stats.tx_frames;
4357         ns->rx_bytes   = stats.rx_octets;
4358         ns->rx_packets = stats.rx_frames;
4359         ns->multicast  = stats.rx_mcast_frames;
4360
4361         /* detailed rx_errors */
4362         ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
4363                                stats.rx_runt;
4364         ns->rx_over_errors   = 0;
4365         ns->rx_crc_errors    = stats.rx_fcs_err;
4366         ns->rx_frame_errors  = stats.rx_symbol_err;
4367         ns->rx_fifo_errors   = stats.rx_ovflow0 + stats.rx_ovflow1 +
4368                                stats.rx_ovflow2 + stats.rx_ovflow3 +
4369                                stats.rx_trunc0 + stats.rx_trunc1 +
4370                                stats.rx_trunc2 + stats.rx_trunc3;
4371         ns->rx_missed_errors = 0;
4372
4373         /* detailed tx_errors */
4374         ns->tx_aborted_errors   = 0;
4375         ns->tx_carrier_errors   = 0;
4376         ns->tx_fifo_errors      = 0;
4377         ns->tx_heartbeat_errors = 0;
4378         ns->tx_window_errors    = 0;
4379
4380         ns->tx_errors = stats.tx_error_frames;
4381         ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
4382                 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
4383         return ns;
4384 }
4385
4386 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
4387 {
4388         unsigned int mbox;
4389         int ret = 0, prtad, devad;
4390         struct port_info *pi = netdev_priv(dev);
4391         struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
4392
4393         switch (cmd) {
4394         case SIOCGMIIPHY:
4395                 if (pi->mdio_addr < 0)
4396                         return -EOPNOTSUPP;
4397                 data->phy_id = pi->mdio_addr;
4398                 break;
4399         case SIOCGMIIREG:
4400         case SIOCSMIIREG:
4401                 if (mdio_phy_id_is_c45(data->phy_id)) {
4402                         prtad = mdio_phy_id_prtad(data->phy_id);
4403                         devad = mdio_phy_id_devad(data->phy_id);
4404                 } else if (data->phy_id < 32) {
4405                         prtad = data->phy_id;
4406                         devad = 0;
4407                         data->reg_num &= 0x1f;
4408                 } else
4409                         return -EINVAL;
4410
4411                 mbox = pi->adapter->fn;
4412                 if (cmd == SIOCGMIIREG)
4413                         ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
4414                                          data->reg_num, &data->val_out);
4415                 else
4416                         ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
4417                                          data->reg_num, data->val_in);
4418                 break;
4419         default:
4420                 return -EOPNOTSUPP;
4421         }
4422         return ret;
4423 }
4424
4425 static void cxgb_set_rxmode(struct net_device *dev)
4426 {
4427         /* unfortunately we can't return errors to the stack */
4428         set_rxmode(dev, -1, false);
4429 }
4430
4431 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
4432 {
4433         int ret;
4434         struct port_info *pi = netdev_priv(dev);
4435
4436         if (new_mtu < 81 || new_mtu > MAX_MTU)         /* accommodate SACK */
4437                 return -EINVAL;
4438         ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
4439                             -1, -1, -1, true);
4440         if (!ret)
4441                 dev->mtu = new_mtu;
4442         return ret;
4443 }
4444
4445 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
4446 {
4447         int ret;
4448         struct sockaddr *addr = p;
4449         struct port_info *pi = netdev_priv(dev);
4450
4451         if (!is_valid_ether_addr(addr->sa_data))
4452                 return -EADDRNOTAVAIL;
4453
4454         ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
4455                             pi->xact_addr_filt, addr->sa_data, true, true);
4456         if (ret < 0)
4457                 return ret;
4458
4459         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4460         pi->xact_addr_filt = ret;
4461         return 0;
4462 }
4463
4464 #ifdef CONFIG_NET_POLL_CONTROLLER
4465 static void cxgb_netpoll(struct net_device *dev)
4466 {
4467         struct port_info *pi = netdev_priv(dev);
4468         struct adapter *adap = pi->adapter;
4469
4470         if (adap->flags & USING_MSIX) {
4471                 int i;
4472                 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
4473
4474                 for (i = pi->nqsets; i; i--, rx++)
4475                         t4_sge_intr_msix(0, &rx->rspq);
4476         } else
4477                 t4_intr_handler(adap)(0, adap);
4478 }
4479 #endif
4480
4481 static const struct net_device_ops cxgb4_netdev_ops = {
4482         .ndo_open             = cxgb_open,
4483         .ndo_stop             = cxgb_close,
4484         .ndo_start_xmit       = t4_eth_xmit,
4485         .ndo_get_stats64      = cxgb_get_stats,
4486         .ndo_set_rx_mode      = cxgb_set_rxmode,
4487         .ndo_set_mac_address  = cxgb_set_mac_addr,
4488         .ndo_set_features     = cxgb_set_features,
4489         .ndo_validate_addr    = eth_validate_addr,
4490         .ndo_do_ioctl         = cxgb_ioctl,
4491         .ndo_change_mtu       = cxgb_change_mtu,
4492 #ifdef CONFIG_NET_POLL_CONTROLLER
4493         .ndo_poll_controller  = cxgb_netpoll,
4494 #endif
4495 };
4496
4497 void t4_fatal_err(struct adapter *adap)
4498 {
4499         t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0);
4500         t4_intr_disable(adap);
4501         dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
4502 }
4503
4504 static void setup_memwin(struct adapter *adap)
4505 {
4506         u32 bar0, mem_win0_base, mem_win1_base, mem_win2_base;
4507
4508         bar0 = pci_resource_start(adap->pdev, 0);  /* truncation intentional */
4509         if (is_t4(adap->params.chip)) {
4510                 mem_win0_base = bar0 + MEMWIN0_BASE;
4511                 mem_win1_base = bar0 + MEMWIN1_BASE;
4512                 mem_win2_base = bar0 + MEMWIN2_BASE;
4513         } else {
4514                 /* For T5, only relative offset inside the PCIe BAR is passed */
4515                 mem_win0_base = MEMWIN0_BASE;
4516                 mem_win1_base = MEMWIN1_BASE_T5;
4517                 mem_win2_base = MEMWIN2_BASE_T5;
4518         }
4519         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0),
4520                      mem_win0_base | BIR(0) |
4521                      WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
4522         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1),
4523                      mem_win1_base | BIR(0) |
4524                      WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
4525         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2),
4526                      mem_win2_base | BIR(0) |
4527                      WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
4528 }
4529
4530 static void setup_memwin_rdma(struct adapter *adap)
4531 {
4532         if (adap->vres.ocq.size) {
4533                 unsigned int start, sz_kb;
4534
4535                 start = pci_resource_start(adap->pdev, 2) +
4536                         OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
4537                 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
4538                 t4_write_reg(adap,
4539                              PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 3),
4540                              start | BIR(1) | WINDOW(ilog2(sz_kb)));
4541                 t4_write_reg(adap,
4542                              PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3),
4543                              adap->vres.ocq.start);
4544                 t4_read_reg(adap,
4545                             PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3));
4546         }
4547 }
4548
4549 static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
4550 {
4551         u32 v;
4552         int ret;
4553
4554         /* get device capabilities */
4555         memset(c, 0, sizeof(*c));
4556         c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4557                                FW_CMD_REQUEST | FW_CMD_READ);
4558         c->cfvalid_to_len16 = htonl(FW_LEN16(*c));
4559         ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
4560         if (ret < 0)
4561                 return ret;
4562
4563         /* select capabilities we'll be using */
4564         if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
4565                 if (!vf_acls)
4566                         c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
4567                 else
4568                         c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
4569         } else if (vf_acls) {
4570                 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
4571                 return ret;
4572         }
4573         c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4574                                FW_CMD_REQUEST | FW_CMD_WRITE);
4575         ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
4576         if (ret < 0)
4577                 return ret;
4578
4579         ret = t4_config_glbl_rss(adap, adap->fn,
4580                                  FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
4581                                  FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
4582                                  FW_RSS_GLB_CONFIG_CMD_TNLALLLKP);
4583         if (ret < 0)
4584                 return ret;
4585
4586         ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ,
4587                           0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
4588         if (ret < 0)
4589                 return ret;
4590
4591         t4_sge_init(adap);
4592
4593         /* tweak some settings */
4594         t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849);
4595         t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12));
4596         t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG);
4597         v = t4_read_reg(adap, TP_PIO_DATA);
4598         t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR);
4599
4600         /* first 4 Tx modulation queues point to consecutive Tx channels */
4601         adap->params.tp.tx_modq_map = 0xE4;
4602         t4_write_reg(adap, A_TP_TX_MOD_QUEUE_REQ_MAP,
4603                      V_TX_MOD_QUEUE_REQ_MAP(adap->params.tp.tx_modq_map));
4604
4605         /* associate each Tx modulation queue with consecutive Tx channels */
4606         v = 0x84218421;
4607         t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
4608                           &v, 1, A_TP_TX_SCHED_HDR);
4609         t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
4610                           &v, 1, A_TP_TX_SCHED_FIFO);
4611         t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
4612                           &v, 1, A_TP_TX_SCHED_PCMD);
4613
4614 #define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */
4615         if (is_offload(adap)) {
4616                 t4_write_reg(adap, A_TP_TX_MOD_QUEUE_WEIGHT0,
4617                              V_TX_MODQ_WEIGHT0(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4618                              V_TX_MODQ_WEIGHT1(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4619                              V_TX_MODQ_WEIGHT2(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4620                              V_TX_MODQ_WEIGHT3(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
4621                 t4_write_reg(adap, A_TP_TX_MOD_CHANNEL_WEIGHT,
4622                              V_TX_MODQ_WEIGHT0(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4623                              V_TX_MODQ_WEIGHT1(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4624                              V_TX_MODQ_WEIGHT2(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4625                              V_TX_MODQ_WEIGHT3(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
4626         }
4627
4628         /* get basic stuff going */
4629         return t4_early_init(adap, adap->fn);
4630 }
4631
4632 /*
4633  * Max # of ATIDs.  The absolute HW max is 16K but we keep it lower.
4634  */
4635 #define MAX_ATIDS 8192U
4636
4637 /*
4638  * Phase 0 of initialization: contact FW, obtain config, perform basic init.
4639  *
4640  * If the firmware we're dealing with has Configuration File support, then
4641  * we use that to perform all configuration
4642  */
4643
4644 /*
4645  * Tweak configuration based on module parameters, etc.  Most of these have
4646  * defaults assigned to them by Firmware Configuration Files (if we're using
4647  * them) but need to be explicitly set if we're using hard-coded
4648  * initialization.  But even in the case of using Firmware Configuration
4649  * Files, we'd like to expose the ability to change these via module
4650  * parameters so these are essentially common tweaks/settings for
4651  * Configuration Files and hard-coded initialization ...
4652  */
4653 static int adap_init0_tweaks(struct adapter *adapter)
4654 {
4655         /*
4656          * Fix up various Host-Dependent Parameters like Page Size, Cache
4657          * Line Size, etc.  The firmware default is for a 4KB Page Size and
4658          * 64B Cache Line Size ...
4659          */
4660         t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES);
4661
4662         /*
4663          * Process module parameters which affect early initialization.
4664          */
4665         if (rx_dma_offset != 2 && rx_dma_offset != 0) {
4666                 dev_err(&adapter->pdev->dev,
4667                         "Ignoring illegal rx_dma_offset=%d, using 2\n",
4668                         rx_dma_offset);
4669                 rx_dma_offset = 2;
4670         }
4671         t4_set_reg_field(adapter, SGE_CONTROL,
4672                          PKTSHIFT_MASK,
4673                          PKTSHIFT(rx_dma_offset));
4674
4675         /*
4676          * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux
4677          * adds the pseudo header itself.
4678          */
4679         t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG,
4680                                CSUM_HAS_PSEUDO_HDR, 0);
4681
4682         return 0;
4683 }
4684
4685 /*
4686  * Attempt to initialize the adapter via a Firmware Configuration File.
4687  */
4688 static int adap_init0_config(struct adapter *adapter, int reset)
4689 {
4690         struct fw_caps_config_cmd caps_cmd;
4691         const struct firmware *cf;
4692         unsigned long mtype = 0, maddr = 0;
4693         u32 finiver, finicsum, cfcsum;
4694         int ret;
4695         int config_issued = 0;
4696         char *fw_config_file, fw_config_file_path[256];
4697         char *config_name = NULL;
4698
4699         /*
4700          * Reset device if necessary.
4701          */
4702         if (reset) {
4703                 ret = t4_fw_reset(adapter, adapter->mbox,
4704                                   PIORSTMODE | PIORST);
4705                 if (ret < 0)
4706                         goto bye;
4707         }
4708
4709         /*
4710          * If we have a T4 configuration file under /lib/firmware/cxgb4/,
4711          * then use that.  Otherwise, use the configuration file stored
4712          * in the adapter flash ...
4713          */
4714         switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) {
4715         case CHELSIO_T4:
4716                 fw_config_file = FW4_CFNAME;
4717                 break;
4718         case CHELSIO_T5:
4719                 fw_config_file = FW5_CFNAME;
4720                 break;
4721         default:
4722                 dev_err(adapter->pdev_dev, "Device %d is not supported\n",
4723                        adapter->pdev->device);
4724                 ret = -EINVAL;
4725                 goto bye;
4726         }
4727
4728         ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev);
4729         if (ret < 0) {
4730                 config_name = "On FLASH";
4731                 mtype = FW_MEMTYPE_CF_FLASH;
4732                 maddr = t4_flash_cfg_addr(adapter);
4733         } else {
4734                 u32 params[7], val[7];
4735
4736                 sprintf(fw_config_file_path,
4737                         "/lib/firmware/%s", fw_config_file);
4738                 config_name = fw_config_file_path;
4739
4740                 if (cf->size >= FLASH_CFG_MAX_SIZE)
4741                         ret = -ENOMEM;
4742                 else {
4743                         params[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
4744                              FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF));
4745                         ret = t4_query_params(adapter, adapter->mbox,
4746                                               adapter->fn, 0, 1, params, val);
4747                         if (ret == 0) {
4748                                 /*
4749                                  * For t4_memory_write() below addresses and
4750                                  * sizes have to be in terms of multiples of 4
4751                                  * bytes.  So, if the Configuration File isn't
4752                                  * a multiple of 4 bytes in length we'll have
4753                                  * to write that out separately since we can't
4754                                  * guarantee that the bytes following the
4755                                  * residual byte in the buffer returned by
4756                                  * request_firmware() are zeroed out ...
4757                                  */
4758                                 size_t resid = cf->size & 0x3;
4759                                 size_t size = cf->size & ~0x3;
4760                                 __be32 *data = (__be32 *)cf->data;
4761
4762                                 mtype = FW_PARAMS_PARAM_Y_GET(val[0]);
4763                                 maddr = FW_PARAMS_PARAM_Z_GET(val[0]) << 16;
4764
4765                                 ret = t4_memory_write(adapter, mtype, maddr,
4766                                                       size, data);
4767                                 if (ret == 0 && resid != 0) {
4768                                         union {
4769                                                 __be32 word;
4770                                                 char buf[4];
4771                                         } last;
4772                                         int i;
4773
4774                                         last.word = data[size >> 2];
4775                                         for (i = resid; i < 4; i++)
4776                                                 last.buf[i] = 0;
4777                                         ret = t4_memory_write(adapter, mtype,
4778                                                               maddr + size,
4779                                                               4, &last.word);
4780                                 }
4781                         }
4782                 }
4783
4784                 release_firmware(cf);
4785                 if (ret)
4786                         goto bye;
4787         }
4788
4789         /*
4790          * Issue a Capability Configuration command to the firmware to get it
4791          * to parse the Configuration File.  We don't use t4_fw_config_file()
4792          * because we want the ability to modify various features after we've
4793          * processed the configuration file ...
4794          */
4795         memset(&caps_cmd, 0, sizeof(caps_cmd));
4796         caps_cmd.op_to_write =
4797                 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4798                       FW_CMD_REQUEST |
4799                       FW_CMD_READ);
4800         caps_cmd.cfvalid_to_len16 =
4801                 htonl(FW_CAPS_CONFIG_CMD_CFVALID |
4802                       FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
4803                       FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) |
4804                       FW_LEN16(caps_cmd));
4805         ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4806                          &caps_cmd);
4807
4808         /* If the CAPS_CONFIG failed with an ENOENT (for a Firmware
4809          * Configuration File in FLASH), our last gasp effort is to use the
4810          * Firmware Configuration File which is embedded in the firmware.  A
4811          * very few early versions of the firmware didn't have one embedded
4812          * but we can ignore those.
4813          */
4814         if (ret == -ENOENT) {
4815                 memset(&caps_cmd, 0, sizeof(caps_cmd));
4816                 caps_cmd.op_to_write =
4817                         htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4818                                         FW_CMD_REQUEST |
4819                                         FW_CMD_READ);
4820                 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
4821                 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd,
4822                                 sizeof(caps_cmd), &caps_cmd);
4823                 config_name = "Firmware Default";
4824         }
4825
4826         config_issued = 1;
4827         if (ret < 0)
4828                 goto bye;
4829
4830         finiver = ntohl(caps_cmd.finiver);
4831         finicsum = ntohl(caps_cmd.finicsum);
4832         cfcsum = ntohl(caps_cmd.cfcsum);
4833         if (finicsum != cfcsum)
4834                 dev_warn(adapter->pdev_dev, "Configuration File checksum "\
4835                          "mismatch: [fini] csum=%#x, computed csum=%#x\n",
4836                          finicsum, cfcsum);
4837
4838         /*
4839          * And now tell the firmware to use the configuration we just loaded.
4840          */
4841         caps_cmd.op_to_write =
4842                 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4843                       FW_CMD_REQUEST |
4844                       FW_CMD_WRITE);
4845         caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
4846         ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4847                          NULL);
4848         if (ret < 0)
4849                 goto bye;
4850
4851         /*
4852          * Tweak configuration based on system architecture, module
4853          * parameters, etc.
4854          */
4855         ret = adap_init0_tweaks(adapter);
4856         if (ret < 0)
4857                 goto bye;
4858
4859         /*
4860          * And finally tell the firmware to initialize itself using the
4861          * parameters from the Configuration File.
4862          */
4863         ret = t4_fw_initialize(adapter, adapter->mbox);
4864         if (ret < 0)
4865                 goto bye;
4866
4867         /*
4868          * Return successfully and note that we're operating with parameters
4869          * not supplied by the driver, rather than from hard-wired
4870          * initialization constants burried in the driver.
4871          */
4872         adapter->flags |= USING_SOFT_PARAMS;
4873         dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\
4874                  "Configuration File \"%s\", version %#x, computed checksum %#x\n",
4875                  config_name, finiver, cfcsum);
4876         return 0;
4877
4878         /*
4879          * Something bad happened.  Return the error ...  (If the "error"
4880          * is that there's no Configuration File on the adapter we don't
4881          * want to issue a warning since this is fairly common.)
4882          */
4883 bye:
4884         if (config_issued && ret != -ENOENT)
4885                 dev_warn(adapter->pdev_dev, "\"%s\" configuration file error %d\n",
4886                          config_name, -ret);
4887         return ret;
4888 }
4889
4890 /*
4891  * Attempt to initialize the adapter via hard-coded, driver supplied
4892  * parameters ...
4893  */
4894 static int adap_init0_no_config(struct adapter *adapter, int reset)
4895 {
4896         struct sge *s = &adapter->sge;
4897         struct fw_caps_config_cmd caps_cmd;
4898         u32 v;
4899         int i, ret;
4900
4901         /*
4902          * Reset device if necessary
4903          */
4904         if (reset) {
4905                 ret = t4_fw_reset(adapter, adapter->mbox,
4906                                   PIORSTMODE | PIORST);
4907                 if (ret < 0)
4908                         goto bye;
4909         }
4910
4911         /*
4912          * Get device capabilities and select which we'll be using.
4913          */
4914         memset(&caps_cmd, 0, sizeof(caps_cmd));
4915         caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4916                                      FW_CMD_REQUEST | FW_CMD_READ);
4917         caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
4918         ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4919                          &caps_cmd);
4920         if (ret < 0)
4921                 goto bye;
4922
4923         if (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
4924                 if (!vf_acls)
4925                         caps_cmd.niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
4926                 else
4927                         caps_cmd.niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
4928         } else if (vf_acls) {
4929                 dev_err(adapter->pdev_dev, "virtualization ACLs not supported");
4930                 goto bye;
4931         }
4932         caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4933                               FW_CMD_REQUEST | FW_CMD_WRITE);
4934         ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4935                          NULL);
4936         if (ret < 0)
4937                 goto bye;
4938
4939         /*
4940          * Tweak configuration based on system architecture, module
4941          * parameters, etc.
4942          */
4943         ret = adap_init0_tweaks(adapter);
4944         if (ret < 0)
4945                 goto bye;
4946
4947         /*
4948          * Select RSS Global Mode we want to use.  We use "Basic Virtual"
4949          * mode which maps each Virtual Interface to its own section of
4950          * the RSS Table and we turn on all map and hash enables ...
4951          */
4952         adapter->flags |= RSS_TNLALLLOOKUP;
4953         ret = t4_config_glbl_rss(adapter, adapter->mbox,
4954                                  FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
4955                                  FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
4956                                  FW_RSS_GLB_CONFIG_CMD_HASHTOEPLITZ |
4957                                  ((adapter->flags & RSS_TNLALLLOOKUP) ?
4958                                         FW_RSS_GLB_CONFIG_CMD_TNLALLLKP : 0));
4959         if (ret < 0)
4960                 goto bye;
4961
4962         /*
4963          * Set up our own fundamental resource provisioning ...
4964          */
4965         ret = t4_cfg_pfvf(adapter, adapter->mbox, adapter->fn, 0,
4966                           PFRES_NEQ, PFRES_NETHCTRL,
4967                           PFRES_NIQFLINT, PFRES_NIQ,
4968                           PFRES_TC, PFRES_NVI,
4969                           FW_PFVF_CMD_CMASK_MASK,
4970                           pfvfres_pmask(adapter, adapter->fn, 0),
4971                           PFRES_NEXACTF,
4972                           PFRES_R_CAPS, PFRES_WX_CAPS);
4973         if (ret < 0)
4974                 goto bye;
4975
4976         /*
4977          * Perform low level SGE initialization.  We need to do this before we
4978          * send the firmware the INITIALIZE command because that will cause
4979          * any other PF Drivers which are waiting for the Master
4980          * Initialization to proceed forward.
4981          */
4982         for (i = 0; i < SGE_NTIMERS - 1; i++)
4983                 s->timer_val[i] = min(intr_holdoff[i], MAX_SGE_TIMERVAL);
4984         s->timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL;
4985         s->counter_val[0] = 1;
4986         for (i = 1; i < SGE_NCOUNTERS; i++)
4987                 s->counter_val[i] = min(intr_cnt[i - 1],
4988                                         THRESHOLD_0_GET(THRESHOLD_0_MASK));
4989         t4_sge_init(adapter);
4990
4991 #ifdef CONFIG_PCI_IOV
4992         /*
4993          * Provision resource limits for Virtual Functions.  We currently
4994          * grant them all the same static resource limits except for the Port
4995          * Access Rights Mask which we're assigning based on the PF.  All of
4996          * the static provisioning stuff for both the PF and VF really needs
4997          * to be managed in a persistent manner for each device which the
4998          * firmware controls.
4999          */
5000         {
5001                 int pf, vf;
5002
5003                 for (pf = 0; pf < ARRAY_SIZE(num_vf); pf++) {
5004                         if (num_vf[pf] <= 0)
5005                                 continue;
5006
5007                         /* VF numbering starts at 1! */
5008                         for (vf = 1; vf <= num_vf[pf]; vf++) {
5009                                 ret = t4_cfg_pfvf(adapter, adapter->mbox,
5010                                                   pf, vf,
5011                                                   VFRES_NEQ, VFRES_NETHCTRL,
5012                                                   VFRES_NIQFLINT, VFRES_NIQ,
5013                                                   VFRES_TC, VFRES_NVI,
5014                                                   FW_PFVF_CMD_CMASK_MASK,
5015                                                   pfvfres_pmask(
5016                                                   adapter, pf, vf),
5017                                                   VFRES_NEXACTF,
5018                                                   VFRES_R_CAPS, VFRES_WX_CAPS);
5019                                 if (ret < 0)
5020                                         dev_warn(adapter->pdev_dev,
5021                                                  "failed to "\
5022                                                  "provision pf/vf=%d/%d; "
5023                                                  "err=%d\n", pf, vf, ret);
5024                         }
5025                 }
5026         }
5027 #endif
5028
5029         /*
5030          * Set up the default filter mode.  Later we'll want to implement this
5031          * via a firmware command, etc. ...  This needs to be done before the
5032          * firmare initialization command ...  If the selected set of fields
5033          * isn't equal to the default value, we'll need to make sure that the
5034          * field selections will fit in the 36-bit budget.
5035          */
5036         if (tp_vlan_pri_map != TP_VLAN_PRI_MAP_DEFAULT) {
5037                 int j, bits = 0;
5038
5039                 for (j = TP_VLAN_PRI_MAP_FIRST; j <= TP_VLAN_PRI_MAP_LAST; j++)
5040                         switch (tp_vlan_pri_map & (1 << j)) {
5041                         case 0:
5042                                 /* compressed filter field not enabled */
5043                                 break;
5044                         case FCOE_MASK:
5045                                 bits +=  1;
5046                                 break;
5047                         case PORT_MASK:
5048                                 bits +=  3;
5049                                 break;
5050                         case VNIC_ID_MASK:
5051                                 bits += 17;
5052                                 break;
5053                         case VLAN_MASK:
5054                                 bits += 17;
5055                                 break;
5056                         case TOS_MASK:
5057                                 bits +=  8;
5058                                 break;
5059                         case PROTOCOL_MASK:
5060                                 bits +=  8;
5061                                 break;
5062                         case ETHERTYPE_MASK:
5063                                 bits += 16;
5064                                 break;
5065                         case MACMATCH_MASK:
5066                                 bits +=  9;
5067                                 break;
5068                         case MPSHITTYPE_MASK:
5069                                 bits +=  3;
5070                                 break;
5071                         case FRAGMENTATION_MASK:
5072                                 bits +=  1;
5073                                 break;
5074                         }
5075
5076                 if (bits > 36) {
5077                         dev_err(adapter->pdev_dev,
5078                                 "tp_vlan_pri_map=%#x needs %d bits > 36;"\
5079                                 " using %#x\n", tp_vlan_pri_map, bits,
5080                                 TP_VLAN_PRI_MAP_DEFAULT);
5081                         tp_vlan_pri_map = TP_VLAN_PRI_MAP_DEFAULT;
5082                 }
5083         }
5084         v = tp_vlan_pri_map;
5085         t4_write_indirect(adapter, TP_PIO_ADDR, TP_PIO_DATA,
5086                           &v, 1, TP_VLAN_PRI_MAP);
5087
5088         /*
5089          * We need Five Tuple Lookup mode to be set in TP_GLOBAL_CONFIG order
5090          * to support any of the compressed filter fields above.  Newer
5091          * versions of the firmware do this automatically but it doesn't hurt
5092          * to set it here.  Meanwhile, we do _not_ need to set Lookup Every
5093          * Packet in TP_INGRESS_CONFIG to support matching non-TCP packets
5094          * since the firmware automatically turns this on and off when we have
5095          * a non-zero number of filters active (since it does have a
5096          * performance impact).
5097          */
5098         if (tp_vlan_pri_map)
5099                 t4_set_reg_field(adapter, TP_GLOBAL_CONFIG,
5100                                  FIVETUPLELOOKUP_MASK,
5101                                  FIVETUPLELOOKUP_MASK);
5102
5103         /*
5104          * Tweak some settings.
5105          */
5106         t4_write_reg(adapter, TP_SHIFT_CNT, SYNSHIFTMAX(6) |
5107                      RXTSHIFTMAXR1(4) | RXTSHIFTMAXR2(15) |
5108                      PERSHIFTBACKOFFMAX(8) | PERSHIFTMAX(8) |
5109                      KEEPALIVEMAXR1(4) | KEEPALIVEMAXR2(9));
5110
5111         /*
5112          * Get basic stuff going by issuing the Firmware Initialize command.
5113          * Note that this _must_ be after all PFVF commands ...
5114          */
5115         ret = t4_fw_initialize(adapter, adapter->mbox);
5116         if (ret < 0)
5117                 goto bye;
5118
5119         /*
5120          * Return successfully!
5121          */
5122         dev_info(adapter->pdev_dev, "Successfully configured using built-in "\
5123                  "driver parameters\n");
5124         return 0;
5125
5126         /*
5127          * Something bad happened.  Return the error ...
5128          */
5129 bye:
5130         return ret;
5131 }
5132
5133 static struct fw_info fw_info_array[] = {
5134         {
5135                 .chip = CHELSIO_T4,
5136                 .fs_name = FW4_CFNAME,
5137                 .fw_mod_name = FW4_FNAME,
5138                 .fw_hdr = {
5139                         .chip = FW_HDR_CHIP_T4,
5140                         .fw_ver = __cpu_to_be32(FW_VERSION(T4)),
5141                         .intfver_nic = FW_INTFVER(T4, NIC),
5142                         .intfver_vnic = FW_INTFVER(T4, VNIC),
5143                         .intfver_ri = FW_INTFVER(T4, RI),
5144                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
5145                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
5146                 },
5147         }, {
5148                 .chip = CHELSIO_T5,
5149                 .fs_name = FW5_CFNAME,
5150                 .fw_mod_name = FW5_FNAME,
5151                 .fw_hdr = {
5152                         .chip = FW_HDR_CHIP_T5,
5153                         .fw_ver = __cpu_to_be32(FW_VERSION(T5)),
5154                         .intfver_nic = FW_INTFVER(T5, NIC),
5155                         .intfver_vnic = FW_INTFVER(T5, VNIC),
5156                         .intfver_ri = FW_INTFVER(T5, RI),
5157                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
5158                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
5159                 },
5160         }
5161 };
5162
5163 static struct fw_info *find_fw_info(int chip)
5164 {
5165         int i;
5166
5167         for (i = 0; i < ARRAY_SIZE(fw_info_array); i++) {
5168                 if (fw_info_array[i].chip == chip)
5169                         return &fw_info_array[i];
5170         }
5171         return NULL;
5172 }
5173
5174 /*
5175  * Phase 0 of initialization: contact FW, obtain config, perform basic init.
5176  */
5177 static int adap_init0(struct adapter *adap)
5178 {
5179         int ret;
5180         u32 v, port_vec;
5181         enum dev_state state;
5182         u32 params[7], val[7];
5183         struct fw_caps_config_cmd caps_cmd;
5184         int reset = 1;
5185
5186         /*
5187          * Contact FW, advertising Master capability (and potentially forcing
5188          * ourselves as the Master PF if our module parameter force_init is
5189          * set).
5190          */
5191         ret = t4_fw_hello(adap, adap->mbox, adap->fn,
5192                           force_init ? MASTER_MUST : MASTER_MAY,
5193                           &state);
5194         if (ret < 0) {
5195                 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
5196                         ret);
5197                 return ret;
5198         }
5199         if (ret == adap->mbox)
5200                 adap->flags |= MASTER_PF;
5201         if (force_init && state == DEV_STATE_INIT)
5202                 state = DEV_STATE_UNINIT;
5203
5204         /*
5205          * If we're the Master PF Driver and the device is uninitialized,
5206          * then let's consider upgrading the firmware ...  (We always want
5207          * to check the firmware version number in order to A. get it for
5208          * later reporting and B. to warn if the currently loaded firmware
5209          * is excessively mismatched relative to the driver.)
5210          */
5211         t4_get_fw_version(adap, &adap->params.fw_vers);
5212         t4_get_tp_version(adap, &adap->params.tp_vers);
5213         if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) {
5214                 struct fw_info *fw_info;
5215                 struct fw_hdr *card_fw;
5216                 const struct firmware *fw;
5217                 const u8 *fw_data = NULL;
5218                 unsigned int fw_size = 0;
5219
5220                 /* This is the firmware whose headers the driver was compiled
5221                  * against
5222                  */
5223                 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(adap->params.chip));
5224                 if (fw_info == NULL) {
5225                         dev_err(adap->pdev_dev,
5226                                 "unable to get firmware info for chip %d.\n",
5227                                 CHELSIO_CHIP_VERSION(adap->params.chip));
5228                         return -EINVAL;
5229                 }
5230
5231                 /* allocate memory to read the header of the firmware on the
5232                  * card
5233                  */
5234                 card_fw = t4_alloc_mem(sizeof(*card_fw));
5235
5236                 /* Get FW from from /lib/firmware/ */
5237                 ret = request_firmware(&fw, fw_info->fw_mod_name,
5238                                        adap->pdev_dev);
5239                 if (ret < 0) {
5240                         dev_err(adap->pdev_dev,
5241                                 "unable to load firmware image %s, error %d\n",
5242                                 fw_info->fw_mod_name, ret);
5243                 } else {
5244                         fw_data = fw->data;
5245                         fw_size = fw->size;
5246                 }
5247
5248                 /* upgrade FW logic */
5249                 ret = t4_prep_fw(adap, fw_info, fw_data, fw_size, card_fw,
5250                                  state, &reset);
5251
5252                 /* Cleaning up */
5253                 if (fw != NULL)
5254                         release_firmware(fw);
5255                 t4_free_mem(card_fw);
5256
5257                 if (ret < 0)
5258                         goto bye;
5259         }
5260
5261         /*
5262          * Grab VPD parameters.  This should be done after we establish a
5263          * connection to the firmware since some of the VPD parameters
5264          * (notably the Core Clock frequency) are retrieved via requests to
5265          * the firmware.  On the other hand, we need these fairly early on
5266          * so we do this right after getting ahold of the firmware.
5267          */
5268         ret = get_vpd_params(adap, &adap->params.vpd);
5269         if (ret < 0)
5270                 goto bye;
5271
5272         /*
5273          * Find out what ports are available to us.  Note that we need to do
5274          * this before calling adap_init0_no_config() since it needs nports
5275          * and portvec ...
5276          */
5277         v =
5278             FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5279             FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_PORTVEC);
5280         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1, &v, &port_vec);
5281         if (ret < 0)
5282                 goto bye;
5283
5284         adap->params.nports = hweight32(port_vec);
5285         adap->params.portvec = port_vec;
5286
5287         /*
5288          * If the firmware is initialized already (and we're not forcing a
5289          * master initialization), note that we're living with existing
5290          * adapter parameters.  Otherwise, it's time to try initializing the
5291          * adapter ...
5292          */
5293         if (state == DEV_STATE_INIT) {
5294                 dev_info(adap->pdev_dev, "Coming up as %s: "\
5295                          "Adapter already initialized\n",
5296                          adap->flags & MASTER_PF ? "MASTER" : "SLAVE");
5297                 adap->flags |= USING_SOFT_PARAMS;
5298         } else {
5299                 dev_info(adap->pdev_dev, "Coming up as MASTER: "\
5300                          "Initializing adapter\n");
5301
5302                 /*
5303                  * If the firmware doesn't support Configuration
5304                  * Files warn user and exit,
5305                  */
5306                 if (ret < 0)
5307                         dev_warn(adap->pdev_dev, "Firmware doesn't support "
5308                                  "configuration file.\n");
5309                 if (force_old_init)
5310                         ret = adap_init0_no_config(adap, reset);
5311                 else {
5312                         /*
5313                          * Find out whether we're dealing with a version of
5314                          * the firmware which has configuration file support.
5315                          */
5316                         params[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5317                                      FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF));
5318                         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1,
5319                                               params, val);
5320
5321                         /*
5322                          * If the firmware doesn't support Configuration
5323                          * Files, use the old Driver-based, hard-wired
5324                          * initialization.  Otherwise, try using the
5325                          * Configuration File support and fall back to the
5326                          * Driver-based initialization if there's no
5327                          * Configuration File found.
5328                          */
5329                         if (ret < 0)
5330                                 ret = adap_init0_no_config(adap, reset);
5331                         else {
5332                                 /*
5333                                  * The firmware provides us with a memory
5334                                  * buffer where we can load a Configuration
5335                                  * File from the host if we want to override
5336                                  * the Configuration File in flash.
5337                                  */
5338
5339                                 ret = adap_init0_config(adap, reset);
5340                                 if (ret == -ENOENT) {
5341                                         dev_info(adap->pdev_dev,
5342                                             "No Configuration File present "
5343                                             "on adapter. Using hard-wired "
5344                                             "configuration parameters.\n");
5345                                         ret = adap_init0_no_config(adap, reset);
5346                                 }
5347                         }
5348                 }
5349                 if (ret < 0) {
5350                         dev_err(adap->pdev_dev,
5351                                 "could not initialize adapter, error %d\n",
5352                                 -ret);
5353                         goto bye;
5354                 }
5355         }
5356
5357         /*
5358          * If we're living with non-hard-coded parameters (either from a
5359          * Firmware Configuration File or values programmed by a different PF
5360          * Driver), give the SGE code a chance to pull in anything that it
5361          * needs ...  Note that this must be called after we retrieve our VPD
5362          * parameters in order to know how to convert core ticks to seconds.
5363          */
5364         if (adap->flags & USING_SOFT_PARAMS) {
5365                 ret = t4_sge_init(adap);
5366                 if (ret < 0)
5367                         goto bye;
5368         }
5369
5370         if (is_bypass_device(adap->pdev->device))
5371                 adap->params.bypass = 1;
5372
5373         /*
5374          * Grab some of our basic fundamental operating parameters.
5375          */
5376 #define FW_PARAM_DEV(param) \
5377         (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
5378         FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
5379
5380 #define FW_PARAM_PFVF(param) \
5381         FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
5382         FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)|  \
5383         FW_PARAMS_PARAM_Y(0) | \
5384         FW_PARAMS_PARAM_Z(0)
5385
5386         params[0] = FW_PARAM_PFVF(EQ_START);
5387         params[1] = FW_PARAM_PFVF(L2T_START);
5388         params[2] = FW_PARAM_PFVF(L2T_END);
5389         params[3] = FW_PARAM_PFVF(FILTER_START);
5390         params[4] = FW_PARAM_PFVF(FILTER_END);
5391         params[5] = FW_PARAM_PFVF(IQFLINT_START);
5392         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params, val);
5393         if (ret < 0)
5394                 goto bye;
5395         adap->sge.egr_start = val[0];
5396         adap->l2t_start = val[1];
5397         adap->l2t_end = val[2];
5398         adap->tids.ftid_base = val[3];
5399         adap->tids.nftids = val[4] - val[3] + 1;
5400         adap->sge.ingr_start = val[5];
5401
5402         /* query params related to active filter region */
5403         params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START);
5404         params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END);
5405         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
5406         /* If Active filter size is set we enable establishing
5407          * offload connection through firmware work request
5408          */
5409         if ((val[0] != val[1]) && (ret >= 0)) {
5410                 adap->flags |= FW_OFLD_CONN;
5411                 adap->tids.aftid_base = val[0];
5412                 adap->tids.aftid_end = val[1];
5413         }
5414
5415         /* If we're running on newer firmware, let it know that we're
5416          * prepared to deal with encapsulated CPL messages.  Older
5417          * firmware won't understand this and we'll just get
5418          * unencapsulated messages ...
5419          */
5420         params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
5421         val[0] = 1;
5422         (void) t4_set_params(adap, adap->mbox, adap->fn, 0, 1, params, val);
5423
5424         /*
5425          * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL
5426          * capability.  Earlier versions of the firmware didn't have the
5427          * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no
5428          * permission to use ULPTX MEMWRITE DSGL.
5429          */
5430         if (is_t4(adap->params.chip)) {
5431                 adap->params.ulptx_memwrite_dsgl = false;
5432         } else {
5433                 params[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL);
5434                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0,
5435                                       1, params, val);
5436                 adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0);
5437         }
5438
5439         /*
5440          * Get device capabilities so we can determine what resources we need
5441          * to manage.
5442          */
5443         memset(&caps_cmd, 0, sizeof(caps_cmd));
5444         caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
5445                                      FW_CMD_REQUEST | FW_CMD_READ);
5446         caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
5447         ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd),
5448                          &caps_cmd);
5449         if (ret < 0)
5450                 goto bye;
5451
5452         if (caps_cmd.ofldcaps) {
5453                 /* query offload-related parameters */
5454                 params[0] = FW_PARAM_DEV(NTID);
5455                 params[1] = FW_PARAM_PFVF(SERVER_START);
5456                 params[2] = FW_PARAM_PFVF(SERVER_END);
5457                 params[3] = FW_PARAM_PFVF(TDDP_START);
5458                 params[4] = FW_PARAM_PFVF(TDDP_END);
5459                 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
5460                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5461                                       params, val);
5462                 if (ret < 0)
5463                         goto bye;
5464                 adap->tids.ntids = val[0];
5465                 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
5466                 adap->tids.stid_base = val[1];
5467                 adap->tids.nstids = val[2] - val[1] + 1;
5468                 /*
5469                  * Setup server filter region. Divide the availble filter
5470                  * region into two parts. Regular filters get 1/3rd and server
5471                  * filters get 2/3rd part. This is only enabled if workarond
5472                  * path is enabled.
5473                  * 1. For regular filters.
5474                  * 2. Server filter: This are special filters which are used
5475                  * to redirect SYN packets to offload queue.
5476                  */
5477                 if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) {
5478                         adap->tids.sftid_base = adap->tids.ftid_base +
5479                                         DIV_ROUND_UP(adap->tids.nftids, 3);
5480                         adap->tids.nsftids = adap->tids.nftids -
5481                                          DIV_ROUND_UP(adap->tids.nftids, 3);
5482                         adap->tids.nftids = adap->tids.sftid_base -
5483                                                 adap->tids.ftid_base;
5484                 }
5485                 adap->vres.ddp.start = val[3];
5486                 adap->vres.ddp.size = val[4] - val[3] + 1;
5487                 adap->params.ofldq_wr_cred = val[5];
5488
5489                 adap->params.offload = 1;
5490         }
5491         if (caps_cmd.rdmacaps) {
5492                 params[0] = FW_PARAM_PFVF(STAG_START);
5493                 params[1] = FW_PARAM_PFVF(STAG_END);
5494                 params[2] = FW_PARAM_PFVF(RQ_START);
5495                 params[3] = FW_PARAM_PFVF(RQ_END);
5496                 params[4] = FW_PARAM_PFVF(PBL_START);
5497                 params[5] = FW_PARAM_PFVF(PBL_END);
5498                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5499                                       params, val);
5500                 if (ret < 0)
5501                         goto bye;
5502                 adap->vres.stag.start = val[0];
5503                 adap->vres.stag.size = val[1] - val[0] + 1;
5504                 adap->vres.rq.start = val[2];
5505                 adap->vres.rq.size = val[3] - val[2] + 1;
5506                 adap->vres.pbl.start = val[4];
5507                 adap->vres.pbl.size = val[5] - val[4] + 1;
5508
5509                 params[0] = FW_PARAM_PFVF(SQRQ_START);
5510                 params[1] = FW_PARAM_PFVF(SQRQ_END);
5511                 params[2] = FW_PARAM_PFVF(CQ_START);
5512                 params[3] = FW_PARAM_PFVF(CQ_END);
5513                 params[4] = FW_PARAM_PFVF(OCQ_START);
5514                 params[5] = FW_PARAM_PFVF(OCQ_END);
5515                 ret = t4_query_params(adap, 0, 0, 0, 6, params, val);
5516                 if (ret < 0)
5517                         goto bye;
5518                 adap->vres.qp.start = val[0];
5519                 adap->vres.qp.size = val[1] - val[0] + 1;
5520                 adap->vres.cq.start = val[2];
5521                 adap->vres.cq.size = val[3] - val[2] + 1;
5522                 adap->vres.ocq.start = val[4];
5523                 adap->vres.ocq.size = val[5] - val[4] + 1;
5524         }
5525         if (caps_cmd.iscsicaps) {
5526                 params[0] = FW_PARAM_PFVF(ISCSI_START);
5527                 params[1] = FW_PARAM_PFVF(ISCSI_END);
5528                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2,
5529                                       params, val);
5530                 if (ret < 0)
5531                         goto bye;
5532                 adap->vres.iscsi.start = val[0];
5533                 adap->vres.iscsi.size = val[1] - val[0] + 1;
5534         }
5535 #undef FW_PARAM_PFVF
5536 #undef FW_PARAM_DEV
5537
5538         /*
5539          * These are finalized by FW initialization, load their values now.
5540          */
5541         t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
5542         t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5543                      adap->params.b_wnd);
5544
5545         t4_init_tp_params(adap);
5546         adap->flags |= FW_OK;
5547         return 0;
5548
5549         /*
5550          * Something bad happened.  If a command timed out or failed with EIO
5551          * FW does not operate within its spec or something catastrophic
5552          * happened to HW/FW, stop issuing commands.
5553          */
5554 bye:
5555         if (ret != -ETIMEDOUT && ret != -EIO)
5556                 t4_fw_bye(adap, adap->mbox);
5557         return ret;
5558 }
5559
5560 /* EEH callbacks */
5561
5562 static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
5563                                          pci_channel_state_t state)
5564 {
5565         int i;
5566         struct adapter *adap = pci_get_drvdata(pdev);
5567
5568         if (!adap)
5569                 goto out;
5570
5571         rtnl_lock();
5572         adap->flags &= ~FW_OK;
5573         notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
5574         spin_lock(&adap->stats_lock);
5575         for_each_port(adap, i) {
5576                 struct net_device *dev = adap->port[i];
5577
5578                 netif_device_detach(dev);
5579                 netif_carrier_off(dev);
5580         }
5581         spin_unlock(&adap->stats_lock);
5582         if (adap->flags & FULL_INIT_DONE)
5583                 cxgb_down(adap);
5584         rtnl_unlock();
5585         if ((adap->flags & DEV_ENABLED)) {
5586                 pci_disable_device(pdev);
5587                 adap->flags &= ~DEV_ENABLED;
5588         }
5589 out:    return state == pci_channel_io_perm_failure ?
5590                 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
5591 }
5592
5593 static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
5594 {
5595         int i, ret;
5596         struct fw_caps_config_cmd c;
5597         struct adapter *adap = pci_get_drvdata(pdev);
5598
5599         if (!adap) {
5600                 pci_restore_state(pdev);
5601                 pci_save_state(pdev);
5602                 return PCI_ERS_RESULT_RECOVERED;
5603         }
5604
5605         if (!(adap->flags & DEV_ENABLED)) {
5606                 if (pci_enable_device(pdev)) {
5607                         dev_err(&pdev->dev, "Cannot reenable PCI "
5608                                             "device after reset\n");
5609                         return PCI_ERS_RESULT_DISCONNECT;
5610                 }
5611                 adap->flags |= DEV_ENABLED;
5612         }
5613
5614         pci_set_master(pdev);
5615         pci_restore_state(pdev);
5616         pci_save_state(pdev);
5617         pci_cleanup_aer_uncorrect_error_status(pdev);
5618
5619         if (t4_wait_dev_ready(adap) < 0)
5620                 return PCI_ERS_RESULT_DISCONNECT;
5621         if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL) < 0)
5622                 return PCI_ERS_RESULT_DISCONNECT;
5623         adap->flags |= FW_OK;
5624         if (adap_init1(adap, &c))
5625                 return PCI_ERS_RESULT_DISCONNECT;
5626
5627         for_each_port(adap, i) {
5628                 struct port_info *p = adap2pinfo(adap, i);
5629
5630                 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
5631                                   NULL, NULL);
5632                 if (ret < 0)
5633                         return PCI_ERS_RESULT_DISCONNECT;
5634                 p->viid = ret;
5635                 p->xact_addr_filt = -1;
5636         }
5637
5638         t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5639                      adap->params.b_wnd);
5640         setup_memwin(adap);
5641         if (cxgb_up(adap))
5642                 return PCI_ERS_RESULT_DISCONNECT;
5643         return PCI_ERS_RESULT_RECOVERED;
5644 }
5645
5646 static void eeh_resume(struct pci_dev *pdev)
5647 {
5648         int i;
5649         struct adapter *adap = pci_get_drvdata(pdev);
5650
5651         if (!adap)
5652                 return;
5653
5654         rtnl_lock();
5655         for_each_port(adap, i) {
5656                 struct net_device *dev = adap->port[i];
5657
5658                 if (netif_running(dev)) {
5659                         link_start(dev);
5660                         cxgb_set_rxmode(dev);
5661                 }
5662                 netif_device_attach(dev);
5663         }
5664         rtnl_unlock();
5665 }
5666
5667 static const struct pci_error_handlers cxgb4_eeh = {
5668         .error_detected = eeh_err_detected,
5669         .slot_reset     = eeh_slot_reset,
5670         .resume         = eeh_resume,
5671 };
5672
5673 static inline bool is_x_10g_port(const struct link_config *lc)
5674 {
5675         return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0 ||
5676                (lc->supported & FW_PORT_CAP_SPEED_40G) != 0;
5677 }
5678
5679 static inline void init_rspq(struct sge_rspq *q, u8 timer_idx, u8 pkt_cnt_idx,
5680                              unsigned int size, unsigned int iqe_size)
5681 {
5682         q->intr_params = QINTR_TIMER_IDX(timer_idx) |
5683                          (pkt_cnt_idx < SGE_NCOUNTERS ? QINTR_CNT_EN : 0);
5684         q->pktcnt_idx = pkt_cnt_idx < SGE_NCOUNTERS ? pkt_cnt_idx : 0;
5685         q->iqe_len = iqe_size;
5686         q->size = size;
5687 }
5688
5689 /*
5690  * Perform default configuration of DMA queues depending on the number and type
5691  * of ports we found and the number of available CPUs.  Most settings can be
5692  * modified by the admin prior to actual use.
5693  */
5694 static void cfg_queues(struct adapter *adap)
5695 {
5696         struct sge *s = &adap->sge;
5697         int i, q10g = 0, n10g = 0, qidx = 0;
5698
5699         for_each_port(adap, i)
5700                 n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg);
5701
5702         /*
5703          * We default to 1 queue per non-10G port and up to # of cores queues
5704          * per 10G port.
5705          */
5706         if (n10g)
5707                 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
5708         if (q10g > netif_get_num_default_rss_queues())
5709                 q10g = netif_get_num_default_rss_queues();
5710
5711         for_each_port(adap, i) {
5712                 struct port_info *pi = adap2pinfo(adap, i);
5713
5714                 pi->first_qset = qidx;
5715                 pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1;
5716                 qidx += pi->nqsets;
5717         }
5718
5719         s->ethqsets = qidx;
5720         s->max_ethqsets = qidx;   /* MSI-X may lower it later */
5721
5722         if (is_offload(adap)) {
5723                 /*
5724                  * For offload we use 1 queue/channel if all ports are up to 1G,
5725                  * otherwise we divide all available queues amongst the channels
5726                  * capped by the number of available cores.
5727                  */
5728                 if (n10g) {
5729                         i = min_t(int, ARRAY_SIZE(s->ofldrxq),
5730                                   num_online_cpus());
5731                         s->ofldqsets = roundup(i, adap->params.nports);
5732                 } else
5733                         s->ofldqsets = adap->params.nports;
5734                 /* For RDMA one Rx queue per channel suffices */
5735                 s->rdmaqs = adap->params.nports;
5736         }
5737
5738         for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
5739                 struct sge_eth_rxq *r = &s->ethrxq[i];
5740
5741                 init_rspq(&r->rspq, 0, 0, 1024, 64);
5742                 r->fl.size = 72;
5743         }
5744
5745         for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
5746                 s->ethtxq[i].q.size = 1024;
5747
5748         for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
5749                 s->ctrlq[i].q.size = 512;
5750
5751         for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
5752                 s->ofldtxq[i].q.size = 1024;
5753
5754         for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
5755                 struct sge_ofld_rxq *r = &s->ofldrxq[i];
5756
5757                 init_rspq(&r->rspq, 0, 0, 1024, 64);
5758                 r->rspq.uld = CXGB4_ULD_ISCSI;
5759                 r->fl.size = 72;
5760         }
5761
5762         for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
5763                 struct sge_ofld_rxq *r = &s->rdmarxq[i];
5764
5765                 init_rspq(&r->rspq, 0, 0, 511, 64);
5766                 r->rspq.uld = CXGB4_ULD_RDMA;
5767                 r->fl.size = 72;
5768         }
5769
5770         init_rspq(&s->fw_evtq, 6, 0, 512, 64);
5771         init_rspq(&s->intrq, 6, 0, 2 * MAX_INGQ, 64);
5772 }
5773
5774 /*
5775  * Reduce the number of Ethernet queues across all ports to at most n.
5776  * n provides at least one queue per port.
5777  */
5778 static void reduce_ethqs(struct adapter *adap, int n)
5779 {
5780         int i;
5781         struct port_info *pi;
5782
5783         while (n < adap->sge.ethqsets)
5784                 for_each_port(adap, i) {
5785                         pi = adap2pinfo(adap, i);
5786                         if (pi->nqsets > 1) {
5787                                 pi->nqsets--;
5788                                 adap->sge.ethqsets--;
5789                                 if (adap->sge.ethqsets <= n)
5790                                         break;
5791                         }
5792                 }
5793
5794         n = 0;
5795         for_each_port(adap, i) {
5796                 pi = adap2pinfo(adap, i);
5797                 pi->first_qset = n;
5798                 n += pi->nqsets;
5799         }
5800 }
5801
5802 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
5803 #define EXTRA_VECS 2
5804
5805 static int enable_msix(struct adapter *adap)
5806 {
5807         int ofld_need = 0;
5808         int i, want, need;
5809         struct sge *s = &adap->sge;
5810         unsigned int nchan = adap->params.nports;
5811         struct msix_entry entries[MAX_INGQ + 1];
5812
5813         for (i = 0; i < ARRAY_SIZE(entries); ++i)
5814                 entries[i].entry = i;
5815
5816         want = s->max_ethqsets + EXTRA_VECS;
5817         if (is_offload(adap)) {
5818                 want += s->rdmaqs + s->ofldqsets;
5819                 /* need nchan for each possible ULD */
5820                 ofld_need = 2 * nchan;
5821         }
5822         need = adap->params.nports + EXTRA_VECS + ofld_need;
5823
5824         want = pci_enable_msix_range(adap->pdev, entries, need, want);
5825         if (want < 0)
5826                 return want;
5827
5828         /*
5829          * Distribute available vectors to the various queue groups.
5830          * Every group gets its minimum requirement and NIC gets top
5831          * priority for leftovers.
5832          */
5833         i = want - EXTRA_VECS - ofld_need;
5834         if (i < s->max_ethqsets) {
5835                 s->max_ethqsets = i;
5836                 if (i < s->ethqsets)
5837                         reduce_ethqs(adap, i);
5838         }
5839         if (is_offload(adap)) {
5840                 i = want - EXTRA_VECS - s->max_ethqsets;
5841                 i -= ofld_need - nchan;
5842                 s->ofldqsets = (i / nchan) * nchan;  /* round down */
5843         }
5844         for (i = 0; i < want; ++i)
5845                 adap->msix_info[i].vec = entries[i].vector;
5846
5847         return 0;
5848 }
5849
5850 #undef EXTRA_VECS
5851
5852 static int init_rss(struct adapter *adap)
5853 {
5854         unsigned int i, j;
5855
5856         for_each_port(adap, i) {
5857                 struct port_info *pi = adap2pinfo(adap, i);
5858
5859                 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
5860                 if (!pi->rss)
5861                         return -ENOMEM;
5862                 for (j = 0; j < pi->rss_size; j++)
5863                         pi->rss[j] = ethtool_rxfh_indir_default(j, pi->nqsets);
5864         }
5865         return 0;
5866 }
5867
5868 static void print_port_info(const struct net_device *dev)
5869 {
5870         char buf[80];
5871         char *bufp = buf;
5872         const char *spd = "";
5873         const struct port_info *pi = netdev_priv(dev);
5874         const struct adapter *adap = pi->adapter;
5875
5876         if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
5877                 spd = " 2.5 GT/s";
5878         else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
5879                 spd = " 5 GT/s";
5880         else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_8_0GB)
5881                 spd = " 8 GT/s";
5882
5883         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
5884                 bufp += sprintf(bufp, "100/");
5885         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
5886                 bufp += sprintf(bufp, "1000/");
5887         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
5888                 bufp += sprintf(bufp, "10G/");
5889         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G)
5890                 bufp += sprintf(bufp, "40G/");
5891         if (bufp != buf)
5892                 --bufp;
5893         sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type));
5894
5895         netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
5896                     adap->params.vpd.id,
5897                     CHELSIO_CHIP_RELEASE(adap->params.chip), buf,
5898                     is_offload(adap) ? "R" : "", adap->params.pci.width, spd,
5899                     (adap->flags & USING_MSIX) ? " MSI-X" :
5900                     (adap->flags & USING_MSI) ? " MSI" : "");
5901         netdev_info(dev, "S/N: %s, P/N: %s\n",
5902                     adap->params.vpd.sn, adap->params.vpd.pn);
5903 }
5904
5905 static void enable_pcie_relaxed_ordering(struct pci_dev *dev)
5906 {
5907         pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
5908 }
5909
5910 /*
5911  * Free the following resources:
5912  * - memory used for tables
5913  * - MSI/MSI-X
5914  * - net devices
5915  * - resources FW is holding for us
5916  */
5917 static void free_some_resources(struct adapter *adapter)
5918 {
5919         unsigned int i;
5920
5921         t4_free_mem(adapter->l2t);
5922         t4_free_mem(adapter->tids.tid_tab);
5923         disable_msi(adapter);
5924
5925         for_each_port(adapter, i)
5926                 if (adapter->port[i]) {
5927                         kfree(adap2pinfo(adapter, i)->rss);
5928                         free_netdev(adapter->port[i]);
5929                 }
5930         if (adapter->flags & FW_OK)
5931                 t4_fw_bye(adapter, adapter->fn);
5932 }
5933
5934 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
5935 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
5936                    NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
5937 #define SEGMENT_SIZE 128
5938
5939 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
5940 {
5941         int func, i, err, s_qpp, qpp, num_seg;
5942         struct port_info *pi;
5943         bool highdma = false;
5944         struct adapter *adapter = NULL;
5945
5946         printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
5947
5948         err = pci_request_regions(pdev, KBUILD_MODNAME);
5949         if (err) {
5950                 /* Just info, some other driver may have claimed the device. */
5951                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
5952                 return err;
5953         }
5954
5955         /* We control everything through one PF */
5956         func = PCI_FUNC(pdev->devfn);
5957         if (func != ent->driver_data) {
5958                 pci_save_state(pdev);        /* to restore SR-IOV later */
5959                 goto sriov;
5960         }
5961
5962         err = pci_enable_device(pdev);
5963         if (err) {
5964                 dev_err(&pdev->dev, "cannot enable PCI device\n");
5965                 goto out_release_regions;
5966         }
5967
5968         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
5969                 highdma = true;
5970                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
5971                 if (err) {
5972                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
5973                                 "coherent allocations\n");
5974                         goto out_disable_device;
5975                 }
5976         } else {
5977                 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
5978                 if (err) {
5979                         dev_err(&pdev->dev, "no usable DMA configuration\n");
5980                         goto out_disable_device;
5981                 }
5982         }
5983
5984         pci_enable_pcie_error_reporting(pdev);
5985         enable_pcie_relaxed_ordering(pdev);
5986         pci_set_master(pdev);
5987         pci_save_state(pdev);
5988
5989         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
5990         if (!adapter) {
5991                 err = -ENOMEM;
5992                 goto out_disable_device;
5993         }
5994
5995         /* PCI device has been enabled */
5996         adapter->flags |= DEV_ENABLED;
5997
5998         adapter->regs = pci_ioremap_bar(pdev, 0);
5999         if (!adapter->regs) {
6000                 dev_err(&pdev->dev, "cannot map device registers\n");
6001                 err = -ENOMEM;
6002                 goto out_free_adapter;
6003         }
6004
6005         adapter->pdev = pdev;
6006         adapter->pdev_dev = &pdev->dev;
6007         adapter->mbox = func;
6008         adapter->fn = func;
6009         adapter->msg_enable = dflt_msg_enable;
6010         memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
6011
6012         spin_lock_init(&adapter->stats_lock);
6013         spin_lock_init(&adapter->tid_release_lock);
6014
6015         INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
6016         INIT_WORK(&adapter->db_full_task, process_db_full);
6017         INIT_WORK(&adapter->db_drop_task, process_db_drop);
6018
6019         err = t4_prep_adapter(adapter);
6020         if (err)
6021                 goto out_unmap_bar0;
6022
6023         if (!is_t4(adapter->params.chip)) {
6024                 s_qpp = QUEUESPERPAGEPF1 * adapter->fn;
6025                 qpp = 1 << QUEUESPERPAGEPF0_GET(t4_read_reg(adapter,
6026                       SGE_EGRESS_QUEUES_PER_PAGE_PF) >> s_qpp);
6027                 num_seg = PAGE_SIZE / SEGMENT_SIZE;
6028
6029                 /* Each segment size is 128B. Write coalescing is enabled only
6030                  * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the
6031                  * queue is less no of segments that can be accommodated in
6032                  * a page size.
6033                  */
6034                 if (qpp > num_seg) {
6035                         dev_err(&pdev->dev,
6036                                 "Incorrect number of egress queues per page\n");
6037                         err = -EINVAL;
6038                         goto out_unmap_bar0;
6039                 }
6040                 adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2),
6041                 pci_resource_len(pdev, 2));
6042                 if (!adapter->bar2) {
6043                         dev_err(&pdev->dev, "cannot map device bar2 region\n");
6044                         err = -ENOMEM;
6045                         goto out_unmap_bar0;
6046                 }
6047         }
6048
6049         setup_memwin(adapter);
6050         err = adap_init0(adapter);
6051         setup_memwin_rdma(adapter);
6052         if (err)
6053                 goto out_unmap_bar;
6054
6055         for_each_port(adapter, i) {
6056                 struct net_device *netdev;
6057
6058                 netdev = alloc_etherdev_mq(sizeof(struct port_info),
6059                                            MAX_ETH_QSETS);
6060                 if (!netdev) {
6061                         err = -ENOMEM;
6062                         goto out_free_dev;
6063                 }
6064
6065                 SET_NETDEV_DEV(netdev, &pdev->dev);
6066
6067                 adapter->port[i] = netdev;
6068                 pi = netdev_priv(netdev);
6069                 pi->adapter = adapter;
6070                 pi->xact_addr_filt = -1;
6071                 pi->port_id = i;
6072                 netdev->irq = pdev->irq;
6073
6074                 netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
6075                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
6076                         NETIF_F_RXCSUM | NETIF_F_RXHASH |
6077                         NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
6078                 if (highdma)
6079                         netdev->hw_features |= NETIF_F_HIGHDMA;
6080                 netdev->features |= netdev->hw_features;
6081                 netdev->vlan_features = netdev->features & VLAN_FEAT;
6082
6083                 netdev->priv_flags |= IFF_UNICAST_FLT;
6084
6085                 netdev->netdev_ops = &cxgb4_netdev_ops;
6086                 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
6087         }
6088
6089         pci_set_drvdata(pdev, adapter);
6090
6091         if (adapter->flags & FW_OK) {
6092                 err = t4_port_init(adapter, func, func, 0);
6093                 if (err)
6094                         goto out_free_dev;
6095         }
6096
6097         /*
6098          * Configure queues and allocate tables now, they can be needed as
6099          * soon as the first register_netdev completes.
6100          */
6101         cfg_queues(adapter);
6102
6103         adapter->l2t = t4_init_l2t();
6104         if (!adapter->l2t) {
6105                 /* We tolerate a lack of L2T, giving up some functionality */
6106                 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
6107                 adapter->params.offload = 0;
6108         }
6109
6110         if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
6111                 dev_warn(&pdev->dev, "could not allocate TID table, "
6112                          "continuing\n");
6113                 adapter->params.offload = 0;
6114         }
6115
6116         /* See what interrupts we'll be using */
6117         if (msi > 1 && enable_msix(adapter) == 0)
6118                 adapter->flags |= USING_MSIX;
6119         else if (msi > 0 && pci_enable_msi(pdev) == 0)
6120                 adapter->flags |= USING_MSI;
6121
6122         err = init_rss(adapter);
6123         if (err)
6124                 goto out_free_dev;
6125
6126         /*
6127          * The card is now ready to go.  If any errors occur during device
6128          * registration we do not fail the whole card but rather proceed only
6129          * with the ports we manage to register successfully.  However we must
6130          * register at least one net device.
6131          */
6132         for_each_port(adapter, i) {
6133                 pi = adap2pinfo(adapter, i);
6134                 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);
6135                 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);
6136
6137                 err = register_netdev(adapter->port[i]);
6138                 if (err)
6139                         break;
6140                 adapter->chan_map[pi->tx_chan] = i;
6141                 print_port_info(adapter->port[i]);
6142         }
6143         if (i == 0) {
6144                 dev_err(&pdev->dev, "could not register any net devices\n");
6145                 goto out_free_dev;
6146         }
6147         if (err) {
6148                 dev_warn(&pdev->dev, "only %d net devices registered\n", i);
6149                 err = 0;
6150         }
6151
6152         if (cxgb4_debugfs_root) {
6153                 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
6154                                                            cxgb4_debugfs_root);
6155                 setup_debugfs(adapter);
6156         }
6157
6158         /* PCIe EEH recovery on powerpc platforms needs fundamental reset */
6159         pdev->needs_freset = 1;
6160
6161         if (is_offload(adapter))
6162                 attach_ulds(adapter);
6163
6164 sriov:
6165 #ifdef CONFIG_PCI_IOV
6166         if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
6167                 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
6168                         dev_info(&pdev->dev,
6169                                  "instantiated %u virtual functions\n",
6170                                  num_vf[func]);
6171 #endif
6172         return 0;
6173
6174  out_free_dev:
6175         free_some_resources(adapter);
6176  out_unmap_bar:
6177         if (!is_t4(adapter->params.chip))
6178                 iounmap(adapter->bar2);
6179  out_unmap_bar0:
6180         iounmap(adapter->regs);
6181  out_free_adapter:
6182         kfree(adapter);
6183  out_disable_device:
6184         pci_disable_pcie_error_reporting(pdev);
6185         pci_disable_device(pdev);
6186  out_release_regions:
6187         pci_release_regions(pdev);
6188         return err;
6189 }
6190
6191 static void remove_one(struct pci_dev *pdev)
6192 {
6193         struct adapter *adapter = pci_get_drvdata(pdev);
6194
6195 #ifdef CONFIG_PCI_IOV
6196         pci_disable_sriov(pdev);
6197
6198 #endif
6199
6200         if (adapter) {
6201                 int i;
6202
6203                 if (is_offload(adapter))
6204                         detach_ulds(adapter);
6205
6206                 for_each_port(adapter, i)
6207                         if (adapter->port[i]->reg_state == NETREG_REGISTERED)
6208                                 unregister_netdev(adapter->port[i]);
6209
6210                 if (adapter->debugfs_root)
6211                         debugfs_remove_recursive(adapter->debugfs_root);
6212
6213                 /* If we allocated filters, free up state associated with any
6214                  * valid filters ...
6215                  */
6216                 if (adapter->tids.ftid_tab) {
6217                         struct filter_entry *f = &adapter->tids.ftid_tab[0];
6218                         for (i = 0; i < (adapter->tids.nftids +
6219                                         adapter->tids.nsftids); i++, f++)
6220                                 if (f->valid)
6221                                         clear_filter(adapter, f);
6222                 }
6223
6224                 if (adapter->flags & FULL_INIT_DONE)
6225                         cxgb_down(adapter);
6226
6227                 free_some_resources(adapter);
6228                 iounmap(adapter->regs);
6229                 if (!is_t4(adapter->params.chip))
6230                         iounmap(adapter->bar2);
6231                 pci_disable_pcie_error_reporting(pdev);
6232                 if ((adapter->flags & DEV_ENABLED)) {
6233                         pci_disable_device(pdev);
6234                         adapter->flags &= ~DEV_ENABLED;
6235                 }
6236                 pci_release_regions(pdev);
6237                 kfree(adapter);
6238         } else
6239                 pci_release_regions(pdev);
6240 }
6241
6242 static struct pci_driver cxgb4_driver = {
6243         .name     = KBUILD_MODNAME,
6244         .id_table = cxgb4_pci_tbl,
6245         .probe    = init_one,
6246         .remove   = remove_one,
6247         .shutdown = remove_one,
6248         .err_handler = &cxgb4_eeh,
6249 };
6250
6251 static int __init cxgb4_init_module(void)
6252 {
6253         int ret;
6254
6255         workq = create_singlethread_workqueue("cxgb4");
6256         if (!workq)
6257                 return -ENOMEM;
6258
6259         /* Debugfs support is optional, just warn if this fails */
6260         cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
6261         if (!cxgb4_debugfs_root)
6262                 pr_warn("could not create debugfs entry, continuing\n");
6263
6264         ret = pci_register_driver(&cxgb4_driver);
6265         if (ret < 0) {
6266                 debugfs_remove(cxgb4_debugfs_root);
6267                 destroy_workqueue(workq);
6268         }
6269
6270         register_inet6addr_notifier(&cxgb4_inet6addr_notifier);
6271
6272         return ret;
6273 }
6274
6275 static void __exit cxgb4_cleanup_module(void)
6276 {
6277         unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier);
6278         pci_unregister_driver(&cxgb4_driver);
6279         debugfs_remove(cxgb4_debugfs_root);  /* NULL ok */
6280         flush_workqueue(workq);
6281         destroy_workqueue(workq);
6282 }
6283
6284 module_init(cxgb4_init_module);
6285 module_exit(cxgb4_cleanup_module);