2 * This file is part of the Chelsio T4 PCI-E SR-IOV Virtual Function Ethernet
5 * Copyright (c) 2009-2010 Chelsio Communications, Inc. All rights reserved.
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
38 #include <linux/module.h>
39 #include <linux/moduleparam.h>
40 #include <linux/init.h>
41 #include <linux/pci.h>
42 #include <linux/dma-mapping.h>
43 #include <linux/netdevice.h>
44 #include <linux/etherdevice.h>
45 #include <linux/debugfs.h>
46 #include <linux/ethtool.h>
47 #include <linux/mdio.h>
49 #include "t4vf_common.h"
50 #include "t4vf_defs.h"
52 #include "../cxgb4/t4_regs.h"
53 #include "../cxgb4/t4_msg.h"
56 * Generic information about the driver.
58 #define DRV_VERSION "2.0.0-ko"
59 #define DRV_DESC "Chelsio T4/T5/T6 Virtual Function (VF) Network Driver"
67 * Default ethtool "message level" for adapters.
69 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
70 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
71 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
73 static int dflt_msg_enable = DFLT_MSG_ENABLE;
75 module_param(dflt_msg_enable, int, 0644);
76 MODULE_PARM_DESC(dflt_msg_enable,
77 "default adapter ethtool message level bitmap");
80 * The driver uses the best interrupt scheme available on a platform in the
81 * order MSI-X then MSI. This parameter determines which of these schemes the
82 * driver may consider as follows:
84 * msi = 2: choose from among MSI-X and MSI
85 * msi = 1: only consider MSI interrupts
87 * Note that unlike the Physical Function driver, this Virtual Function driver
88 * does _not_ support legacy INTx interrupts (this limitation is mandated by
89 * the PCI-E SR-IOV standard).
93 #define MSI_DEFAULT MSI_MSIX
95 static int msi = MSI_DEFAULT;
97 module_param(msi, int, 0644);
98 MODULE_PARM_DESC(msi, "whether to use MSI-X or MSI");
101 * Fundamental constants.
102 * ======================
106 MAX_TXQ_ENTRIES = 16384,
107 MAX_RSPQ_ENTRIES = 16384,
108 MAX_RX_BUFFERS = 16384,
110 MIN_TXQ_ENTRIES = 32,
111 MIN_RSPQ_ENTRIES = 128,
115 * For purposes of manipulating the Free List size we need to
116 * recognize that Free Lists are actually Egress Queues (the host
117 * produces free buffers which the hardware consumes), Egress Queues
118 * indices are all in units of Egress Context Units bytes, and free
119 * list entries are 64-bit PCI DMA addresses. And since the state of
120 * the Producer Index == the Consumer Index implies an EMPTY list, we
121 * always have at least one Egress Unit's worth of Free List entries
122 * unused. See sge.c for more details ...
124 EQ_UNIT = SGE_EQ_IDXSIZE,
125 FL_PER_EQ_UNIT = EQ_UNIT / sizeof(__be64),
126 MIN_FL_RESID = FL_PER_EQ_UNIT,
130 * Global driver state.
131 * ====================
134 static struct dentry *cxgb4vf_debugfs_root;
137 * OS "Callback" functions.
138 * ========================
142 * The link status has changed on the indicated "port" (Virtual Interface).
144 void t4vf_os_link_changed(struct adapter *adapter, int pidx, int link_ok)
146 struct net_device *dev = adapter->port[pidx];
149 * If the port is disabled or the current recorded "link up"
150 * status matches the new status, just return.
152 if (!netif_running(dev) || link_ok == netif_carrier_ok(dev))
156 * Tell the OS that the link status has changed and print a short
157 * informative message on the console about the event.
162 const struct port_info *pi = netdev_priv(dev);
164 netif_carrier_on(dev);
166 switch (pi->link_cfg.speed) {
188 switch (pi->link_cfg.fc) {
197 case PAUSE_RX|PAUSE_TX:
206 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s, fc);
208 netif_carrier_off(dev);
209 netdev_info(dev, "link down\n");
214 * THe port module type has changed on the indicated "port" (Virtual
217 void t4vf_os_portmod_changed(struct adapter *adapter, int pidx)
219 static const char * const mod_str[] = {
220 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
222 const struct net_device *dev = adapter->port[pidx];
223 const struct port_info *pi = netdev_priv(dev);
225 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
226 dev_info(adapter->pdev_dev, "%s: port module unplugged\n",
228 else if (pi->mod_type < ARRAY_SIZE(mod_str))
229 dev_info(adapter->pdev_dev, "%s: %s port module inserted\n",
230 dev->name, mod_str[pi->mod_type]);
231 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
232 dev_info(adapter->pdev_dev, "%s: unsupported optical port "
233 "module inserted\n", dev->name);
234 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
235 dev_info(adapter->pdev_dev, "%s: unknown port module inserted,"
236 "forcing TWINAX\n", dev->name);
237 else if (pi->mod_type == FW_PORT_MOD_TYPE_ERROR)
238 dev_info(adapter->pdev_dev, "%s: transceiver module error\n",
241 dev_info(adapter->pdev_dev, "%s: unknown module type %d "
242 "inserted\n", dev->name, pi->mod_type);
246 * Net device operations.
247 * ======================
254 * Perform the MAC and PHY actions needed to enable a "port" (Virtual
257 static int link_start(struct net_device *dev)
260 struct port_info *pi = netdev_priv(dev);
263 * We do not set address filters and promiscuity here, the stack does
264 * that step explicitly. Enable vlan accel.
266 ret = t4vf_set_rxmode(pi->adapter, pi->viid, dev->mtu, -1, -1, -1, 1,
269 ret = t4vf_change_mac(pi->adapter, pi->viid,
270 pi->xact_addr_filt, dev->dev_addr, true);
272 pi->xact_addr_filt = ret;
278 * We don't need to actually "start the link" itself since the
279 * firmware will do that for us when the first Virtual Interface
280 * is enabled on a port.
283 ret = t4vf_enable_vi(pi->adapter, pi->viid, true, true);
288 * Name the MSI-X interrupts.
290 static void name_msix_vecs(struct adapter *adapter)
292 int namelen = sizeof(adapter->msix_info[0].desc) - 1;
298 snprintf(adapter->msix_info[MSIX_FW].desc, namelen,
299 "%s-FWeventq", adapter->name);
300 adapter->msix_info[MSIX_FW].desc[namelen] = 0;
305 for_each_port(adapter, pidx) {
306 struct net_device *dev = adapter->port[pidx];
307 const struct port_info *pi = netdev_priv(dev);
310 for (qs = 0, msi = MSIX_IQFLINT; qs < pi->nqsets; qs++, msi++) {
311 snprintf(adapter->msix_info[msi].desc, namelen,
312 "%s-%d", dev->name, qs);
313 adapter->msix_info[msi].desc[namelen] = 0;
319 * Request all of our MSI-X resources.
321 static int request_msix_queue_irqs(struct adapter *adapter)
323 struct sge *s = &adapter->sge;
329 err = request_irq(adapter->msix_info[MSIX_FW].vec, t4vf_sge_intr_msix,
330 0, adapter->msix_info[MSIX_FW].desc, &s->fw_evtq);
338 for_each_ethrxq(s, rxq) {
339 err = request_irq(adapter->msix_info[msi].vec,
340 t4vf_sge_intr_msix, 0,
341 adapter->msix_info[msi].desc,
342 &s->ethrxq[rxq].rspq);
351 free_irq(adapter->msix_info[--msi].vec, &s->ethrxq[rxq].rspq);
352 free_irq(adapter->msix_info[MSIX_FW].vec, &s->fw_evtq);
357 * Free our MSI-X resources.
359 static void free_msix_queue_irqs(struct adapter *adapter)
361 struct sge *s = &adapter->sge;
364 free_irq(adapter->msix_info[MSIX_FW].vec, &s->fw_evtq);
366 for_each_ethrxq(s, rxq)
367 free_irq(adapter->msix_info[msi++].vec,
368 &s->ethrxq[rxq].rspq);
372 * Turn on NAPI and start up interrupts on a response queue.
374 static void qenable(struct sge_rspq *rspq)
376 napi_enable(&rspq->napi);
379 * 0-increment the Going To Sleep register to start the timer and
382 t4_write_reg(rspq->adapter, T4VF_SGE_BASE_ADDR + SGE_VF_GTS,
384 SEINTARM_V(rspq->intr_params) |
385 INGRESSQID_V(rspq->cntxt_id));
389 * Enable NAPI scheduling and interrupt generation for all Receive Queues.
391 static void enable_rx(struct adapter *adapter)
394 struct sge *s = &adapter->sge;
396 for_each_ethrxq(s, rxq)
397 qenable(&s->ethrxq[rxq].rspq);
398 qenable(&s->fw_evtq);
401 * The interrupt queue doesn't use NAPI so we do the 0-increment of
402 * its Going To Sleep register here to get it started.
404 if (adapter->flags & USING_MSI)
405 t4_write_reg(adapter, T4VF_SGE_BASE_ADDR + SGE_VF_GTS,
407 SEINTARM_V(s->intrq.intr_params) |
408 INGRESSQID_V(s->intrq.cntxt_id));
413 * Wait until all NAPI handlers are descheduled.
415 static void quiesce_rx(struct adapter *adapter)
417 struct sge *s = &adapter->sge;
420 for_each_ethrxq(s, rxq)
421 napi_disable(&s->ethrxq[rxq].rspq.napi);
422 napi_disable(&s->fw_evtq.napi);
426 * Response queue handler for the firmware event queue.
428 static int fwevtq_handler(struct sge_rspq *rspq, const __be64 *rsp,
429 const struct pkt_gl *gl)
432 * Extract response opcode and get pointer to CPL message body.
434 struct adapter *adapter = rspq->adapter;
435 u8 opcode = ((const struct rss_header *)rsp)->opcode;
436 void *cpl = (void *)(rsp + 1);
441 * We've received an asynchronous message from the firmware.
443 const struct cpl_fw6_msg *fw_msg = cpl;
444 if (fw_msg->type == FW6_TYPE_CMD_RPL)
445 t4vf_handle_fw_rpl(adapter, fw_msg->data);
450 /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
452 const struct cpl_sge_egr_update *p = (void *)(rsp + 3);
453 opcode = CPL_OPCODE_G(ntohl(p->opcode_qid));
454 if (opcode != CPL_SGE_EGR_UPDATE) {
455 dev_err(adapter->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n"
463 case CPL_SGE_EGR_UPDATE: {
465 * We've received an Egress Queue Status Update message. We
466 * get these, if the SGE is configured to send these when the
467 * firmware passes certain points in processing our TX
468 * Ethernet Queue or if we make an explicit request for one.
469 * We use these updates to determine when we may need to
470 * restart a TX Ethernet Queue which was stopped for lack of
471 * free TX Queue Descriptors ...
473 const struct cpl_sge_egr_update *p = cpl;
474 unsigned int qid = EGR_QID_G(be32_to_cpu(p->opcode_qid));
475 struct sge *s = &adapter->sge;
477 struct sge_eth_txq *txq;
481 * Perform sanity checking on the Queue ID to make sure it
482 * really refers to one of our TX Ethernet Egress Queues which
483 * is active and matches the queue's ID. None of these error
484 * conditions should ever happen so we may want to either make
485 * them fatal and/or conditionalized under DEBUG.
487 eq_idx = EQ_IDX(s, qid);
488 if (unlikely(eq_idx >= MAX_EGRQ)) {
489 dev_err(adapter->pdev_dev,
490 "Egress Update QID %d out of range\n", qid);
493 tq = s->egr_map[eq_idx];
494 if (unlikely(tq == NULL)) {
495 dev_err(adapter->pdev_dev,
496 "Egress Update QID %d TXQ=NULL\n", qid);
499 txq = container_of(tq, struct sge_eth_txq, q);
500 if (unlikely(tq->abs_id != qid)) {
501 dev_err(adapter->pdev_dev,
502 "Egress Update QID %d refers to TXQ %d\n",
508 * Restart a stopped TX Queue which has less than half of its
512 netif_tx_wake_queue(txq->txq);
517 dev_err(adapter->pdev_dev,
518 "unexpected CPL %#x on FW event queue\n", opcode);
525 * Allocate SGE TX/RX response queues. Determine how many sets of SGE queues
526 * to use and initializes them. We support multiple "Queue Sets" per port if
527 * we have MSI-X, otherwise just one queue set per port.
529 static int setup_sge_queues(struct adapter *adapter)
531 struct sge *s = &adapter->sge;
535 * Clear "Queue Set" Free List Starving and TX Queue Mapping Error
538 bitmap_zero(s->starving_fl, MAX_EGRQ);
541 * If we're using MSI interrupt mode we need to set up a "forwarded
542 * interrupt" queue which we'll set up with our MSI vector. The rest
543 * of the ingress queues will be set up to forward their interrupts to
544 * this queue ... This must be first since t4vf_sge_alloc_rxq() uses
545 * the intrq's queue ID as the interrupt forwarding queue for the
546 * subsequent calls ...
548 if (adapter->flags & USING_MSI) {
549 err = t4vf_sge_alloc_rxq(adapter, &s->intrq, false,
550 adapter->port[0], 0, NULL, NULL);
552 goto err_free_queues;
556 * Allocate our ingress queue for asynchronous firmware messages.
558 err = t4vf_sge_alloc_rxq(adapter, &s->fw_evtq, true, adapter->port[0],
559 MSIX_FW, NULL, fwevtq_handler);
561 goto err_free_queues;
564 * Allocate each "port"'s initial Queue Sets. These can be changed
565 * later on ... up to the point where any interface on the adapter is
566 * brought up at which point lots of things get nailed down
570 for_each_port(adapter, pidx) {
571 struct net_device *dev = adapter->port[pidx];
572 struct port_info *pi = netdev_priv(dev);
573 struct sge_eth_rxq *rxq = &s->ethrxq[pi->first_qset];
574 struct sge_eth_txq *txq = &s->ethtxq[pi->first_qset];
577 for (qs = 0; qs < pi->nqsets; qs++, rxq++, txq++) {
578 err = t4vf_sge_alloc_rxq(adapter, &rxq->rspq, false,
580 &rxq->fl, t4vf_ethrx_handler);
582 goto err_free_queues;
584 err = t4vf_sge_alloc_eth_txq(adapter, txq, dev,
585 netdev_get_tx_queue(dev, qs),
586 s->fw_evtq.cntxt_id);
588 goto err_free_queues;
591 memset(&rxq->stats, 0, sizeof(rxq->stats));
596 * Create the reverse mappings for the queues.
598 s->egr_base = s->ethtxq[0].q.abs_id - s->ethtxq[0].q.cntxt_id;
599 s->ingr_base = s->ethrxq[0].rspq.abs_id - s->ethrxq[0].rspq.cntxt_id;
600 IQ_MAP(s, s->fw_evtq.abs_id) = &s->fw_evtq;
601 for_each_port(adapter, pidx) {
602 struct net_device *dev = adapter->port[pidx];
603 struct port_info *pi = netdev_priv(dev);
604 struct sge_eth_rxq *rxq = &s->ethrxq[pi->first_qset];
605 struct sge_eth_txq *txq = &s->ethtxq[pi->first_qset];
608 for (qs = 0; qs < pi->nqsets; qs++, rxq++, txq++) {
609 IQ_MAP(s, rxq->rspq.abs_id) = &rxq->rspq;
610 EQ_MAP(s, txq->q.abs_id) = &txq->q;
613 * The FW_IQ_CMD doesn't return the Absolute Queue IDs
614 * for Free Lists but since all of the Egress Queues
615 * (including Free Lists) have Relative Queue IDs
616 * which are computed as Absolute - Base Queue ID, we
617 * can synthesize the Absolute Queue IDs for the Free
618 * Lists. This is useful for debugging purposes when
619 * we want to dump Queue Contexts via the PF Driver.
621 rxq->fl.abs_id = rxq->fl.cntxt_id + s->egr_base;
622 EQ_MAP(s, rxq->fl.abs_id) = &rxq->fl;
628 t4vf_free_sge_resources(adapter);
633 * Set up Receive Side Scaling (RSS) to distribute packets to multiple receive
634 * queues. We configure the RSS CPU lookup table to distribute to the number
635 * of HW receive queues, and the response queue lookup table to narrow that
636 * down to the response queues actually configured for each "port" (Virtual
637 * Interface). We always configure the RSS mapping for all ports since the
638 * mapping table has plenty of entries.
640 static int setup_rss(struct adapter *adapter)
644 for_each_port(adapter, pidx) {
645 struct port_info *pi = adap2pinfo(adapter, pidx);
646 struct sge_eth_rxq *rxq = &adapter->sge.ethrxq[pi->first_qset];
647 u16 rss[MAX_PORT_QSETS];
650 for (qs = 0; qs < pi->nqsets; qs++)
651 rss[qs] = rxq[qs].rspq.abs_id;
653 err = t4vf_config_rss_range(adapter, pi->viid,
654 0, pi->rss_size, rss, pi->nqsets);
659 * Perform Global RSS Mode-specific initialization.
661 switch (adapter->params.rss.mode) {
662 case FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL:
664 * If Tunnel All Lookup isn't specified in the global
665 * RSS Configuration, then we need to specify a
666 * default Ingress Queue for any ingress packets which
667 * aren't hashed. We'll use our first ingress queue
670 if (!adapter->params.rss.u.basicvirtual.tnlalllookup) {
671 union rss_vi_config config;
672 err = t4vf_read_rss_vi_config(adapter,
677 config.basicvirtual.defaultq =
679 err = t4vf_write_rss_vi_config(adapter,
693 * Bring the adapter up. Called whenever we go from no "ports" open to having
694 * one open. This function performs the actions necessary to make an adapter
695 * operational, such as completing the initialization of HW modules, and
696 * enabling interrupts. Must be called with the rtnl lock held. (Note that
697 * this is called "cxgb_up" in the PF Driver.)
699 static int adapter_up(struct adapter *adapter)
704 * If this is the first time we've been called, perform basic
705 * adapter setup. Once we've done this, many of our adapter
706 * parameters can no longer be changed ...
708 if ((adapter->flags & FULL_INIT_DONE) == 0) {
709 err = setup_sge_queues(adapter);
712 err = setup_rss(adapter);
714 t4vf_free_sge_resources(adapter);
718 if (adapter->flags & USING_MSIX)
719 name_msix_vecs(adapter);
720 adapter->flags |= FULL_INIT_DONE;
724 * Acquire our interrupt resources. We only support MSI-X and MSI.
726 BUG_ON((adapter->flags & (USING_MSIX|USING_MSI)) == 0);
727 if (adapter->flags & USING_MSIX)
728 err = request_msix_queue_irqs(adapter);
730 err = request_irq(adapter->pdev->irq,
731 t4vf_intr_handler(adapter), 0,
732 adapter->name, adapter);
734 dev_err(adapter->pdev_dev, "request_irq failed, err %d\n",
740 * Enable NAPI ingress processing and return success.
743 t4vf_sge_start(adapter);
745 /* Initialize hash mac addr list*/
746 INIT_LIST_HEAD(&adapter->mac_hlist);
751 * Bring the adapter down. Called whenever the last "port" (Virtual
752 * Interface) closed. (Note that this routine is called "cxgb_down" in the PF
755 static void adapter_down(struct adapter *adapter)
758 * Free interrupt resources.
760 if (adapter->flags & USING_MSIX)
761 free_msix_queue_irqs(adapter);
763 free_irq(adapter->pdev->irq, adapter);
766 * Wait for NAPI handlers to finish.
772 * Start up a net device.
774 static int cxgb4vf_open(struct net_device *dev)
777 struct port_info *pi = netdev_priv(dev);
778 struct adapter *adapter = pi->adapter;
781 * If this is the first interface that we're opening on the "adapter",
782 * bring the "adapter" up now.
784 if (adapter->open_device_map == 0) {
785 err = adapter_up(adapter);
791 * Note that this interface is up and start everything up ...
793 netif_set_real_num_tx_queues(dev, pi->nqsets);
794 err = netif_set_real_num_rx_queues(dev, pi->nqsets);
797 err = link_start(dev);
801 netif_tx_start_all_queues(dev);
802 set_bit(pi->port_id, &adapter->open_device_map);
806 if (adapter->open_device_map == 0)
807 adapter_down(adapter);
812 * Shut down a net device. This routine is called "cxgb_close" in the PF
815 static int cxgb4vf_stop(struct net_device *dev)
817 struct port_info *pi = netdev_priv(dev);
818 struct adapter *adapter = pi->adapter;
820 netif_tx_stop_all_queues(dev);
821 netif_carrier_off(dev);
822 t4vf_enable_vi(adapter, pi->viid, false, false);
823 pi->link_cfg.link_ok = 0;
825 clear_bit(pi->port_id, &adapter->open_device_map);
826 if (adapter->open_device_map == 0)
827 adapter_down(adapter);
832 * Translate our basic statistics into the standard "ifconfig" statistics.
834 static struct net_device_stats *cxgb4vf_get_stats(struct net_device *dev)
836 struct t4vf_port_stats stats;
837 struct port_info *pi = netdev2pinfo(dev);
838 struct adapter *adapter = pi->adapter;
839 struct net_device_stats *ns = &dev->stats;
842 spin_lock(&adapter->stats_lock);
843 err = t4vf_get_port_stats(adapter, pi->pidx, &stats);
844 spin_unlock(&adapter->stats_lock);
846 memset(ns, 0, sizeof(*ns));
850 ns->tx_bytes = (stats.tx_bcast_bytes + stats.tx_mcast_bytes +
851 stats.tx_ucast_bytes + stats.tx_offload_bytes);
852 ns->tx_packets = (stats.tx_bcast_frames + stats.tx_mcast_frames +
853 stats.tx_ucast_frames + stats.tx_offload_frames);
854 ns->rx_bytes = (stats.rx_bcast_bytes + stats.rx_mcast_bytes +
855 stats.rx_ucast_bytes);
856 ns->rx_packets = (stats.rx_bcast_frames + stats.rx_mcast_frames +
857 stats.rx_ucast_frames);
858 ns->multicast = stats.rx_mcast_frames;
859 ns->tx_errors = stats.tx_drop_frames;
860 ns->rx_errors = stats.rx_err_frames;
865 static inline int cxgb4vf_set_addr_hash(struct port_info *pi)
867 struct adapter *adapter = pi->adapter;
870 struct hash_mac_addr *entry;
872 /* Calculate the hash vector for the updated list and program it */
873 list_for_each_entry(entry, &adapter->mac_hlist, list) {
874 ucast |= is_unicast_ether_addr(entry->addr);
875 vec |= (1ULL << hash_mac_addr(entry->addr));
877 return t4vf_set_addr_hash(adapter, pi->viid, ucast, vec, false);
880 static int cxgb4vf_mac_sync(struct net_device *netdev, const u8 *mac_addr)
882 struct port_info *pi = netdev_priv(netdev);
883 struct adapter *adapter = pi->adapter;
888 bool ucast = is_unicast_ether_addr(mac_addr);
889 const u8 *maclist[1] = {mac_addr};
890 struct hash_mac_addr *new_entry;
892 ret = t4vf_alloc_mac_filt(adapter, pi->viid, free, 1, maclist,
893 NULL, ucast ? &uhash : &mhash, false);
896 /* if hash != 0, then add the addr to hash addr list
897 * so on the end we will calculate the hash for the
898 * list and program it
900 if (uhash || mhash) {
901 new_entry = kzalloc(sizeof(*new_entry), GFP_ATOMIC);
904 ether_addr_copy(new_entry->addr, mac_addr);
905 list_add_tail(&new_entry->list, &adapter->mac_hlist);
906 ret = cxgb4vf_set_addr_hash(pi);
909 return ret < 0 ? ret : 0;
912 static int cxgb4vf_mac_unsync(struct net_device *netdev, const u8 *mac_addr)
914 struct port_info *pi = netdev_priv(netdev);
915 struct adapter *adapter = pi->adapter;
917 const u8 *maclist[1] = {mac_addr};
918 struct hash_mac_addr *entry, *tmp;
920 /* If the MAC address to be removed is in the hash addr
921 * list, delete it from the list and update hash vector
923 list_for_each_entry_safe(entry, tmp, &adapter->mac_hlist, list) {
924 if (ether_addr_equal(entry->addr, mac_addr)) {
925 list_del(&entry->list);
927 return cxgb4vf_set_addr_hash(pi);
931 ret = t4vf_free_mac_filt(adapter, pi->viid, 1, maclist, false);
932 return ret < 0 ? -EINVAL : 0;
936 * Set RX properties of a port, such as promiscruity, address filters, and MTU.
937 * If @mtu is -1 it is left unchanged.
939 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
941 struct port_info *pi = netdev_priv(dev);
943 if (!(dev->flags & IFF_PROMISC)) {
944 __dev_uc_sync(dev, cxgb4vf_mac_sync, cxgb4vf_mac_unsync);
945 if (!(dev->flags & IFF_ALLMULTI))
946 __dev_mc_sync(dev, cxgb4vf_mac_sync,
949 return t4vf_set_rxmode(pi->adapter, pi->viid, -1,
950 (dev->flags & IFF_PROMISC) != 0,
951 (dev->flags & IFF_ALLMULTI) != 0,
956 * Set the current receive modes on the device.
958 static void cxgb4vf_set_rxmode(struct net_device *dev)
960 /* unfortunately we can't return errors to the stack */
961 set_rxmode(dev, -1, false);
965 * Find the entry in the interrupt holdoff timer value array which comes
966 * closest to the specified interrupt holdoff value.
968 static int closest_timer(const struct sge *s, int us)
970 int i, timer_idx = 0, min_delta = INT_MAX;
972 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
973 int delta = us - s->timer_val[i];
976 if (delta < min_delta) {
984 static int closest_thres(const struct sge *s, int thres)
986 int i, delta, pktcnt_idx = 0, min_delta = INT_MAX;
988 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
989 delta = thres - s->counter_val[i];
992 if (delta < min_delta) {
1001 * Return a queue's interrupt hold-off time in us. 0 means no timer.
1003 static unsigned int qtimer_val(const struct adapter *adapter,
1004 const struct sge_rspq *rspq)
1006 unsigned int timer_idx = QINTR_TIMER_IDX_G(rspq->intr_params);
1008 return timer_idx < SGE_NTIMERS
1009 ? adapter->sge.timer_val[timer_idx]
1014 * set_rxq_intr_params - set a queue's interrupt holdoff parameters
1015 * @adapter: the adapter
1016 * @rspq: the RX response queue
1017 * @us: the hold-off time in us, or 0 to disable timer
1018 * @cnt: the hold-off packet count, or 0 to disable counter
1020 * Sets an RX response queue's interrupt hold-off time and packet count.
1021 * At least one of the two needs to be enabled for the queue to generate
1024 static int set_rxq_intr_params(struct adapter *adapter, struct sge_rspq *rspq,
1025 unsigned int us, unsigned int cnt)
1027 unsigned int timer_idx;
1030 * If both the interrupt holdoff timer and count are specified as
1031 * zero, default to a holdoff count of 1 ...
1033 if ((us | cnt) == 0)
1037 * If an interrupt holdoff count has been specified, then find the
1038 * closest configured holdoff count and use that. If the response
1039 * queue has already been created, then update its queue context
1046 pktcnt_idx = closest_thres(&adapter->sge, cnt);
1047 if (rspq->desc && rspq->pktcnt_idx != pktcnt_idx) {
1048 v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
1049 FW_PARAMS_PARAM_X_V(
1050 FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
1051 FW_PARAMS_PARAM_YZ_V(rspq->cntxt_id);
1052 err = t4vf_set_params(adapter, 1, &v, &pktcnt_idx);
1056 rspq->pktcnt_idx = pktcnt_idx;
1060 * Compute the closest holdoff timer index from the supplied holdoff
1063 timer_idx = (us == 0
1064 ? SGE_TIMER_RSTRT_CNTR
1065 : closest_timer(&adapter->sge, us));
1068 * Update the response queue's interrupt coalescing parameters and
1071 rspq->intr_params = (QINTR_TIMER_IDX_V(timer_idx) |
1072 QINTR_CNT_EN_V(cnt > 0));
1077 * Return a version number to identify the type of adapter. The scheme is:
1078 * - bits 0..9: chip version
1079 * - bits 10..15: chip revision
1081 static inline unsigned int mk_adap_vers(const struct adapter *adapter)
1084 * Chip version 4, revision 0x3f (cxgb4vf).
1086 return CHELSIO_CHIP_VERSION(adapter->params.chip) | (0x3f << 10);
1090 * Execute the specified ioctl command.
1092 static int cxgb4vf_do_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
1098 * The VF Driver doesn't have access to any of the other
1099 * common Ethernet device ioctl()'s (like reading/writing
1100 * PHY registers, etc.
1111 * Change the device's MTU.
1113 static int cxgb4vf_change_mtu(struct net_device *dev, int new_mtu)
1116 struct port_info *pi = netdev_priv(dev);
1118 /* accommodate SACK */
1122 ret = t4vf_set_rxmode(pi->adapter, pi->viid, new_mtu,
1123 -1, -1, -1, -1, true);
1129 static netdev_features_t cxgb4vf_fix_features(struct net_device *dev,
1130 netdev_features_t features)
1133 * Since there is no support for separate rx/tx vlan accel
1134 * enable/disable make sure tx flag is always in same state as rx.
1136 if (features & NETIF_F_HW_VLAN_CTAG_RX)
1137 features |= NETIF_F_HW_VLAN_CTAG_TX;
1139 features &= ~NETIF_F_HW_VLAN_CTAG_TX;
1144 static int cxgb4vf_set_features(struct net_device *dev,
1145 netdev_features_t features)
1147 struct port_info *pi = netdev_priv(dev);
1148 netdev_features_t changed = dev->features ^ features;
1150 if (changed & NETIF_F_HW_VLAN_CTAG_RX)
1151 t4vf_set_rxmode(pi->adapter, pi->viid, -1, -1, -1, -1,
1152 features & NETIF_F_HW_VLAN_CTAG_TX, 0);
1158 * Change the devices MAC address.
1160 static int cxgb4vf_set_mac_addr(struct net_device *dev, void *_addr)
1163 struct sockaddr *addr = _addr;
1164 struct port_info *pi = netdev_priv(dev);
1166 if (!is_valid_ether_addr(addr->sa_data))
1167 return -EADDRNOTAVAIL;
1169 ret = t4vf_change_mac(pi->adapter, pi->viid, pi->xact_addr_filt,
1170 addr->sa_data, true);
1174 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
1175 pi->xact_addr_filt = ret;
1179 #ifdef CONFIG_NET_POLL_CONTROLLER
1181 * Poll all of our receive queues. This is called outside of normal interrupt
1184 static void cxgb4vf_poll_controller(struct net_device *dev)
1186 struct port_info *pi = netdev_priv(dev);
1187 struct adapter *adapter = pi->adapter;
1189 if (adapter->flags & USING_MSIX) {
1190 struct sge_eth_rxq *rxq;
1193 rxq = &adapter->sge.ethrxq[pi->first_qset];
1194 for (nqsets = pi->nqsets; nqsets; nqsets--) {
1195 t4vf_sge_intr_msix(0, &rxq->rspq);
1199 t4vf_intr_handler(adapter)(0, adapter);
1204 * Ethtool operations.
1205 * ===================
1207 * Note that we don't support any ethtool operations which change the physical
1208 * state of the port to which we're linked.
1211 static unsigned int t4vf_from_fw_linkcaps(enum fw_port_type type,
1216 if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
1217 type == FW_PORT_TYPE_BT_XAUI) {
1219 if (caps & FW_PORT_CAP_SPEED_100M)
1220 v |= SUPPORTED_100baseT_Full;
1221 if (caps & FW_PORT_CAP_SPEED_1G)
1222 v |= SUPPORTED_1000baseT_Full;
1223 if (caps & FW_PORT_CAP_SPEED_10G)
1224 v |= SUPPORTED_10000baseT_Full;
1225 } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
1226 v |= SUPPORTED_Backplane;
1227 if (caps & FW_PORT_CAP_SPEED_1G)
1228 v |= SUPPORTED_1000baseKX_Full;
1229 if (caps & FW_PORT_CAP_SPEED_10G)
1230 v |= SUPPORTED_10000baseKX4_Full;
1231 } else if (type == FW_PORT_TYPE_KR)
1232 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
1233 else if (type == FW_PORT_TYPE_BP_AP)
1234 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
1235 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full;
1236 else if (type == FW_PORT_TYPE_BP4_AP)
1237 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
1238 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full |
1239 SUPPORTED_10000baseKX4_Full;
1240 else if (type == FW_PORT_TYPE_FIBER_XFI ||
1241 type == FW_PORT_TYPE_FIBER_XAUI ||
1242 type == FW_PORT_TYPE_SFP ||
1243 type == FW_PORT_TYPE_QSFP_10G ||
1244 type == FW_PORT_TYPE_QSA) {
1245 v |= SUPPORTED_FIBRE;
1246 if (caps & FW_PORT_CAP_SPEED_1G)
1247 v |= SUPPORTED_1000baseT_Full;
1248 if (caps & FW_PORT_CAP_SPEED_10G)
1249 v |= SUPPORTED_10000baseT_Full;
1250 } else if (type == FW_PORT_TYPE_BP40_BA ||
1251 type == FW_PORT_TYPE_QSFP) {
1252 v |= SUPPORTED_40000baseSR4_Full;
1253 v |= SUPPORTED_FIBRE;
1256 if (caps & FW_PORT_CAP_ANEG)
1257 v |= SUPPORTED_Autoneg;
1261 static int cxgb4vf_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1263 const struct port_info *p = netdev_priv(dev);
1265 if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
1266 p->port_type == FW_PORT_TYPE_BT_XFI ||
1267 p->port_type == FW_PORT_TYPE_BT_XAUI)
1268 cmd->port = PORT_TP;
1269 else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
1270 p->port_type == FW_PORT_TYPE_FIBER_XAUI)
1271 cmd->port = PORT_FIBRE;
1272 else if (p->port_type == FW_PORT_TYPE_SFP ||
1273 p->port_type == FW_PORT_TYPE_QSFP_10G ||
1274 p->port_type == FW_PORT_TYPE_QSA ||
1275 p->port_type == FW_PORT_TYPE_QSFP) {
1276 if (p->mod_type == FW_PORT_MOD_TYPE_LR ||
1277 p->mod_type == FW_PORT_MOD_TYPE_SR ||
1278 p->mod_type == FW_PORT_MOD_TYPE_ER ||
1279 p->mod_type == FW_PORT_MOD_TYPE_LRM)
1280 cmd->port = PORT_FIBRE;
1281 else if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
1282 p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
1283 cmd->port = PORT_DA;
1285 cmd->port = PORT_OTHER;
1287 cmd->port = PORT_OTHER;
1289 if (p->mdio_addr >= 0) {
1290 cmd->phy_address = p->mdio_addr;
1291 cmd->transceiver = XCVR_EXTERNAL;
1292 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
1293 MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
1295 cmd->phy_address = 0; /* not really, but no better option */
1296 cmd->transceiver = XCVR_INTERNAL;
1297 cmd->mdio_support = 0;
1300 cmd->supported = t4vf_from_fw_linkcaps(p->port_type,
1301 p->link_cfg.supported);
1302 cmd->advertising = t4vf_from_fw_linkcaps(p->port_type,
1303 p->link_cfg.advertising);
1304 ethtool_cmd_speed_set(cmd,
1305 netif_carrier_ok(dev) ? p->link_cfg.speed : 0);
1306 cmd->duplex = DUPLEX_FULL;
1307 cmd->autoneg = p->link_cfg.autoneg;
1314 * Return our driver information.
1316 static void cxgb4vf_get_drvinfo(struct net_device *dev,
1317 struct ethtool_drvinfo *drvinfo)
1319 struct adapter *adapter = netdev2adap(dev);
1321 strlcpy(drvinfo->driver, KBUILD_MODNAME, sizeof(drvinfo->driver));
1322 strlcpy(drvinfo->version, DRV_VERSION, sizeof(drvinfo->version));
1323 strlcpy(drvinfo->bus_info, pci_name(to_pci_dev(dev->dev.parent)),
1324 sizeof(drvinfo->bus_info));
1325 snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version),
1326 "%u.%u.%u.%u, TP %u.%u.%u.%u",
1327 FW_HDR_FW_VER_MAJOR_G(adapter->params.dev.fwrev),
1328 FW_HDR_FW_VER_MINOR_G(adapter->params.dev.fwrev),
1329 FW_HDR_FW_VER_MICRO_G(adapter->params.dev.fwrev),
1330 FW_HDR_FW_VER_BUILD_G(adapter->params.dev.fwrev),
1331 FW_HDR_FW_VER_MAJOR_G(adapter->params.dev.tprev),
1332 FW_HDR_FW_VER_MINOR_G(adapter->params.dev.tprev),
1333 FW_HDR_FW_VER_MICRO_G(adapter->params.dev.tprev),
1334 FW_HDR_FW_VER_BUILD_G(adapter->params.dev.tprev));
1338 * Return current adapter message level.
1340 static u32 cxgb4vf_get_msglevel(struct net_device *dev)
1342 return netdev2adap(dev)->msg_enable;
1346 * Set current adapter message level.
1348 static void cxgb4vf_set_msglevel(struct net_device *dev, u32 msglevel)
1350 netdev2adap(dev)->msg_enable = msglevel;
1354 * Return the device's current Queue Set ring size parameters along with the
1355 * allowed maximum values. Since ethtool doesn't understand the concept of
1356 * multi-queue devices, we just return the current values associated with the
1359 static void cxgb4vf_get_ringparam(struct net_device *dev,
1360 struct ethtool_ringparam *rp)
1362 const struct port_info *pi = netdev_priv(dev);
1363 const struct sge *s = &pi->adapter->sge;
1365 rp->rx_max_pending = MAX_RX_BUFFERS;
1366 rp->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
1367 rp->rx_jumbo_max_pending = 0;
1368 rp->tx_max_pending = MAX_TXQ_ENTRIES;
1370 rp->rx_pending = s->ethrxq[pi->first_qset].fl.size - MIN_FL_RESID;
1371 rp->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
1372 rp->rx_jumbo_pending = 0;
1373 rp->tx_pending = s->ethtxq[pi->first_qset].q.size;
1377 * Set the Queue Set ring size parameters for the device. Again, since
1378 * ethtool doesn't allow for the concept of multiple queues per device, we'll
1379 * apply these new values across all of the Queue Sets associated with the
1380 * device -- after vetting them of course!
1382 static int cxgb4vf_set_ringparam(struct net_device *dev,
1383 struct ethtool_ringparam *rp)
1385 const struct port_info *pi = netdev_priv(dev);
1386 struct adapter *adapter = pi->adapter;
1387 struct sge *s = &adapter->sge;
1390 if (rp->rx_pending > MAX_RX_BUFFERS ||
1391 rp->rx_jumbo_pending ||
1392 rp->tx_pending > MAX_TXQ_ENTRIES ||
1393 rp->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1394 rp->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1395 rp->rx_pending < MIN_FL_ENTRIES ||
1396 rp->tx_pending < MIN_TXQ_ENTRIES)
1399 if (adapter->flags & FULL_INIT_DONE)
1402 for (qs = pi->first_qset; qs < pi->first_qset + pi->nqsets; qs++) {
1403 s->ethrxq[qs].fl.size = rp->rx_pending + MIN_FL_RESID;
1404 s->ethrxq[qs].rspq.size = rp->rx_mini_pending;
1405 s->ethtxq[qs].q.size = rp->tx_pending;
1411 * Return the interrupt holdoff timer and count for the first Queue Set on the
1412 * device. Our extension ioctl() (the cxgbtool interface) allows the
1413 * interrupt holdoff timer to be read on all of the device's Queue Sets.
1415 static int cxgb4vf_get_coalesce(struct net_device *dev,
1416 struct ethtool_coalesce *coalesce)
1418 const struct port_info *pi = netdev_priv(dev);
1419 const struct adapter *adapter = pi->adapter;
1420 const struct sge_rspq *rspq = &adapter->sge.ethrxq[pi->first_qset].rspq;
1422 coalesce->rx_coalesce_usecs = qtimer_val(adapter, rspq);
1423 coalesce->rx_max_coalesced_frames =
1424 ((rspq->intr_params & QINTR_CNT_EN_F)
1425 ? adapter->sge.counter_val[rspq->pktcnt_idx]
1431 * Set the RX interrupt holdoff timer and count for the first Queue Set on the
1432 * interface. Our extension ioctl() (the cxgbtool interface) allows us to set
1433 * the interrupt holdoff timer on any of the device's Queue Sets.
1435 static int cxgb4vf_set_coalesce(struct net_device *dev,
1436 struct ethtool_coalesce *coalesce)
1438 const struct port_info *pi = netdev_priv(dev);
1439 struct adapter *adapter = pi->adapter;
1441 return set_rxq_intr_params(adapter,
1442 &adapter->sge.ethrxq[pi->first_qset].rspq,
1443 coalesce->rx_coalesce_usecs,
1444 coalesce->rx_max_coalesced_frames);
1448 * Report current port link pause parameter settings.
1450 static void cxgb4vf_get_pauseparam(struct net_device *dev,
1451 struct ethtool_pauseparam *pauseparam)
1453 struct port_info *pi = netdev_priv(dev);
1455 pauseparam->autoneg = (pi->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
1456 pauseparam->rx_pause = (pi->link_cfg.fc & PAUSE_RX) != 0;
1457 pauseparam->tx_pause = (pi->link_cfg.fc & PAUSE_TX) != 0;
1461 * Identify the port by blinking the port's LED.
1463 static int cxgb4vf_phys_id(struct net_device *dev,
1464 enum ethtool_phys_id_state state)
1467 struct port_info *pi = netdev_priv(dev);
1469 if (state == ETHTOOL_ID_ACTIVE)
1471 else if (state == ETHTOOL_ID_INACTIVE)
1476 return t4vf_identify_port(pi->adapter, pi->viid, val);
1480 * Port stats maintained per queue of the port.
1482 struct queue_port_stats {
1493 * Strings for the ETH_SS_STATS statistics set ("ethtool -S"). Note that
1494 * these need to match the order of statistics returned by
1495 * t4vf_get_port_stats().
1497 static const char stats_strings[][ETH_GSTRING_LEN] = {
1499 * These must match the layout of the t4vf_port_stats structure.
1501 "TxBroadcastBytes ",
1502 "TxBroadcastFrames ",
1503 "TxMulticastBytes ",
1504 "TxMulticastFrames ",
1510 "RxBroadcastBytes ",
1511 "RxBroadcastFrames ",
1512 "RxMulticastBytes ",
1513 "RxMulticastFrames ",
1519 * These are accumulated per-queue statistics and must match the
1520 * order of the fields in the queue_port_stats structure.
1532 * Return the number of statistics in the specified statistics set.
1534 static int cxgb4vf_get_sset_count(struct net_device *dev, int sset)
1538 return ARRAY_SIZE(stats_strings);
1546 * Return the strings for the specified statistics set.
1548 static void cxgb4vf_get_strings(struct net_device *dev,
1554 memcpy(data, stats_strings, sizeof(stats_strings));
1560 * Small utility routine to accumulate queue statistics across the queues of
1563 static void collect_sge_port_stats(const struct adapter *adapter,
1564 const struct port_info *pi,
1565 struct queue_port_stats *stats)
1567 const struct sge_eth_txq *txq = &adapter->sge.ethtxq[pi->first_qset];
1568 const struct sge_eth_rxq *rxq = &adapter->sge.ethrxq[pi->first_qset];
1571 memset(stats, 0, sizeof(*stats));
1572 for (qs = 0; qs < pi->nqsets; qs++, rxq++, txq++) {
1573 stats->tso += txq->tso;
1574 stats->tx_csum += txq->tx_cso;
1575 stats->rx_csum += rxq->stats.rx_cso;
1576 stats->vlan_ex += rxq->stats.vlan_ex;
1577 stats->vlan_ins += txq->vlan_ins;
1578 stats->lro_pkts += rxq->stats.lro_pkts;
1579 stats->lro_merged += rxq->stats.lro_merged;
1584 * Return the ETH_SS_STATS statistics set.
1586 static void cxgb4vf_get_ethtool_stats(struct net_device *dev,
1587 struct ethtool_stats *stats,
1590 struct port_info *pi = netdev2pinfo(dev);
1591 struct adapter *adapter = pi->adapter;
1592 int err = t4vf_get_port_stats(adapter, pi->pidx,
1593 (struct t4vf_port_stats *)data);
1595 memset(data, 0, sizeof(struct t4vf_port_stats));
1597 data += sizeof(struct t4vf_port_stats) / sizeof(u64);
1598 collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1602 * Return the size of our register map.
1604 static int cxgb4vf_get_regs_len(struct net_device *dev)
1606 return T4VF_REGMAP_SIZE;
1610 * Dump a block of registers, start to end inclusive, into a buffer.
1612 static void reg_block_dump(struct adapter *adapter, void *regbuf,
1613 unsigned int start, unsigned int end)
1615 u32 *bp = regbuf + start - T4VF_REGMAP_START;
1617 for ( ; start <= end; start += sizeof(u32)) {
1619 * Avoid reading the Mailbox Control register since that
1620 * can trigger a Mailbox Ownership Arbitration cycle and
1621 * interfere with communication with the firmware.
1623 if (start == T4VF_CIM_BASE_ADDR + CIM_VF_EXT_MAILBOX_CTRL)
1626 *bp++ = t4_read_reg(adapter, start);
1631 * Copy our entire register map into the provided buffer.
1633 static void cxgb4vf_get_regs(struct net_device *dev,
1634 struct ethtool_regs *regs,
1637 struct adapter *adapter = netdev2adap(dev);
1639 regs->version = mk_adap_vers(adapter);
1642 * Fill in register buffer with our register map.
1644 memset(regbuf, 0, T4VF_REGMAP_SIZE);
1646 reg_block_dump(adapter, regbuf,
1647 T4VF_SGE_BASE_ADDR + T4VF_MOD_MAP_SGE_FIRST,
1648 T4VF_SGE_BASE_ADDR + T4VF_MOD_MAP_SGE_LAST);
1649 reg_block_dump(adapter, regbuf,
1650 T4VF_MPS_BASE_ADDR + T4VF_MOD_MAP_MPS_FIRST,
1651 T4VF_MPS_BASE_ADDR + T4VF_MOD_MAP_MPS_LAST);
1653 /* T5 adds new registers in the PL Register map.
1655 reg_block_dump(adapter, regbuf,
1656 T4VF_PL_BASE_ADDR + T4VF_MOD_MAP_PL_FIRST,
1657 T4VF_PL_BASE_ADDR + (is_t4(adapter->params.chip)
1658 ? PL_VF_WHOAMI_A : PL_VF_REVISION_A));
1659 reg_block_dump(adapter, regbuf,
1660 T4VF_CIM_BASE_ADDR + T4VF_MOD_MAP_CIM_FIRST,
1661 T4VF_CIM_BASE_ADDR + T4VF_MOD_MAP_CIM_LAST);
1663 reg_block_dump(adapter, regbuf,
1664 T4VF_MBDATA_BASE_ADDR + T4VF_MBDATA_FIRST,
1665 T4VF_MBDATA_BASE_ADDR + T4VF_MBDATA_LAST);
1669 * Report current Wake On LAN settings.
1671 static void cxgb4vf_get_wol(struct net_device *dev,
1672 struct ethtool_wolinfo *wol)
1676 memset(&wol->sopass, 0, sizeof(wol->sopass));
1680 * TCP Segmentation Offload flags which we support.
1682 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
1684 static const struct ethtool_ops cxgb4vf_ethtool_ops = {
1685 .get_settings = cxgb4vf_get_settings,
1686 .get_drvinfo = cxgb4vf_get_drvinfo,
1687 .get_msglevel = cxgb4vf_get_msglevel,
1688 .set_msglevel = cxgb4vf_set_msglevel,
1689 .get_ringparam = cxgb4vf_get_ringparam,
1690 .set_ringparam = cxgb4vf_set_ringparam,
1691 .get_coalesce = cxgb4vf_get_coalesce,
1692 .set_coalesce = cxgb4vf_set_coalesce,
1693 .get_pauseparam = cxgb4vf_get_pauseparam,
1694 .get_link = ethtool_op_get_link,
1695 .get_strings = cxgb4vf_get_strings,
1696 .set_phys_id = cxgb4vf_phys_id,
1697 .get_sset_count = cxgb4vf_get_sset_count,
1698 .get_ethtool_stats = cxgb4vf_get_ethtool_stats,
1699 .get_regs_len = cxgb4vf_get_regs_len,
1700 .get_regs = cxgb4vf_get_regs,
1701 .get_wol = cxgb4vf_get_wol,
1705 * /sys/kernel/debug/cxgb4vf support code and data.
1706 * ================================================
1710 * Show SGE Queue Set information. We display QPL Queues Sets per line.
1714 static int sge_qinfo_show(struct seq_file *seq, void *v)
1716 struct adapter *adapter = seq->private;
1717 int eth_entries = DIV_ROUND_UP(adapter->sge.ethqsets, QPL);
1718 int qs, r = (uintptr_t)v - 1;
1721 seq_putc(seq, '\n');
1723 #define S3(fmt_spec, s, v) \
1725 seq_printf(seq, "%-12s", s); \
1726 for (qs = 0; qs < n; ++qs) \
1727 seq_printf(seq, " %16" fmt_spec, v); \
1728 seq_putc(seq, '\n'); \
1730 #define S(s, v) S3("s", s, v)
1731 #define T(s, v) S3("u", s, txq[qs].v)
1732 #define R(s, v) S3("u", s, rxq[qs].v)
1734 if (r < eth_entries) {
1735 const struct sge_eth_rxq *rxq = &adapter->sge.ethrxq[r * QPL];
1736 const struct sge_eth_txq *txq = &adapter->sge.ethtxq[r * QPL];
1737 int n = min(QPL, adapter->sge.ethqsets - QPL * r);
1739 S("QType:", "Ethernet");
1741 (rxq[qs].rspq.netdev
1742 ? rxq[qs].rspq.netdev->name
1745 (rxq[qs].rspq.netdev
1746 ? ((struct port_info *)
1747 netdev_priv(rxq[qs].rspq.netdev))->port_id
1749 T("TxQ ID:", q.abs_id);
1750 T("TxQ size:", q.size);
1751 T("TxQ inuse:", q.in_use);
1752 T("TxQ PIdx:", q.pidx);
1753 T("TxQ CIdx:", q.cidx);
1754 R("RspQ ID:", rspq.abs_id);
1755 R("RspQ size:", rspq.size);
1756 R("RspQE size:", rspq.iqe_len);
1757 S3("u", "Intr delay:", qtimer_val(adapter, &rxq[qs].rspq));
1758 S3("u", "Intr pktcnt:",
1759 adapter->sge.counter_val[rxq[qs].rspq.pktcnt_idx]);
1760 R("RspQ CIdx:", rspq.cidx);
1761 R("RspQ Gen:", rspq.gen);
1762 R("FL ID:", fl.abs_id);
1763 R("FL size:", fl.size - MIN_FL_RESID);
1764 R("FL avail:", fl.avail);
1765 R("FL PIdx:", fl.pidx);
1766 R("FL CIdx:", fl.cidx);
1772 const struct sge_rspq *evtq = &adapter->sge.fw_evtq;
1774 seq_printf(seq, "%-12s %16s\n", "QType:", "FW event queue");
1775 seq_printf(seq, "%-12s %16u\n", "RspQ ID:", evtq->abs_id);
1776 seq_printf(seq, "%-12s %16u\n", "Intr delay:",
1777 qtimer_val(adapter, evtq));
1778 seq_printf(seq, "%-12s %16u\n", "Intr pktcnt:",
1779 adapter->sge.counter_val[evtq->pktcnt_idx]);
1780 seq_printf(seq, "%-12s %16u\n", "RspQ Cidx:", evtq->cidx);
1781 seq_printf(seq, "%-12s %16u\n", "RspQ Gen:", evtq->gen);
1782 } else if (r == 1) {
1783 const struct sge_rspq *intrq = &adapter->sge.intrq;
1785 seq_printf(seq, "%-12s %16s\n", "QType:", "Interrupt Queue");
1786 seq_printf(seq, "%-12s %16u\n", "RspQ ID:", intrq->abs_id);
1787 seq_printf(seq, "%-12s %16u\n", "Intr delay:",
1788 qtimer_val(adapter, intrq));
1789 seq_printf(seq, "%-12s %16u\n", "Intr pktcnt:",
1790 adapter->sge.counter_val[intrq->pktcnt_idx]);
1791 seq_printf(seq, "%-12s %16u\n", "RspQ Cidx:", intrq->cidx);
1792 seq_printf(seq, "%-12s %16u\n", "RspQ Gen:", intrq->gen);
1804 * Return the number of "entries" in our "file". We group the multi-Queue
1805 * sections with QPL Queue Sets per "entry". The sections of the output are:
1807 * Ethernet RX/TX Queue Sets
1808 * Firmware Event Queue
1809 * Forwarded Interrupt Queue (if in MSI mode)
1811 static int sge_queue_entries(const struct adapter *adapter)
1813 return DIV_ROUND_UP(adapter->sge.ethqsets, QPL) + 1 +
1814 ((adapter->flags & USING_MSI) != 0);
1817 static void *sge_queue_start(struct seq_file *seq, loff_t *pos)
1819 int entries = sge_queue_entries(seq->private);
1821 return *pos < entries ? (void *)((uintptr_t)*pos + 1) : NULL;
1824 static void sge_queue_stop(struct seq_file *seq, void *v)
1828 static void *sge_queue_next(struct seq_file *seq, void *v, loff_t *pos)
1830 int entries = sge_queue_entries(seq->private);
1833 return *pos < entries ? (void *)((uintptr_t)*pos + 1) : NULL;
1836 static const struct seq_operations sge_qinfo_seq_ops = {
1837 .start = sge_queue_start,
1838 .next = sge_queue_next,
1839 .stop = sge_queue_stop,
1840 .show = sge_qinfo_show
1843 static int sge_qinfo_open(struct inode *inode, struct file *file)
1845 int res = seq_open(file, &sge_qinfo_seq_ops);
1848 struct seq_file *seq = file->private_data;
1849 seq->private = inode->i_private;
1854 static const struct file_operations sge_qinfo_debugfs_fops = {
1855 .owner = THIS_MODULE,
1856 .open = sge_qinfo_open,
1858 .llseek = seq_lseek,
1859 .release = seq_release,
1863 * Show SGE Queue Set statistics. We display QPL Queues Sets per line.
1867 static int sge_qstats_show(struct seq_file *seq, void *v)
1869 struct adapter *adapter = seq->private;
1870 int eth_entries = DIV_ROUND_UP(adapter->sge.ethqsets, QPL);
1871 int qs, r = (uintptr_t)v - 1;
1874 seq_putc(seq, '\n');
1876 #define S3(fmt, s, v) \
1878 seq_printf(seq, "%-16s", s); \
1879 for (qs = 0; qs < n; ++qs) \
1880 seq_printf(seq, " %8" fmt, v); \
1881 seq_putc(seq, '\n'); \
1883 #define S(s, v) S3("s", s, v)
1885 #define T3(fmt, s, v) S3(fmt, s, txq[qs].v)
1886 #define T(s, v) T3("lu", s, v)
1888 #define R3(fmt, s, v) S3(fmt, s, rxq[qs].v)
1889 #define R(s, v) R3("lu", s, v)
1891 if (r < eth_entries) {
1892 const struct sge_eth_rxq *rxq = &adapter->sge.ethrxq[r * QPL];
1893 const struct sge_eth_txq *txq = &adapter->sge.ethtxq[r * QPL];
1894 int n = min(QPL, adapter->sge.ethqsets - QPL * r);
1896 S("QType:", "Ethernet");
1898 (rxq[qs].rspq.netdev
1899 ? rxq[qs].rspq.netdev->name
1901 R3("u", "RspQNullInts:", rspq.unhandled_irqs);
1902 R("RxPackets:", stats.pkts);
1903 R("RxCSO:", stats.rx_cso);
1904 R("VLANxtract:", stats.vlan_ex);
1905 R("LROmerged:", stats.lro_merged);
1906 R("LROpackets:", stats.lro_pkts);
1907 R("RxDrops:", stats.rx_drops);
1909 T("TxCSO:", tx_cso);
1910 T("VLANins:", vlan_ins);
1911 T("TxQFull:", q.stops);
1912 T("TxQRestarts:", q.restarts);
1913 T("TxMapErr:", mapping_err);
1914 R("FLAllocErr:", fl.alloc_failed);
1915 R("FLLrgAlcErr:", fl.large_alloc_failed);
1916 R("FLStarving:", fl.starving);
1922 const struct sge_rspq *evtq = &adapter->sge.fw_evtq;
1924 seq_printf(seq, "%-8s %16s\n", "QType:", "FW event queue");
1925 seq_printf(seq, "%-16s %8u\n", "RspQNullInts:",
1926 evtq->unhandled_irqs);
1927 seq_printf(seq, "%-16s %8u\n", "RspQ CIdx:", evtq->cidx);
1928 seq_printf(seq, "%-16s %8u\n", "RspQ Gen:", evtq->gen);
1929 } else if (r == 1) {
1930 const struct sge_rspq *intrq = &adapter->sge.intrq;
1932 seq_printf(seq, "%-8s %16s\n", "QType:", "Interrupt Queue");
1933 seq_printf(seq, "%-16s %8u\n", "RspQNullInts:",
1934 intrq->unhandled_irqs);
1935 seq_printf(seq, "%-16s %8u\n", "RspQ CIdx:", intrq->cidx);
1936 seq_printf(seq, "%-16s %8u\n", "RspQ Gen:", intrq->gen);
1950 * Return the number of "entries" in our "file". We group the multi-Queue
1951 * sections with QPL Queue Sets per "entry". The sections of the output are:
1953 * Ethernet RX/TX Queue Sets
1954 * Firmware Event Queue
1955 * Forwarded Interrupt Queue (if in MSI mode)
1957 static int sge_qstats_entries(const struct adapter *adapter)
1959 return DIV_ROUND_UP(adapter->sge.ethqsets, QPL) + 1 +
1960 ((adapter->flags & USING_MSI) != 0);
1963 static void *sge_qstats_start(struct seq_file *seq, loff_t *pos)
1965 int entries = sge_qstats_entries(seq->private);
1967 return *pos < entries ? (void *)((uintptr_t)*pos + 1) : NULL;
1970 static void sge_qstats_stop(struct seq_file *seq, void *v)
1974 static void *sge_qstats_next(struct seq_file *seq, void *v, loff_t *pos)
1976 int entries = sge_qstats_entries(seq->private);
1979 return *pos < entries ? (void *)((uintptr_t)*pos + 1) : NULL;
1982 static const struct seq_operations sge_qstats_seq_ops = {
1983 .start = sge_qstats_start,
1984 .next = sge_qstats_next,
1985 .stop = sge_qstats_stop,
1986 .show = sge_qstats_show
1989 static int sge_qstats_open(struct inode *inode, struct file *file)
1991 int res = seq_open(file, &sge_qstats_seq_ops);
1994 struct seq_file *seq = file->private_data;
1995 seq->private = inode->i_private;
2000 static const struct file_operations sge_qstats_proc_fops = {
2001 .owner = THIS_MODULE,
2002 .open = sge_qstats_open,
2004 .llseek = seq_lseek,
2005 .release = seq_release,
2009 * Show PCI-E SR-IOV Virtual Function Resource Limits.
2011 static int resources_show(struct seq_file *seq, void *v)
2013 struct adapter *adapter = seq->private;
2014 struct vf_resources *vfres = &adapter->params.vfres;
2016 #define S(desc, fmt, var) \
2017 seq_printf(seq, "%-60s " fmt "\n", \
2018 desc " (" #var "):", vfres->var)
2020 S("Virtual Interfaces", "%d", nvi);
2021 S("Egress Queues", "%d", neq);
2022 S("Ethernet Control", "%d", nethctrl);
2023 S("Ingress Queues/w Free Lists/Interrupts", "%d", niqflint);
2024 S("Ingress Queues", "%d", niq);
2025 S("Traffic Class", "%d", tc);
2026 S("Port Access Rights Mask", "%#x", pmask);
2027 S("MAC Address Filters", "%d", nexactf);
2028 S("Firmware Command Read Capabilities", "%#x", r_caps);
2029 S("Firmware Command Write/Execute Capabilities", "%#x", wx_caps);
2036 static int resources_open(struct inode *inode, struct file *file)
2038 return single_open(file, resources_show, inode->i_private);
2041 static const struct file_operations resources_proc_fops = {
2042 .owner = THIS_MODULE,
2043 .open = resources_open,
2045 .llseek = seq_lseek,
2046 .release = single_release,
2050 * Show Virtual Interfaces.
2052 static int interfaces_show(struct seq_file *seq, void *v)
2054 if (v == SEQ_START_TOKEN) {
2055 seq_puts(seq, "Interface Port VIID\n");
2057 struct adapter *adapter = seq->private;
2058 int pidx = (uintptr_t)v - 2;
2059 struct net_device *dev = adapter->port[pidx];
2060 struct port_info *pi = netdev_priv(dev);
2062 seq_printf(seq, "%9s %4d %#5x\n",
2063 dev->name, pi->port_id, pi->viid);
2068 static inline void *interfaces_get_idx(struct adapter *adapter, loff_t pos)
2070 return pos <= adapter->params.nports
2071 ? (void *)(uintptr_t)(pos + 1)
2075 static void *interfaces_start(struct seq_file *seq, loff_t *pos)
2078 ? interfaces_get_idx(seq->private, *pos)
2082 static void *interfaces_next(struct seq_file *seq, void *v, loff_t *pos)
2085 return interfaces_get_idx(seq->private, *pos);
2088 static void interfaces_stop(struct seq_file *seq, void *v)
2092 static const struct seq_operations interfaces_seq_ops = {
2093 .start = interfaces_start,
2094 .next = interfaces_next,
2095 .stop = interfaces_stop,
2096 .show = interfaces_show
2099 static int interfaces_open(struct inode *inode, struct file *file)
2101 int res = seq_open(file, &interfaces_seq_ops);
2104 struct seq_file *seq = file->private_data;
2105 seq->private = inode->i_private;
2110 static const struct file_operations interfaces_proc_fops = {
2111 .owner = THIS_MODULE,
2112 .open = interfaces_open,
2114 .llseek = seq_lseek,
2115 .release = seq_release,
2119 * /sys/kernel/debugfs/cxgb4vf/ files list.
2121 struct cxgb4vf_debugfs_entry {
2122 const char *name; /* name of debugfs node */
2123 umode_t mode; /* file system mode */
2124 const struct file_operations *fops;
2127 static struct cxgb4vf_debugfs_entry debugfs_files[] = {
2128 { "sge_qinfo", S_IRUGO, &sge_qinfo_debugfs_fops },
2129 { "sge_qstats", S_IRUGO, &sge_qstats_proc_fops },
2130 { "resources", S_IRUGO, &resources_proc_fops },
2131 { "interfaces", S_IRUGO, &interfaces_proc_fops },
2135 * Module and device initialization and cleanup code.
2136 * ==================================================
2140 * Set up out /sys/kernel/debug/cxgb4vf sub-nodes. We assume that the
2141 * directory (debugfs_root) has already been set up.
2143 static int setup_debugfs(struct adapter *adapter)
2147 BUG_ON(IS_ERR_OR_NULL(adapter->debugfs_root));
2150 * Debugfs support is best effort.
2152 for (i = 0; i < ARRAY_SIZE(debugfs_files); i++)
2153 (void)debugfs_create_file(debugfs_files[i].name,
2154 debugfs_files[i].mode,
2155 adapter->debugfs_root,
2157 debugfs_files[i].fops);
2163 * Tear down the /sys/kernel/debug/cxgb4vf sub-nodes created above. We leave
2164 * it to our caller to tear down the directory (debugfs_root).
2166 static void cleanup_debugfs(struct adapter *adapter)
2168 BUG_ON(IS_ERR_OR_NULL(adapter->debugfs_root));
2171 * Unlike our sister routine cleanup_proc(), we don't need to remove
2172 * individual entries because a call will be made to
2173 * debugfs_remove_recursive(). We just need to clean up any ancillary
2180 * Perform early "adapter" initialization. This is where we discover what
2181 * adapter parameters we're going to be using and initialize basic adapter
2184 static int adap_init0(struct adapter *adapter)
2186 struct vf_resources *vfres = &adapter->params.vfres;
2187 struct sge_params *sge_params = &adapter->params.sge;
2188 struct sge *s = &adapter->sge;
2189 unsigned int ethqsets;
2194 * Some environments do not properly handle PCIE FLRs -- e.g. in Linux
2195 * 2.6.31 and later we can't call pci_reset_function() in order to
2196 * issue an FLR because of a self- deadlock on the device semaphore.
2197 * Meanwhile, the OS infrastructure doesn't issue FLRs in all the
2198 * cases where they're needed -- for instance, some versions of KVM
2199 * fail to reset "Assigned Devices" when the VM reboots. Therefore we
2200 * use the firmware based reset in order to reset any per function
2203 err = t4vf_fw_reset(adapter);
2205 dev_err(adapter->pdev_dev, "FW reset failed: err=%d\n", err);
2210 * Grab basic operational parameters. These will predominantly have
2211 * been set up by the Physical Function Driver or will be hard coded
2212 * into the adapter. We just have to live with them ... Note that
2213 * we _must_ get our VPD parameters before our SGE parameters because
2214 * we need to know the adapter's core clock from the VPD in order to
2215 * properly decode the SGE Timer Values.
2217 err = t4vf_get_dev_params(adapter);
2219 dev_err(adapter->pdev_dev, "unable to retrieve adapter"
2220 " device parameters: err=%d\n", err);
2223 err = t4vf_get_vpd_params(adapter);
2225 dev_err(adapter->pdev_dev, "unable to retrieve adapter"
2226 " VPD parameters: err=%d\n", err);
2229 err = t4vf_get_sge_params(adapter);
2231 dev_err(adapter->pdev_dev, "unable to retrieve adapter"
2232 " SGE parameters: err=%d\n", err);
2235 err = t4vf_get_rss_glb_config(adapter);
2237 dev_err(adapter->pdev_dev, "unable to retrieve adapter"
2238 " RSS parameters: err=%d\n", err);
2241 if (adapter->params.rss.mode !=
2242 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL) {
2243 dev_err(adapter->pdev_dev, "unable to operate with global RSS"
2244 " mode %d\n", adapter->params.rss.mode);
2247 err = t4vf_sge_init(adapter);
2249 dev_err(adapter->pdev_dev, "unable to use adapter parameters:"
2254 /* If we're running on newer firmware, let it know that we're
2255 * prepared to deal with encapsulated CPL messages. Older
2256 * firmware won't understand this and we'll just get
2257 * unencapsulated messages ...
2259 param = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) |
2260 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_CPLFW4MSG_ENCAP);
2262 (void) t4vf_set_params(adapter, 1, ¶m, &val);
2265 * Retrieve our RX interrupt holdoff timer values and counter
2266 * threshold values from the SGE parameters.
2268 s->timer_val[0] = core_ticks_to_us(adapter,
2269 TIMERVALUE0_G(sge_params->sge_timer_value_0_and_1));
2270 s->timer_val[1] = core_ticks_to_us(adapter,
2271 TIMERVALUE1_G(sge_params->sge_timer_value_0_and_1));
2272 s->timer_val[2] = core_ticks_to_us(adapter,
2273 TIMERVALUE0_G(sge_params->sge_timer_value_2_and_3));
2274 s->timer_val[3] = core_ticks_to_us(adapter,
2275 TIMERVALUE1_G(sge_params->sge_timer_value_2_and_3));
2276 s->timer_val[4] = core_ticks_to_us(adapter,
2277 TIMERVALUE0_G(sge_params->sge_timer_value_4_and_5));
2278 s->timer_val[5] = core_ticks_to_us(adapter,
2279 TIMERVALUE1_G(sge_params->sge_timer_value_4_and_5));
2281 s->counter_val[0] = THRESHOLD_0_G(sge_params->sge_ingress_rx_threshold);
2282 s->counter_val[1] = THRESHOLD_1_G(sge_params->sge_ingress_rx_threshold);
2283 s->counter_val[2] = THRESHOLD_2_G(sge_params->sge_ingress_rx_threshold);
2284 s->counter_val[3] = THRESHOLD_3_G(sge_params->sge_ingress_rx_threshold);
2287 * Grab our Virtual Interface resource allocation, extract the
2288 * features that we're interested in and do a bit of sanity testing on
2291 err = t4vf_get_vfres(adapter);
2293 dev_err(adapter->pdev_dev, "unable to get virtual interface"
2294 " resources: err=%d\n", err);
2299 * The number of "ports" which we support is equal to the number of
2300 * Virtual Interfaces with which we've been provisioned.
2302 adapter->params.nports = vfres->nvi;
2303 if (adapter->params.nports > MAX_NPORTS) {
2304 dev_warn(adapter->pdev_dev, "only using %d of %d allowed"
2305 " virtual interfaces\n", MAX_NPORTS,
2306 adapter->params.nports);
2307 adapter->params.nports = MAX_NPORTS;
2311 * We need to reserve a number of the ingress queues with Free List
2312 * and Interrupt capabilities for special interrupt purposes (like
2313 * asynchronous firmware messages, or forwarded interrupts if we're
2314 * using MSI). The rest of the FL/Intr-capable ingress queues will be
2315 * matched up one-for-one with Ethernet/Control egress queues in order
2316 * to form "Queue Sets" which will be aportioned between the "ports".
2317 * For each Queue Set, we'll need the ability to allocate two Egress
2318 * Contexts -- one for the Ingress Queue Free List and one for the TX
2321 ethqsets = vfres->niqflint - INGQ_EXTRAS;
2322 if (vfres->nethctrl != ethqsets) {
2323 dev_warn(adapter->pdev_dev, "unequal number of [available]"
2324 " ingress/egress queues (%d/%d); using minimum for"
2325 " number of Queue Sets\n", ethqsets, vfres->nethctrl);
2326 ethqsets = min(vfres->nethctrl, ethqsets);
2328 if (vfres->neq < ethqsets*2) {
2329 dev_warn(adapter->pdev_dev, "Not enough Egress Contexts (%d)"
2330 " to support Queue Sets (%d); reducing allowed Queue"
2331 " Sets\n", vfres->neq, ethqsets);
2332 ethqsets = vfres->neq/2;
2334 if (ethqsets > MAX_ETH_QSETS) {
2335 dev_warn(adapter->pdev_dev, "only using %d of %d allowed Queue"
2336 " Sets\n", MAX_ETH_QSETS, adapter->sge.max_ethqsets);
2337 ethqsets = MAX_ETH_QSETS;
2339 if (vfres->niq != 0 || vfres->neq > ethqsets*2) {
2340 dev_warn(adapter->pdev_dev, "unused resources niq/neq (%d/%d)"
2341 " ignored\n", vfres->niq, vfres->neq - ethqsets*2);
2343 adapter->sge.max_ethqsets = ethqsets;
2346 * Check for various parameter sanity issues. Most checks simply
2347 * result in us using fewer resources than our provissioning but we
2348 * do need at least one "port" with which to work ...
2350 if (adapter->sge.max_ethqsets < adapter->params.nports) {
2351 dev_warn(adapter->pdev_dev, "only using %d of %d available"
2352 " virtual interfaces (too few Queue Sets)\n",
2353 adapter->sge.max_ethqsets, adapter->params.nports);
2354 adapter->params.nports = adapter->sge.max_ethqsets;
2356 if (adapter->params.nports == 0) {
2357 dev_err(adapter->pdev_dev, "no virtual interfaces configured/"
2364 static inline void init_rspq(struct sge_rspq *rspq, u8 timer_idx,
2365 u8 pkt_cnt_idx, unsigned int size,
2366 unsigned int iqe_size)
2368 rspq->intr_params = (QINTR_TIMER_IDX_V(timer_idx) |
2369 (pkt_cnt_idx < SGE_NCOUNTERS ?
2370 QINTR_CNT_EN_F : 0));
2371 rspq->pktcnt_idx = (pkt_cnt_idx < SGE_NCOUNTERS
2374 rspq->iqe_len = iqe_size;
2379 * Perform default configuration of DMA queues depending on the number and
2380 * type of ports we found and the number of available CPUs. Most settings can
2381 * be modified by the admin via ethtool and cxgbtool prior to the adapter
2382 * being brought up for the first time.
2384 static void cfg_queues(struct adapter *adapter)
2386 struct sge *s = &adapter->sge;
2387 int q10g, n10g, qidx, pidx, qs;
2391 * We should not be called till we know how many Queue Sets we can
2392 * support. In particular, this means that we need to know what kind
2393 * of interrupts we'll be using ...
2395 BUG_ON((adapter->flags & (USING_MSIX|USING_MSI)) == 0);
2398 * Count the number of 10GbE Virtual Interfaces that we have.
2401 for_each_port(adapter, pidx)
2402 n10g += is_x_10g_port(&adap2pinfo(adapter, pidx)->link_cfg);
2405 * We default to 1 queue per non-10G port and up to # of cores queues
2411 int n1g = (adapter->params.nports - n10g);
2412 q10g = (adapter->sge.max_ethqsets - n1g) / n10g;
2413 if (q10g > num_online_cpus())
2414 q10g = num_online_cpus();
2418 * Allocate the "Queue Sets" to the various Virtual Interfaces.
2419 * The layout will be established in setup_sge_queues() when the
2420 * adapter is brough up for the first time.
2423 for_each_port(adapter, pidx) {
2424 struct port_info *pi = adap2pinfo(adapter, pidx);
2426 pi->first_qset = qidx;
2427 pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1;
2433 * The Ingress Queue Entry Size for our various Response Queues needs
2434 * to be big enough to accommodate the largest message we can receive
2435 * from the chip/firmware; which is 64 bytes ...
2440 * Set up default Queue Set parameters ... Start off with the
2441 * shortest interrupt holdoff timer.
2443 for (qs = 0; qs < s->max_ethqsets; qs++) {
2444 struct sge_eth_rxq *rxq = &s->ethrxq[qs];
2445 struct sge_eth_txq *txq = &s->ethtxq[qs];
2447 init_rspq(&rxq->rspq, 0, 0, 1024, iqe_size);
2453 * The firmware event queue is used for link state changes and
2454 * notifications of TX DMA completions.
2456 init_rspq(&s->fw_evtq, SGE_TIMER_RSTRT_CNTR, 0, 512, iqe_size);
2459 * The forwarded interrupt queue is used when we're in MSI interrupt
2460 * mode. In this mode all interrupts associated with RX queues will
2461 * be forwarded to a single queue which we'll associate with our MSI
2462 * interrupt vector. The messages dropped in the forwarded interrupt
2463 * queue will indicate which ingress queue needs servicing ... This
2464 * queue needs to be large enough to accommodate all of the ingress
2465 * queues which are forwarding their interrupt (+1 to prevent the PIDX
2466 * from equalling the CIDX if every ingress queue has an outstanding
2467 * interrupt). The queue doesn't need to be any larger because no
2468 * ingress queue will ever have more than one outstanding interrupt at
2471 init_rspq(&s->intrq, SGE_TIMER_RSTRT_CNTR, 0, MSIX_ENTRIES + 1,
2476 * Reduce the number of Ethernet queues across all ports to at most n.
2477 * n provides at least one queue per port.
2479 static void reduce_ethqs(struct adapter *adapter, int n)
2482 struct port_info *pi;
2485 * While we have too many active Ether Queue Sets, interate across the
2486 * "ports" and reduce their individual Queue Set allocations.
2488 BUG_ON(n < adapter->params.nports);
2489 while (n < adapter->sge.ethqsets)
2490 for_each_port(adapter, i) {
2491 pi = adap2pinfo(adapter, i);
2492 if (pi->nqsets > 1) {
2494 adapter->sge.ethqsets--;
2495 if (adapter->sge.ethqsets <= n)
2501 * Reassign the starting Queue Sets for each of the "ports" ...
2504 for_each_port(adapter, i) {
2505 pi = adap2pinfo(adapter, i);
2512 * We need to grab enough MSI-X vectors to cover our interrupt needs. Ideally
2513 * we get a separate MSI-X vector for every "Queue Set" plus any extras we
2514 * need. Minimally we need one for every Virtual Interface plus those needed
2515 * for our "extras". Note that this process may lower the maximum number of
2516 * allowed Queue Sets ...
2518 static int enable_msix(struct adapter *adapter)
2520 int i, want, need, nqsets;
2521 struct msix_entry entries[MSIX_ENTRIES];
2522 struct sge *s = &adapter->sge;
2524 for (i = 0; i < MSIX_ENTRIES; ++i)
2525 entries[i].entry = i;
2528 * We _want_ enough MSI-X interrupts to cover all of our "Queue Sets"
2529 * plus those needed for our "extras" (for example, the firmware
2530 * message queue). We _need_ at least one "Queue Set" per Virtual
2531 * Interface plus those needed for our "extras". So now we get to see
2532 * if the song is right ...
2534 want = s->max_ethqsets + MSIX_EXTRAS;
2535 need = adapter->params.nports + MSIX_EXTRAS;
2537 want = pci_enable_msix_range(adapter->pdev, entries, need, want);
2541 nqsets = want - MSIX_EXTRAS;
2542 if (nqsets < s->max_ethqsets) {
2543 dev_warn(adapter->pdev_dev, "only enough MSI-X vectors"
2544 " for %d Queue Sets\n", nqsets);
2545 s->max_ethqsets = nqsets;
2546 if (nqsets < s->ethqsets)
2547 reduce_ethqs(adapter, nqsets);
2549 for (i = 0; i < want; ++i)
2550 adapter->msix_info[i].vec = entries[i].vector;
2555 static const struct net_device_ops cxgb4vf_netdev_ops = {
2556 .ndo_open = cxgb4vf_open,
2557 .ndo_stop = cxgb4vf_stop,
2558 .ndo_start_xmit = t4vf_eth_xmit,
2559 .ndo_get_stats = cxgb4vf_get_stats,
2560 .ndo_set_rx_mode = cxgb4vf_set_rxmode,
2561 .ndo_set_mac_address = cxgb4vf_set_mac_addr,
2562 .ndo_validate_addr = eth_validate_addr,
2563 .ndo_do_ioctl = cxgb4vf_do_ioctl,
2564 .ndo_change_mtu = cxgb4vf_change_mtu,
2565 .ndo_fix_features = cxgb4vf_fix_features,
2566 .ndo_set_features = cxgb4vf_set_features,
2567 #ifdef CONFIG_NET_POLL_CONTROLLER
2568 .ndo_poll_controller = cxgb4vf_poll_controller,
2573 * "Probe" a device: initialize a device and construct all kernel and driver
2574 * state needed to manage the device. This routine is called "init_one" in
2577 static int cxgb4vf_pci_probe(struct pci_dev *pdev,
2578 const struct pci_device_id *ent)
2583 struct adapter *adapter;
2584 struct port_info *pi;
2585 struct net_device *netdev;
2588 * Print our driver banner the first time we're called to initialize a
2591 pr_info_once("%s - version %s\n", DRV_DESC, DRV_VERSION);
2594 * Initialize generic PCI device state.
2596 err = pci_enable_device(pdev);
2598 dev_err(&pdev->dev, "cannot enable PCI device\n");
2603 * Reserve PCI resources for the device. If we can't get them some
2604 * other driver may have already claimed the device ...
2606 err = pci_request_regions(pdev, KBUILD_MODNAME);
2608 dev_err(&pdev->dev, "cannot obtain PCI resources\n");
2609 goto err_disable_device;
2613 * Set up our DMA mask: try for 64-bit address masking first and
2614 * fall back to 32-bit if we can't get 64 bits ...
2616 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
2618 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
2620 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for"
2621 " coherent allocations\n");
2622 goto err_release_regions;
2626 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
2628 dev_err(&pdev->dev, "no usable DMA configuration\n");
2629 goto err_release_regions;
2635 * Enable bus mastering for the device ...
2637 pci_set_master(pdev);
2640 * Allocate our adapter data structure and attach it to the device.
2642 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
2645 goto err_release_regions;
2647 pci_set_drvdata(pdev, adapter);
2648 adapter->pdev = pdev;
2649 adapter->pdev_dev = &pdev->dev;
2652 * Initialize SMP data synchronization resources.
2654 spin_lock_init(&adapter->stats_lock);
2657 * Map our I/O registers in BAR0.
2659 adapter->regs = pci_ioremap_bar(pdev, 0);
2660 if (!adapter->regs) {
2661 dev_err(&pdev->dev, "cannot map device registers\n");
2663 goto err_free_adapter;
2666 /* Wait for the device to become ready before proceeding ...
2668 err = t4vf_prep_adapter(adapter);
2670 dev_err(adapter->pdev_dev, "device didn't become ready:"
2672 goto err_unmap_bar0;
2675 /* For T5 and later we want to use the new BAR-based User Doorbells,
2676 * so we need to map BAR2 here ...
2678 if (!is_t4(adapter->params.chip)) {
2679 adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2),
2680 pci_resource_len(pdev, 2));
2681 if (!adapter->bar2) {
2682 dev_err(adapter->pdev_dev, "cannot map BAR2 doorbells\n");
2684 goto err_unmap_bar0;
2688 * Initialize adapter level features.
2690 adapter->name = pci_name(pdev);
2691 adapter->msg_enable = dflt_msg_enable;
2692 err = adap_init0(adapter);
2697 * Allocate our "adapter ports" and stitch everything together.
2699 pmask = adapter->params.vfres.pmask;
2700 for_each_port(adapter, pidx) {
2704 * We simplistically allocate our virtual interfaces
2705 * sequentially across the port numbers to which we have
2706 * access rights. This should be configurable in some manner
2711 port_id = ffs(pmask) - 1;
2712 pmask &= ~(1 << port_id);
2713 viid = t4vf_alloc_vi(adapter, port_id);
2715 dev_err(&pdev->dev, "cannot allocate VI for port %d:"
2716 " err=%d\n", port_id, viid);
2722 * Allocate our network device and stitch things together.
2724 netdev = alloc_etherdev_mq(sizeof(struct port_info),
2726 if (netdev == NULL) {
2727 t4vf_free_vi(adapter, viid);
2731 adapter->port[pidx] = netdev;
2732 SET_NETDEV_DEV(netdev, &pdev->dev);
2733 pi = netdev_priv(netdev);
2734 pi->adapter = adapter;
2736 pi->port_id = port_id;
2740 * Initialize the starting state of our "port" and register
2743 pi->xact_addr_filt = -1;
2744 netif_carrier_off(netdev);
2745 netdev->irq = pdev->irq;
2747 netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
2748 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
2749 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_RXCSUM;
2750 netdev->vlan_features = NETIF_F_SG | TSO_FLAGS |
2751 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
2753 netdev->features = netdev->hw_features |
2754 NETIF_F_HW_VLAN_CTAG_TX;
2756 netdev->features |= NETIF_F_HIGHDMA;
2758 netdev->priv_flags |= IFF_UNICAST_FLT;
2760 netdev->netdev_ops = &cxgb4vf_netdev_ops;
2761 netdev->ethtool_ops = &cxgb4vf_ethtool_ops;
2764 * Initialize the hardware/software state for the port.
2766 err = t4vf_port_init(adapter, pidx);
2768 dev_err(&pdev->dev, "cannot initialize port %d\n",
2775 * The "card" is now ready to go. If any errors occur during device
2776 * registration we do not fail the whole "card" but rather proceed
2777 * only with the ports we manage to register successfully. However we
2778 * must register at least one net device.
2780 for_each_port(adapter, pidx) {
2781 netdev = adapter->port[pidx];
2785 err = register_netdev(netdev);
2787 dev_warn(&pdev->dev, "cannot register net device %s,"
2788 " skipping\n", netdev->name);
2792 set_bit(pidx, &adapter->registered_device_map);
2794 if (adapter->registered_device_map == 0) {
2795 dev_err(&pdev->dev, "could not register any net devices\n");
2800 * Set up our debugfs entries.
2802 if (!IS_ERR_OR_NULL(cxgb4vf_debugfs_root)) {
2803 adapter->debugfs_root =
2804 debugfs_create_dir(pci_name(pdev),
2805 cxgb4vf_debugfs_root);
2806 if (IS_ERR_OR_NULL(adapter->debugfs_root))
2807 dev_warn(&pdev->dev, "could not create debugfs"
2810 setup_debugfs(adapter);
2814 * See what interrupts we'll be using. If we've been configured to
2815 * use MSI-X interrupts, try to enable them but fall back to using
2816 * MSI interrupts if we can't enable MSI-X interrupts. If we can't
2817 * get MSI interrupts we bail with the error.
2819 if (msi == MSI_MSIX && enable_msix(adapter) == 0)
2820 adapter->flags |= USING_MSIX;
2822 err = pci_enable_msi(pdev);
2824 dev_err(&pdev->dev, "Unable to allocate %s interrupts;"
2826 msi == MSI_MSIX ? "MSI-X or MSI" : "MSI", err);
2827 goto err_free_debugfs;
2829 adapter->flags |= USING_MSI;
2833 * Now that we know how many "ports" we have and what their types are,
2834 * and how many Queue Sets we can support, we can configure our queue
2837 cfg_queues(adapter);
2840 * Print a short notice on the existence and configuration of the new
2841 * VF network device ...
2843 for_each_port(adapter, pidx) {
2844 dev_info(adapter->pdev_dev, "%s: Chelsio VF NIC PCIe %s\n",
2845 adapter->port[pidx]->name,
2846 (adapter->flags & USING_MSIX) ? "MSI-X" :
2847 (adapter->flags & USING_MSI) ? "MSI" : "");
2856 * Error recovery and exit code. Unwind state that's been created
2857 * so far and return the error.
2861 if (!IS_ERR_OR_NULL(adapter->debugfs_root)) {
2862 cleanup_debugfs(adapter);
2863 debugfs_remove_recursive(adapter->debugfs_root);
2867 for_each_port(adapter, pidx) {
2868 netdev = adapter->port[pidx];
2871 pi = netdev_priv(netdev);
2872 t4vf_free_vi(adapter, pi->viid);
2873 if (test_bit(pidx, &adapter->registered_device_map))
2874 unregister_netdev(netdev);
2875 free_netdev(netdev);
2879 if (!is_t4(adapter->params.chip))
2880 iounmap(adapter->bar2);
2883 iounmap(adapter->regs);
2888 err_release_regions:
2889 pci_release_regions(pdev);
2890 pci_clear_master(pdev);
2893 pci_disable_device(pdev);
2899 * "Remove" a device: tear down all kernel and driver state created in the
2900 * "probe" routine and quiesce the device (disable interrupts, etc.). (Note
2901 * that this is called "remove_one" in the PF Driver.)
2903 static void cxgb4vf_pci_remove(struct pci_dev *pdev)
2905 struct adapter *adapter = pci_get_drvdata(pdev);
2908 * Tear down driver state associated with device.
2914 * Stop all of our activity. Unregister network port,
2915 * disable interrupts, etc.
2917 for_each_port(adapter, pidx)
2918 if (test_bit(pidx, &adapter->registered_device_map))
2919 unregister_netdev(adapter->port[pidx]);
2920 t4vf_sge_stop(adapter);
2921 if (adapter->flags & USING_MSIX) {
2922 pci_disable_msix(adapter->pdev);
2923 adapter->flags &= ~USING_MSIX;
2924 } else if (adapter->flags & USING_MSI) {
2925 pci_disable_msi(adapter->pdev);
2926 adapter->flags &= ~USING_MSI;
2930 * Tear down our debugfs entries.
2932 if (!IS_ERR_OR_NULL(adapter->debugfs_root)) {
2933 cleanup_debugfs(adapter);
2934 debugfs_remove_recursive(adapter->debugfs_root);
2938 * Free all of the various resources which we've acquired ...
2940 t4vf_free_sge_resources(adapter);
2941 for_each_port(adapter, pidx) {
2942 struct net_device *netdev = adapter->port[pidx];
2943 struct port_info *pi;
2948 pi = netdev_priv(netdev);
2949 t4vf_free_vi(adapter, pi->viid);
2950 free_netdev(netdev);
2952 iounmap(adapter->regs);
2953 if (!is_t4(adapter->params.chip))
2954 iounmap(adapter->bar2);
2959 * Disable the device and release its PCI resources.
2961 pci_disable_device(pdev);
2962 pci_clear_master(pdev);
2963 pci_release_regions(pdev);
2967 * "Shutdown" quiesce the device, stopping Ingress Packet and Interrupt
2970 static void cxgb4vf_pci_shutdown(struct pci_dev *pdev)
2972 struct adapter *adapter;
2975 adapter = pci_get_drvdata(pdev);
2979 /* Disable all Virtual Interfaces. This will shut down the
2980 * delivery of all ingress packets into the chip for these
2981 * Virtual Interfaces.
2983 for_each_port(adapter, pidx)
2984 if (test_bit(pidx, &adapter->registered_device_map))
2985 unregister_netdev(adapter->port[pidx]);
2987 /* Free up all Queues which will prevent further DMA and
2988 * Interrupts allowing various internal pathways to drain.
2990 t4vf_sge_stop(adapter);
2991 if (adapter->flags & USING_MSIX) {
2992 pci_disable_msix(adapter->pdev);
2993 adapter->flags &= ~USING_MSIX;
2994 } else if (adapter->flags & USING_MSI) {
2995 pci_disable_msi(adapter->pdev);
2996 adapter->flags &= ~USING_MSI;
3000 * Free up all Queues which will prevent further DMA and
3001 * Interrupts allowing various internal pathways to drain.
3003 t4vf_free_sge_resources(adapter);
3004 pci_set_drvdata(pdev, NULL);
3007 /* Macros needed to support the PCI Device ID Table ...
3009 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \
3010 static const struct pci_device_id cxgb4vf_pci_tbl[] = {
3011 #define CH_PCI_DEVICE_ID_FUNCTION 0x8
3013 #define CH_PCI_ID_TABLE_ENTRY(devid) \
3014 { PCI_VDEVICE(CHELSIO, (devid)), 0 }
3016 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_END { 0, } }
3018 #include "../cxgb4/t4_pci_id_tbl.h"
3020 MODULE_DESCRIPTION(DRV_DESC);
3021 MODULE_AUTHOR("Chelsio Communications");
3022 MODULE_LICENSE("Dual BSD/GPL");
3023 MODULE_VERSION(DRV_VERSION);
3024 MODULE_DEVICE_TABLE(pci, cxgb4vf_pci_tbl);
3026 static struct pci_driver cxgb4vf_driver = {
3027 .name = KBUILD_MODNAME,
3028 .id_table = cxgb4vf_pci_tbl,
3029 .probe = cxgb4vf_pci_probe,
3030 .remove = cxgb4vf_pci_remove,
3031 .shutdown = cxgb4vf_pci_shutdown,
3035 * Initialize global driver state.
3037 static int __init cxgb4vf_module_init(void)
3042 * Vet our module parameters.
3044 if (msi != MSI_MSIX && msi != MSI_MSI) {
3045 pr_warn("bad module parameter msi=%d; must be %d (MSI-X or MSI) or %d (MSI)\n",
3046 msi, MSI_MSIX, MSI_MSI);
3050 /* Debugfs support is optional, just warn if this fails */
3051 cxgb4vf_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
3052 if (IS_ERR_OR_NULL(cxgb4vf_debugfs_root))
3053 pr_warn("could not create debugfs entry, continuing\n");
3055 ret = pci_register_driver(&cxgb4vf_driver);
3056 if (ret < 0 && !IS_ERR_OR_NULL(cxgb4vf_debugfs_root))
3057 debugfs_remove(cxgb4vf_debugfs_root);
3062 * Tear down global driver state.
3064 static void __exit cxgb4vf_module_exit(void)
3066 pci_unregister_driver(&cxgb4vf_driver);
3067 debugfs_remove(cxgb4vf_debugfs_root);
3070 module_init(cxgb4vf_module_init);
3071 module_exit(cxgb4vf_module_exit);