drivers/net/ethernet/mellanox/mlx5/core/en_main.c

   1 /*
   2  * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <net/tc_act/tc_gact.h>
  34 #include <net/pkt_cls.h>
  35 #include <linux/mlx5/fs.h>
  36 #include <net/vxlan.h>
  37 #include <linux/bpf.h>
  38 #include "en.h"
  39 #include "en_tc.h"
  40 #include "eswitch.h"
  41 #include "vxlan.h"
  42
  43 struct mlx5e_rq_param {
  44         u32                     rqc[MLX5_ST_SZ_DW(rqc)];
  45         struct mlx5_wq_param    wq;
  46         bool                    am_enabled;
  47 };
  48
  49 struct mlx5e_sq_param {
  50         u32                        sqc[MLX5_ST_SZ_DW(sqc)];
  51         struct mlx5_wq_param       wq;
  52         u16                        max_inline;
  53         u8                         min_inline_mode;
  54         enum mlx5e_sq_type         type;
  55 };
  56
  57 struct mlx5e_cq_param {
  58         u32                        cqc[MLX5_ST_SZ_DW(cqc)];
  59         struct mlx5_wq_param       wq;
  60         u16                        eq_ix;
  61         u8                         cq_period_mode;
  62 };
  63
  64 struct mlx5e_channel_param {
  65         struct mlx5e_rq_param      rq;
  66         struct mlx5e_sq_param      sq;
  67         struct mlx5e_sq_param      xdp_sq;
  68         struct mlx5e_sq_param      icosq;
  69         struct mlx5e_cq_param      rx_cq;
  70         struct mlx5e_cq_param      tx_cq;
  71         struct mlx5e_cq_param      icosq_cq;
  72 };
  73
  74 static bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev)
  75 {
  76         return MLX5_CAP_GEN(mdev, striding_rq) &&
  77                 MLX5_CAP_GEN(mdev, umr_ptr_rlky) &&
  78                 MLX5_CAP_ETH(mdev, reg_umr_sq);
  79 }
  80
  81 static void mlx5e_set_rq_type_params(struct mlx5e_priv *priv, u8 rq_type)
  82 {
  83         priv->params.rq_wq_type = rq_type;
  84         switch (priv->params.rq_wq_type) {
  85         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
  86                 priv->params.log_rq_size = MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE_MPW;
  87                 priv->params.mpwqe_log_stride_sz = priv->params.rx_cqe_compress ?
  88                         MLX5_MPWRQ_LOG_STRIDE_SIZE_CQE_COMPRESS :
  89                         MLX5_MPWRQ_LOG_STRIDE_SIZE;
  90                 priv->params.mpwqe_log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ -
  91                         priv->params.mpwqe_log_stride_sz;
  92                 break;
  93         default: /* MLX5_WQ_TYPE_LINKED_LIST */
  94                 priv->params.log_rq_size = MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
  95         }
  96         priv->params.min_rx_wqes = mlx5_min_rx_wqes(priv->params.rq_wq_type,
  97                                                BIT(priv->params.log_rq_size));
  98
  99         mlx5_core_info(priv->mdev,
 100                        "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n",
 101                        priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ,
 102                        BIT(priv->params.log_rq_size),
 103                        BIT(priv->params.mpwqe_log_stride_sz),
 104                        priv->params.rx_cqe_compress_admin);
 105 }
 106
 107 static void mlx5e_set_rq_priv_params(struct mlx5e_priv *priv)
 108 {
 109         u8 rq_type = mlx5e_check_fragmented_striding_rq_cap(priv->mdev) &&
 110                     !priv->xdp_prog ?
 111                     MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ :
 112                     MLX5_WQ_TYPE_LINKED_LIST;
 113         mlx5e_set_rq_type_params(priv, rq_type);
 114 }
 115
 116 static void mlx5e_update_carrier(struct mlx5e_priv *priv)
 117 {
 118         struct mlx5_core_dev *mdev = priv->mdev;
 119         u8 port_state;
 120
 121         port_state = mlx5_query_vport_state(mdev,
 122                 MLX5_QUERY_VPORT_STATE_IN_OP_MOD_VNIC_VPORT, 0);
 123
 124         if (port_state == VPORT_STATE_UP) {
 125                 netdev_info(priv->netdev, "Link up\n");
 126                 netif_carrier_on(priv->netdev);
 127         } else {
 128                 netdev_info(priv->netdev, "Link down\n");
 129                 netif_carrier_off(priv->netdev);
 130         }
 131 }
 132
 133 static void mlx5e_update_carrier_work(struct work_struct *work)
 134 {
 135         struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
 136                                                update_carrier_work);
 137
 138         mutex_lock(&priv->state_lock);
 139         if (test_bit(MLX5E_STATE_OPENED, &priv->state))
 140                 mlx5e_update_carrier(priv);
 141         mutex_unlock(&priv->state_lock);
 142 }
 143
 144 static void mlx5e_tx_timeout_work(struct work_struct *work)
 145 {
 146         struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
 147                                                tx_timeout_work);
 148         int err;
 149
 150         rtnl_lock();
 151         mutex_lock(&priv->state_lock);
 152         if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
 153                 goto unlock;
 154         mlx5e_close_locked(priv->netdev);
 155         err = mlx5e_open_locked(priv->netdev);
 156         if (err)
 157                 netdev_err(priv->netdev, "mlx5e_open_locked failed recovering from a tx_timeout, err(%d).\n",
 158                            err);
 159 unlock:
 160         mutex_unlock(&priv->state_lock);
 161         rtnl_unlock();
 162 }
 163
 164 static void mlx5e_update_sw_counters(struct mlx5e_priv *priv)
 165 {
 166         struct mlx5e_sw_stats *s = &priv->stats.sw;
 167         struct mlx5e_rq_stats *rq_stats;
 168         struct mlx5e_sq_stats *sq_stats;
 169         u64 tx_offload_none = 0;
 170         int i, j;
 171
 172         memset(s, 0, sizeof(*s));
 173         for (i = 0; i < priv->params.num_channels; i++) {
 174                 rq_stats = &priv->channel[i]->rq.stats;
 175
 176                 s->rx_packets   += rq_stats->packets;
 177                 s->rx_bytes     += rq_stats->bytes;
 178                 s->rx_lro_packets += rq_stats->lro_packets;
 179                 s->rx_lro_bytes += rq_stats->lro_bytes;
 180                 s->rx_csum_none += rq_stats->csum_none;
 181                 s->rx_csum_complete += rq_stats->csum_complete;
 182                 s->rx_csum_unnecessary_inner += rq_stats->csum_unnecessary_inner;
 183                 s->rx_xdp_drop += rq_stats->xdp_drop;
 184                 s->rx_xdp_tx += rq_stats->xdp_tx;
 185                 s->rx_xdp_tx_full += rq_stats->xdp_tx_full;
 186                 s->rx_wqe_err   += rq_stats->wqe_err;
 187                 s->rx_mpwqe_filler += rq_stats->mpwqe_filler;
 188                 s->rx_buff_alloc_err += rq_stats->buff_alloc_err;
 189                 s->rx_cqe_compress_blks += rq_stats->cqe_compress_blks;
 190                 s->rx_cqe_compress_pkts += rq_stats->cqe_compress_pkts;
 191                 s->rx_cache_reuse += rq_stats->cache_reuse;
 192                 s->rx_cache_full  += rq_stats->cache_full;
 193                 s->rx_cache_empty += rq_stats->cache_empty;
 194                 s->rx_cache_busy  += rq_stats->cache_busy;
 195
 196                 for (j = 0; j < priv->params.num_tc; j++) {
 197                         sq_stats = &priv->channel[i]->sq[j].stats;
 198
 199                         s->tx_packets           += sq_stats->packets;
 200                         s->tx_bytes             += sq_stats->bytes;
 201                         s->tx_tso_packets       += sq_stats->tso_packets;
 202                         s->tx_tso_bytes         += sq_stats->tso_bytes;
 203                         s->tx_tso_inner_packets += sq_stats->tso_inner_packets;
 204                         s->tx_tso_inner_bytes   += sq_stats->tso_inner_bytes;
 205                         s->tx_queue_stopped     += sq_stats->stopped;
 206                         s->tx_queue_wake        += sq_stats->wake;
 207                         s->tx_queue_dropped     += sq_stats->dropped;
 208                         s->tx_xmit_more         += sq_stats->xmit_more;
 209                         s->tx_csum_partial_inner += sq_stats->csum_partial_inner;
 210                         tx_offload_none         += sq_stats->csum_none;
 211                 }
 212         }
 213
 214         /* Update calculated offload counters */
 215         s->tx_csum_partial = s->tx_packets - tx_offload_none - s->tx_csum_partial_inner;
 216         s->rx_csum_unnecessary = s->rx_packets - s->rx_csum_none - s->rx_csum_complete;
 217
 218         s->link_down_events_phy = MLX5_GET(ppcnt_reg,
 219                                 priv->stats.pport.phy_counters,
 220                                 counter_set.phys_layer_cntrs.link_down_events);
 221 }
 222
 223 static void mlx5e_update_vport_counters(struct mlx5e_priv *priv)
 224 {
 225         int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
 226         u32 *out = (u32 *)priv->stats.vport.query_vport_out;
 227         u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)] = {0};
 228         struct mlx5_core_dev *mdev = priv->mdev;
 229
 230         MLX5_SET(query_vport_counter_in, in, opcode,
 231                  MLX5_CMD_OP_QUERY_VPORT_COUNTER);
 232         MLX5_SET(query_vport_counter_in, in, op_mod, 0);
 233         MLX5_SET(query_vport_counter_in, in, other_vport, 0);
 234
 235         memset(out, 0, outlen);
 236         mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen);
 237 }
 238
 239 static void mlx5e_update_pport_counters(struct mlx5e_priv *priv)
 240 {
 241         struct mlx5e_pport_stats *pstats = &priv->stats.pport;
 242         struct mlx5_core_dev *mdev = priv->mdev;
 243         int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
 244         int prio;
 245         void *out;
 246         u32 *in;
 247
 248         in = mlx5_vzalloc(sz);
 249         if (!in)
 250                 goto free_out;
 251
 252         MLX5_SET(ppcnt_reg, in, local_port, 1);
 253
 254         out = pstats->IEEE_802_3_counters;
 255         MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP);
 256         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 257
 258         out = pstats->RFC_2863_counters;
 259         MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2863_COUNTERS_GROUP);
 260         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 261
 262         out = pstats->RFC_2819_counters;
 263         MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2819_COUNTERS_GROUP);
 264         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 265
 266         out = pstats->phy_counters;
 267         MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP);
 268         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 269
 270         MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_PRIORITY_COUNTERS_GROUP);
 271         for (prio = 0; prio < NUM_PPORT_PRIO; prio++) {
 272                 out = pstats->per_prio_counters[prio];
 273                 MLX5_SET(ppcnt_reg, in, prio_tc, prio);
 274                 mlx5_core_access_reg(mdev, in, sz, out, sz,
 275                                      MLX5_REG_PPCNT, 0, 0);
 276         }
 277
 278 free_out:
 279         kvfree(in);
 280 }
 281
 282 static void mlx5e_update_q_counter(struct mlx5e_priv *priv)
 283 {
 284         struct mlx5e_qcounter_stats *qcnt = &priv->stats.qcnt;
 285
 286         if (!priv->q_counter)
 287                 return;
 288
 289         mlx5_core_query_out_of_buffer(priv->mdev, priv->q_counter,
 290                                       &qcnt->rx_out_of_buffer);
 291 }
 292
 293 void mlx5e_update_stats(struct mlx5e_priv *priv)
 294 {
 295         mlx5e_update_q_counter(priv);
 296         mlx5e_update_vport_counters(priv);
 297         mlx5e_update_pport_counters(priv);
 298         mlx5e_update_sw_counters(priv);
 299 }
 300
 301 void mlx5e_update_stats_work(struct work_struct *work)
 302 {
 303         struct delayed_work *dwork = to_delayed_work(work);
 304         struct mlx5e_priv *priv = container_of(dwork, struct mlx5e_priv,
 305                                                update_stats_work);
 306         mutex_lock(&priv->state_lock);
 307         if (test_bit(MLX5E_STATE_OPENED, &priv->state)) {
 308                 priv->profile->update_stats(priv);
 309                 queue_delayed_work(priv->wq, dwork,
 310                                    msecs_to_jiffies(MLX5E_UPDATE_STATS_INTERVAL));
 311         }
 312         mutex_unlock(&priv->state_lock);
 313 }
 314
 315 static void mlx5e_async_event(struct mlx5_core_dev *mdev, void *vpriv,
 316                               enum mlx5_dev_event event, unsigned long param)
 317 {
 318         struct mlx5e_priv *priv = vpriv;
 319
 320         if (!test_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state))
 321                 return;
 322
 323         switch (event) {
 324         case MLX5_DEV_EVENT_PORT_UP:
 325         case MLX5_DEV_EVENT_PORT_DOWN:
 326                 queue_work(priv->wq, &priv->update_carrier_work);
 327                 break;
 328
 329         default:
 330                 break;
 331         }
 332 }
 333
 334 static void mlx5e_enable_async_events(struct mlx5e_priv *priv)
 335 {
 336         set_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state);
 337 }
 338
 339 static void mlx5e_disable_async_events(struct mlx5e_priv *priv)
 340 {
 341         clear_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state);
 342         synchronize_irq(mlx5_get_msix_vec(priv->mdev, MLX5_EQ_VEC_ASYNC));
 343 }
 344
 345 #define MLX5E_HW2SW_MTU(hwmtu) (hwmtu - (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN))
 346 #define MLX5E_SW2HW_MTU(swmtu) (swmtu + (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN))
 347
 348 static inline int mlx5e_get_wqe_mtt_sz(void)
 349 {
 350         /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes.
 351          * To avoid copying garbage after the mtt array, we allocate
 352          * a little more.
 353          */
 354         return ALIGN(MLX5_MPWRQ_PAGES_PER_WQE * sizeof(__be64),
 355                      MLX5_UMR_MTT_ALIGNMENT);
 356 }
 357
 358 static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq, struct mlx5e_sq *sq,
 359                                        struct mlx5e_umr_wqe *wqe, u16 ix)
 360 {
 361         struct mlx5_wqe_ctrl_seg      *cseg = &wqe->ctrl;
 362         struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
 363         struct mlx5_wqe_data_seg      *dseg = &wqe->data;
 364         struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
 365         u8 ds_cnt = DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS);
 366         u32 umr_wqe_mtt_offset = mlx5e_get_wqe_mtt_offset(rq, ix);
 367
 368         cseg->qpn_ds    = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
 369                                       ds_cnt);
 370         cseg->fm_ce_se  = MLX5_WQE_CTRL_CQ_UPDATE;
 371         cseg->imm       = rq->mkey_be;
 372
 373         ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN;
 374         ucseg->klm_octowords =
 375                 cpu_to_be16(MLX5_MTT_OCTW(MLX5_MPWRQ_PAGES_PER_WQE));
 376         ucseg->bsf_octowords =
 377                 cpu_to_be16(MLX5_MTT_OCTW(umr_wqe_mtt_offset));
 378         ucseg->mkey_mask     = cpu_to_be64(MLX5_MKEY_MASK_FREE);
 379
 380         dseg->lkey = sq->mkey_be;
 381         dseg->addr = cpu_to_be64(wi->umr.mtt_addr);
 382 }
 383
 384 static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq,
 385                                      struct mlx5e_channel *c)
 386 {
 387         int wq_sz = mlx5_wq_ll_get_size(&rq->wq);
 388         int mtt_sz = mlx5e_get_wqe_mtt_sz();
 389         int mtt_alloc = mtt_sz + MLX5_UMR_ALIGN - 1;
 390         int i;
 391
 392         rq->mpwqe.info = kzalloc_node(wq_sz * sizeof(*rq->mpwqe.info),
 393                                       GFP_KERNEL, cpu_to_node(c->cpu));
 394         if (!rq->mpwqe.info)
 395                 goto err_out;
 396
 397         /* We allocate more than mtt_sz as we will align the pointer */
 398         rq->mpwqe.mtt_no_align = kzalloc_node(mtt_alloc * wq_sz, GFP_KERNEL,
 399                                         cpu_to_node(c->cpu));
 400         if (unlikely(!rq->mpwqe.mtt_no_align))
 401                 goto err_free_wqe_info;
 402
 403         for (i = 0; i < wq_sz; i++) {
 404                 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[i];
 405
 406                 wi->umr.mtt = PTR_ALIGN(rq->mpwqe.mtt_no_align + i * mtt_alloc,
 407                                         MLX5_UMR_ALIGN);
 408                 wi->umr.mtt_addr = dma_map_single(c->pdev, wi->umr.mtt, mtt_sz,
 409                                                   PCI_DMA_TODEVICE);
 410                 if (unlikely(dma_mapping_error(c->pdev, wi->umr.mtt_addr)))
 411                         goto err_unmap_mtts;
 412
 413                 mlx5e_build_umr_wqe(rq, &c->icosq, &wi->umr.wqe, i);
 414         }
 415
 416         return 0;
 417
 418 err_unmap_mtts:
 419         while (--i >= 0) {
 420                 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[i];
 421
 422                 dma_unmap_single(c->pdev, wi->umr.mtt_addr, mtt_sz,
 423                                  PCI_DMA_TODEVICE);
 424         }
 425         kfree(rq->mpwqe.mtt_no_align);
 426 err_free_wqe_info:
 427         kfree(rq->mpwqe.info);
 428
 429 err_out:
 430         return -ENOMEM;
 431 }
 432
 433 static void mlx5e_rq_free_mpwqe_info(struct mlx5e_rq *rq)
 434 {
 435         int wq_sz = mlx5_wq_ll_get_size(&rq->wq);
 436         int mtt_sz = mlx5e_get_wqe_mtt_sz();
 437         int i;
 438
 439         for (i = 0; i < wq_sz; i++) {
 440                 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[i];
 441
 442                 dma_unmap_single(rq->pdev, wi->umr.mtt_addr, mtt_sz,
 443                                  PCI_DMA_TODEVICE);
 444         }
 445         kfree(rq->mpwqe.mtt_no_align);
 446         kfree(rq->mpwqe.info);
 447 }
 448
 449 static int mlx5e_create_rq(struct mlx5e_channel *c,
 450                            struct mlx5e_rq_param *param,
 451                            struct mlx5e_rq *rq)
 452 {
 453         struct mlx5e_priv *priv = c->priv;
 454         struct mlx5_core_dev *mdev = priv->mdev;
 455         void *rqc = param->rqc;
 456         void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
 457         u32 byte_count;
 458         u32 frag_sz;
 459         int npages;
 460         int wq_sz;
 461         int err;
 462         int i;
 463
 464         param->wq.db_numa_node = cpu_to_node(c->cpu);
 465
 466         err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
 467                                 &rq->wq_ctrl);
 468         if (err)
 469                 return err;
 470
 471         rq->wq.db = &rq->wq.db[MLX5_RCV_DBR];
 472
 473         wq_sz = mlx5_wq_ll_get_size(&rq->wq);
 474
 475         rq->wq_type = priv->params.rq_wq_type;
 476         rq->pdev    = c->pdev;
 477         rq->netdev  = c->netdev;
 478         rq->tstamp  = &priv->tstamp;
 479         rq->channel = c;
 480         rq->ix      = c->ix;
 481         rq->priv    = c->priv;
 482         rq->xdp_prog = priv->xdp_prog;
 483
 484         rq->buff.map_dir = DMA_FROM_DEVICE;
 485         if (rq->xdp_prog)
 486                 rq->buff.map_dir = DMA_BIDIRECTIONAL;
 487
 488         switch (priv->params.rq_wq_type) {
 489         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 490                 rq->handle_rx_cqe = mlx5e_handle_rx_cqe_mpwrq;
 491                 rq->alloc_wqe = mlx5e_alloc_rx_mpwqe;
 492                 rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe;
 493
 494                 rq->mpwqe.mtt_offset = c->ix *
 495                         MLX5E_REQUIRED_MTTS(1, BIT(priv->params.log_rq_size));
 496
 497                 rq->mpwqe_stride_sz = BIT(priv->params.mpwqe_log_stride_sz);
 498                 rq->mpwqe_num_strides = BIT(priv->params.mpwqe_log_num_strides);
 499
 500                 rq->buff.wqe_sz = rq->mpwqe_stride_sz * rq->mpwqe_num_strides;
 501                 byte_count = rq->buff.wqe_sz;
 502                 rq->mkey_be = cpu_to_be32(c->priv->umr_mkey.key);
 503                 err = mlx5e_rq_alloc_mpwqe_info(rq, c);
 504                 if (err)
 505                         goto err_rq_wq_destroy;
 506                 break;
 507         default: /* MLX5_WQ_TYPE_LINKED_LIST */
 508                 rq->dma_info = kzalloc_node(wq_sz * sizeof(*rq->dma_info),
 509                                             GFP_KERNEL, cpu_to_node(c->cpu));
 510                 if (!rq->dma_info) {
 511                         err = -ENOMEM;
 512                         goto err_rq_wq_destroy;
 513                 }
 514
 515                 rq->handle_rx_cqe = mlx5e_handle_rx_cqe;
 516                 rq->alloc_wqe = mlx5e_alloc_rx_wqe;
 517                 rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;
 518
 519                 rq->buff.wqe_sz = (priv->params.lro_en) ?
 520                                 priv->params.lro_wqe_sz :
 521                                 MLX5E_SW2HW_MTU(priv->netdev->mtu);
 522                 byte_count = rq->buff.wqe_sz;
 523
 524                 /* calc the required page order */
 525                 frag_sz = MLX5_RX_HEADROOM +
 526                           byte_count /* packet data */ +
 527                           SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 528                 frag_sz = SKB_DATA_ALIGN(frag_sz);
 529
 530                 npages = DIV_ROUND_UP(frag_sz, PAGE_SIZE);
 531                 rq->buff.page_order = order_base_2(npages);
 532
 533                 byte_count |= MLX5_HW_START_PADDING;
 534                 rq->mkey_be = c->mkey_be;
 535         }
 536
 537         for (i = 0; i < wq_sz; i++) {
 538                 struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);
 539
 540                 wqe->data.byte_count = cpu_to_be32(byte_count);
 541                 wqe->data.lkey = rq->mkey_be;
 542         }
 543
 544         INIT_WORK(&rq->am.work, mlx5e_rx_am_work);
 545         rq->am.mode = priv->params.rx_cq_period_mode;
 546
 547         rq->page_cache.head = 0;
 548         rq->page_cache.tail = 0;
 549
 550         if (rq->xdp_prog)
 551                 bpf_prog_add(rq->xdp_prog, 1);
 552
 553         return 0;
 554
 555 err_rq_wq_destroy:
 556         mlx5_wq_destroy(&rq->wq_ctrl);
 557
 558         return err;
 559 }
 560
 561 static void mlx5e_destroy_rq(struct mlx5e_rq *rq)
 562 {
 563         int i;
 564
 565         if (rq->xdp_prog)
 566                 bpf_prog_put(rq->xdp_prog);
 567
 568         switch (rq->wq_type) {
 569         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 570                 mlx5e_rq_free_mpwqe_info(rq);
 571                 break;
 572         default: /* MLX5_WQ_TYPE_LINKED_LIST */
 573                 kfree(rq->dma_info);
 574         }
 575
 576         for (i = rq->page_cache.head; i != rq->page_cache.tail;
 577              i = (i + 1) & (MLX5E_CACHE_SIZE - 1)) {
 578                 struct mlx5e_dma_info *dma_info = &rq->page_cache.page_cache[i];
 579
 580                 mlx5e_page_release(rq, dma_info, false);
 581         }
 582         mlx5_wq_destroy(&rq->wq_ctrl);
 583 }
 584
 585 static int mlx5e_enable_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
 586 {
 587         struct mlx5e_priv *priv = rq->priv;
 588         struct mlx5_core_dev *mdev = priv->mdev;
 589
 590         void *in;
 591         void *rqc;
 592         void *wq;
 593         int inlen;
 594         int err;
 595
 596         inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
 597                 sizeof(u64) * rq->wq_ctrl.buf.npages;
 598         in = mlx5_vzalloc(inlen);
 599         if (!in)
 600                 return -ENOMEM;
 601
 602         rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
 603         wq  = MLX5_ADDR_OF(rqc, rqc, wq);
 604
 605         memcpy(rqc, param->rqc, sizeof(param->rqc));
 606
 607         MLX5_SET(rqc,  rqc, cqn,                rq->cq.mcq.cqn);
 608         MLX5_SET(rqc,  rqc, state,              MLX5_RQC_STATE_RST);
 609         MLX5_SET(rqc,  rqc, vsd, priv->params.vlan_strip_disable);
 610         MLX5_SET(wq,   wq,  log_wq_pg_sz,       rq->wq_ctrl.buf.page_shift -
 611                                                 MLX5_ADAPTER_PAGE_SHIFT);
 612         MLX5_SET64(wq, wq,  dbr_addr,           rq->wq_ctrl.db.dma);
 613
 614         mlx5_fill_page_array(&rq->wq_ctrl.buf,
 615                              (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
 616
 617         err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
 618
 619         kvfree(in);
 620
 621         return err;
 622 }
 623
 624 static int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state,
 625                                  int next_state)
 626 {
 627         struct mlx5e_channel *c = rq->channel;
 628         struct mlx5e_priv *priv = c->priv;
 629         struct mlx5_core_dev *mdev = priv->mdev;
 630
 631         void *in;
 632         void *rqc;
 633         int inlen;
 634         int err;
 635
 636         inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
 637         in = mlx5_vzalloc(inlen);
 638         if (!in)
 639                 return -ENOMEM;
 640
 641         rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
 642
 643         MLX5_SET(modify_rq_in, in, rq_state, curr_state);
 644         MLX5_SET(rqc, rqc, state, next_state);
 645
 646         err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen);
 647
 648         kvfree(in);
 649
 650         return err;
 651 }
 652
 653 static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd)
 654 {
 655         struct mlx5e_channel *c = rq->channel;
 656         struct mlx5e_priv *priv = c->priv;
 657         struct mlx5_core_dev *mdev = priv->mdev;
 658
 659         void *in;
 660         void *rqc;
 661         int inlen;
 662         int err;
 663
 664         inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
 665         in = mlx5_vzalloc(inlen);
 666         if (!in)
 667                 return -ENOMEM;
 668
 669         rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
 670
 671         MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY);
 672         MLX5_SET64(modify_rq_in, in, modify_bitmask,
 673                    MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD);
 674         MLX5_SET(rqc, rqc, vsd, vsd);
 675         MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
 676
 677         err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen);
 678
 679         kvfree(in);
 680
 681         return err;
 682 }
 683
 684 static void mlx5e_disable_rq(struct mlx5e_rq *rq)
 685 {
 686         mlx5_core_destroy_rq(rq->priv->mdev, rq->rqn);
 687 }
 688
 689 static int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq)
 690 {
 691         unsigned long exp_time = jiffies + msecs_to_jiffies(20000);
 692         struct mlx5e_channel *c = rq->channel;
 693         struct mlx5e_priv *priv = c->priv;
 694         struct mlx5_wq_ll *wq = &rq->wq;
 695
 696         while (time_before(jiffies, exp_time)) {
 697                 if (wq->cur_sz >= priv->params.min_rx_wqes)
 698                         return 0;
 699
 700                 msleep(20);
 701         }
 702
 703         return -ETIMEDOUT;
 704 }
 705
 706 static void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
 707 {
 708         struct mlx5_wq_ll *wq = &rq->wq;
 709         struct mlx5e_rx_wqe *wqe;
 710         __be16 wqe_ix_be;
 711         u16 wqe_ix;
 712
 713         /* UMR WQE (if in progress) is always at wq->head */
 714         if (test_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state))
 715                 mlx5e_free_rx_mpwqe(rq, &rq->mpwqe.info[wq->head]);
 716
 717         while (!mlx5_wq_ll_is_empty(wq)) {
 718                 wqe_ix_be = *wq->tail_next;
 719                 wqe_ix    = be16_to_cpu(wqe_ix_be);
 720                 wqe       = mlx5_wq_ll_get_wqe(&rq->wq, wqe_ix);
 721                 rq->dealloc_wqe(rq, wqe_ix);
 722                 mlx5_wq_ll_pop(&rq->wq, wqe_ix_be,
 723                                &wqe->next.next_wqe_index);
 724         }
 725 }
 726
 727 static int mlx5e_open_rq(struct mlx5e_channel *c,
 728                          struct mlx5e_rq_param *param,
 729                          struct mlx5e_rq *rq)
 730 {
 731         struct mlx5e_sq *sq = &c->icosq;
 732         u16 pi = sq->pc & sq->wq.sz_m1;
 733         int err;
 734
 735         err = mlx5e_create_rq(c, param, rq);
 736         if (err)
 737                 return err;
 738
 739         err = mlx5e_enable_rq(rq, param);
 740         if (err)
 741                 goto err_destroy_rq;
 742
 743         err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
 744         if (err)
 745                 goto err_disable_rq;
 746
 747         if (param->am_enabled)
 748                 set_bit(MLX5E_RQ_STATE_AM, &c->rq.state);
 749
 750         sq->db.ico_wqe[pi].opcode     = MLX5_OPCODE_NOP;
 751         sq->db.ico_wqe[pi].num_wqebbs = 1;
 752         mlx5e_send_nop(sq, true); /* trigger mlx5e_post_rx_wqes() */
 753
 754         return 0;
 755
 756 err_disable_rq:
 757         mlx5e_disable_rq(rq);
 758 err_destroy_rq:
 759         mlx5e_destroy_rq(rq);
 760
 761         return err;
 762 }
 763
 764 static void mlx5e_close_rq(struct mlx5e_rq *rq)
 765 {
 766         set_bit(MLX5E_RQ_STATE_FLUSH, &rq->state);
 767         napi_synchronize(&rq->channel->napi); /* prevent mlx5e_post_rx_wqes */
 768         cancel_work_sync(&rq->am.work);
 769
 770         mlx5e_disable_rq(rq);
 771         mlx5e_free_rx_descs(rq);
 772         mlx5e_destroy_rq(rq);
 773 }
 774
 775 static void mlx5e_free_sq_xdp_db(struct mlx5e_sq *sq)
 776 {
 777         kfree(sq->db.xdp.di);
 778         kfree(sq->db.xdp.wqe_info);
 779 }
 780
 781 static int mlx5e_alloc_sq_xdp_db(struct mlx5e_sq *sq, int numa)
 782 {
 783         int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
 784
 785         sq->db.xdp.di = kzalloc_node(sizeof(*sq->db.xdp.di) * wq_sz,
 786                                      GFP_KERNEL, numa);
 787         sq->db.xdp.wqe_info = kzalloc_node(sizeof(*sq->db.xdp.wqe_info) * wq_sz,
 788                                            GFP_KERNEL, numa);
 789         if (!sq->db.xdp.di || !sq->db.xdp.wqe_info) {
 790                 mlx5e_free_sq_xdp_db(sq);
 791                 return -ENOMEM;
 792         }
 793
 794         return 0;
 795 }
 796
 797 static void mlx5e_free_sq_ico_db(struct mlx5e_sq *sq)
 798 {
 799         kfree(sq->db.ico_wqe);
 800 }
 801
 802 static int mlx5e_alloc_sq_ico_db(struct mlx5e_sq *sq, int numa)
 803 {
 804         u8 wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
 805
 806         sq->db.ico_wqe = kzalloc_node(sizeof(*sq->db.ico_wqe) * wq_sz,
 807                                       GFP_KERNEL, numa);
 808         if (!sq->db.ico_wqe)
 809                 return -ENOMEM;
 810
 811         return 0;
 812 }
 813
 814 static void mlx5e_free_sq_txq_db(struct mlx5e_sq *sq)
 815 {
 816         kfree(sq->db.txq.wqe_info);
 817         kfree(sq->db.txq.dma_fifo);
 818         kfree(sq->db.txq.skb);
 819 }
 820
 821 static int mlx5e_alloc_sq_txq_db(struct mlx5e_sq *sq, int numa)
 822 {
 823         int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
 824         int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS;
 825
 826         sq->db.txq.skb = kzalloc_node(wq_sz * sizeof(*sq->db.txq.skb),
 827                                       GFP_KERNEL, numa);
 828         sq->db.txq.dma_fifo = kzalloc_node(df_sz * sizeof(*sq->db.txq.dma_fifo),
 829                                            GFP_KERNEL, numa);
 830         sq->db.txq.wqe_info = kzalloc_node(wq_sz * sizeof(*sq->db.txq.wqe_info),
 831                                            GFP_KERNEL, numa);
 832         if (!sq->db.txq.skb || !sq->db.txq.dma_fifo || !sq->db.txq.wqe_info) {
 833                 mlx5e_free_sq_txq_db(sq);
 834                 return -ENOMEM;
 835         }
 836
 837         sq->dma_fifo_mask = df_sz - 1;
 838
 839         return 0;
 840 }
 841
 842 static void mlx5e_free_sq_db(struct mlx5e_sq *sq)
 843 {
 844         switch (sq->type) {
 845         case MLX5E_SQ_TXQ:
 846                 mlx5e_free_sq_txq_db(sq);
 847                 break;
 848         case MLX5E_SQ_ICO:
 849                 mlx5e_free_sq_ico_db(sq);
 850                 break;
 851         case MLX5E_SQ_XDP:
 852                 mlx5e_free_sq_xdp_db(sq);
 853                 break;
 854         }
 855 }
 856
 857 static int mlx5e_alloc_sq_db(struct mlx5e_sq *sq, int numa)
 858 {
 859         switch (sq->type) {
 860         case MLX5E_SQ_TXQ:
 861                 return mlx5e_alloc_sq_txq_db(sq, numa);
 862         case MLX5E_SQ_ICO:
 863                 return mlx5e_alloc_sq_ico_db(sq, numa);
 864         case MLX5E_SQ_XDP:
 865                 return mlx5e_alloc_sq_xdp_db(sq, numa);
 866         }
 867
 868         return 0;
 869 }
 870
 871 static int mlx5e_sq_get_max_wqebbs(u8 sq_type)
 872 {
 873         switch (sq_type) {
 874         case MLX5E_SQ_ICO:
 875                 return MLX5E_ICOSQ_MAX_WQEBBS;
 876         case MLX5E_SQ_XDP:
 877                 return MLX5E_XDP_TX_WQEBBS;
 878         }
 879         return MLX5_SEND_WQE_MAX_WQEBBS;
 880 }
 881
 882 static int mlx5e_create_sq(struct mlx5e_channel *c,
 883                            int tc,
 884                            struct mlx5e_sq_param *param,
 885                            struct mlx5e_sq *sq)
 886 {
 887         struct mlx5e_priv *priv = c->priv;
 888         struct mlx5_core_dev *mdev = priv->mdev;
 889
 890         void *sqc = param->sqc;
 891         void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq);
 892         int err;
 893
 894         sq->type      = param->type;
 895         sq->pdev      = c->pdev;
 896         sq->tstamp    = &priv->tstamp;
 897         sq->mkey_be   = c->mkey_be;
 898         sq->channel   = c;
 899         sq->tc        = tc;
 900
 901         err = mlx5_alloc_map_uar(mdev, &sq->uar, !!MLX5_CAP_GEN(mdev, bf));
 902         if (err)
 903                 return err;
 904
 905         param->wq.db_numa_node = cpu_to_node(c->cpu);
 906
 907         err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq,
 908                                  &sq->wq_ctrl);
 909         if (err)
 910                 goto err_unmap_free_uar;
 911
 912         sq->wq.db       = &sq->wq.db[MLX5_SND_DBR];
 913         if (sq->uar.bf_map) {
 914                 set_bit(MLX5E_SQ_STATE_BF_ENABLE, &sq->state);
 915                 sq->uar_map = sq->uar.bf_map;
 916         } else {
 917                 sq->uar_map = sq->uar.map;
 918         }
 919         sq->bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2;
 920         sq->max_inline  = param->max_inline;
 921         sq->min_inline_mode =
 922                 MLX5_CAP_ETH(mdev, wqe_inline_mode) == MLX5E_INLINE_MODE_VPORT_CONTEXT ?
 923                 param->min_inline_mode : 0;
 924
 925         err = mlx5e_alloc_sq_db(sq, cpu_to_node(c->cpu));
 926         if (err)
 927                 goto err_sq_wq_destroy;
 928
 929         if (sq->type == MLX5E_SQ_TXQ) {
 930                 int txq_ix;
 931
 932                 txq_ix = c->ix + tc * priv->params.num_channels;
 933                 sq->txq = netdev_get_tx_queue(priv->netdev, txq_ix);
 934                 priv->txq_to_sq_map[txq_ix] = sq;
 935         }
 936
 937         sq->edge = (sq->wq.sz_m1 + 1) - mlx5e_sq_get_max_wqebbs(sq->type);
 938         sq->bf_budget = MLX5E_SQ_BF_BUDGET;
 939
 940         return 0;
 941
 942 err_sq_wq_destroy:
 943         mlx5_wq_destroy(&sq->wq_ctrl);
 944
 945 err_unmap_free_uar:
 946         mlx5_unmap_free_uar(mdev, &sq->uar);
 947
 948         return err;
 949 }
 950
 951 static void mlx5e_destroy_sq(struct mlx5e_sq *sq)
 952 {
 953         struct mlx5e_channel *c = sq->channel;
 954         struct mlx5e_priv *priv = c->priv;
 955
 956         mlx5e_free_sq_db(sq);
 957         mlx5_wq_destroy(&sq->wq_ctrl);
 958         mlx5_unmap_free_uar(priv->mdev, &sq->uar);
 959 }
 960
 961 static int mlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param)
 962 {
 963         struct mlx5e_channel *c = sq->channel;
 964         struct mlx5e_priv *priv = c->priv;
 965         struct mlx5_core_dev *mdev = priv->mdev;
 966
 967         void *in;
 968         void *sqc;
 969         void *wq;
 970         int inlen;
 971         int err;
 972
 973         inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
 974                 sizeof(u64) * sq->wq_ctrl.buf.npages;
 975         in = mlx5_vzalloc(inlen);
 976         if (!in)
 977                 return -ENOMEM;
 978
 979         sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
 980         wq = MLX5_ADDR_OF(sqc, sqc, wq);
 981
 982         memcpy(sqc, param->sqc, sizeof(param->sqc));
 983
 984         MLX5_SET(sqc,  sqc, tis_num_0, param->type == MLX5E_SQ_ICO ?
 985                                        0 : priv->tisn[sq->tc]);
 986         MLX5_SET(sqc,  sqc, cqn,                sq->cq.mcq.cqn);
 987         MLX5_SET(sqc,  sqc, min_wqe_inline_mode, sq->min_inline_mode);
 988         MLX5_SET(sqc,  sqc, state,              MLX5_SQC_STATE_RST);
 989         MLX5_SET(sqc,  sqc, tis_lst_sz, param->type == MLX5E_SQ_ICO ? 0 : 1);
 990         MLX5_SET(sqc,  sqc, flush_in_error_en,  1);
 991
 992         MLX5_SET(wq,   wq, wq_type,       MLX5_WQ_TYPE_CYCLIC);
 993         MLX5_SET(wq,   wq, uar_page,      sq->uar.index);
 994         MLX5_SET(wq,   wq, log_wq_pg_sz,  sq->wq_ctrl.buf.page_shift -
 995                                           MLX5_ADAPTER_PAGE_SHIFT);
 996         MLX5_SET64(wq, wq, dbr_addr,      sq->wq_ctrl.db.dma);
 997
 998         mlx5_fill_page_array(&sq->wq_ctrl.buf,
 999                              (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
1000
1001         err = mlx5_core_create_sq(mdev, in, inlen, &sq->sqn);
1002
1003         kvfree(in);
1004
1005         return err;
1006 }
1007
1008 static int mlx5e_modify_sq(struct mlx5e_sq *sq, int curr_state,
1009                            int next_state, bool update_rl, int rl_index)
1010 {
1011         struct mlx5e_channel *c = sq->channel;
1012         struct mlx5e_priv *priv = c->priv;
1013         struct mlx5_core_dev *mdev = priv->mdev;
1014
1015         void *in;
1016         void *sqc;
1017         int inlen;
1018         int err;
1019
1020         inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
1021         in = mlx5_vzalloc(inlen);
1022         if (!in)
1023                 return -ENOMEM;
1024
1025         sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
1026
1027         MLX5_SET(modify_sq_in, in, sq_state, curr_state);
1028         MLX5_SET(sqc, sqc, state, next_state);
1029         if (update_rl && next_state == MLX5_SQC_STATE_RDY) {
1030                 MLX5_SET64(modify_sq_in, in, modify_bitmask, 1);
1031                 MLX5_SET(sqc,  sqc, packet_pacing_rate_limit_index, rl_index);
1032         }
1033
1034         err = mlx5_core_modify_sq(mdev, sq->sqn, in, inlen);
1035
1036         kvfree(in);
1037
1038         return err;
1039 }
1040
1041 static void mlx5e_disable_sq(struct mlx5e_sq *sq)
1042 {
1043         struct mlx5e_channel *c = sq->channel;
1044         struct mlx5e_priv *priv = c->priv;
1045         struct mlx5_core_dev *mdev = priv->mdev;
1046
1047         mlx5_core_destroy_sq(mdev, sq->sqn);
1048         if (sq->rate_limit)
1049                 mlx5_rl_remove_rate(mdev, sq->rate_limit);
1050 }
1051
1052 static int mlx5e_open_sq(struct mlx5e_channel *c,
1053                          int tc,
1054                          struct mlx5e_sq_param *param,
1055                          struct mlx5e_sq *sq)
1056 {
1057         int err;
1058
1059         err = mlx5e_create_sq(c, tc, param, sq);
1060         if (err)
1061                 return err;
1062
1063         err = mlx5e_enable_sq(sq, param);
1064         if (err)
1065                 goto err_destroy_sq;
1066
1067         err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY,
1068                               false, 0);
1069         if (err)
1070                 goto err_disable_sq;
1071
1072         if (sq->txq) {
1073                 netdev_tx_reset_queue(sq->txq);
1074                 netif_tx_start_queue(sq->txq);
1075         }
1076
1077         return 0;
1078
1079 err_disable_sq:
1080         mlx5e_disable_sq(sq);
1081 err_destroy_sq:
1082         mlx5e_destroy_sq(sq);
1083
1084         return err;
1085 }
1086
1087 static inline void netif_tx_disable_queue(struct netdev_queue *txq)
1088 {
1089         __netif_tx_lock_bh(txq);
1090         netif_tx_stop_queue(txq);
1091         __netif_tx_unlock_bh(txq);
1092 }
1093
1094 static void mlx5e_close_sq(struct mlx5e_sq *sq)
1095 {
1096         set_bit(MLX5E_SQ_STATE_FLUSH, &sq->state);
1097         /* prevent netif_tx_wake_queue */
1098         napi_synchronize(&sq->channel->napi);
1099
1100         if (sq->txq) {
1101                 netif_tx_disable_queue(sq->txq);
1102
1103                 /* last doorbell out, godspeed .. */
1104                 if (mlx5e_sq_has_room_for(sq, 1)) {
1105                         sq->db.txq.skb[(sq->pc & sq->wq.sz_m1)] = NULL;
1106                         mlx5e_send_nop(sq, true);
1107                 }
1108         }
1109
1110         mlx5e_disable_sq(sq);
1111         mlx5e_free_sq_descs(sq);
1112         mlx5e_destroy_sq(sq);
1113 }
1114
1115 static int mlx5e_create_cq(struct mlx5e_channel *c,
1116                            struct mlx5e_cq_param *param,
1117                            struct mlx5e_cq *cq)
1118 {
1119         struct mlx5e_priv *priv = c->priv;
1120         struct mlx5_core_dev *mdev = priv->mdev;
1121         struct mlx5_core_cq *mcq = &cq->mcq;
1122         int eqn_not_used;
1123         unsigned int irqn;
1124         int err;
1125         u32 i;
1126
1127         param->wq.buf_numa_node = cpu_to_node(c->cpu);
1128         param->wq.db_numa_node  = cpu_to_node(c->cpu);
1129         param->eq_ix   = c->ix;
1130
1131         err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
1132                                &cq->wq_ctrl);
1133         if (err)
1134                 return err;
1135
1136         mlx5_vector2eqn(mdev, param->eq_ix, &eqn_not_used, &irqn);
1137
1138         cq->napi        = &c->napi;
1139
1140         mcq->cqe_sz     = 64;
1141         mcq->set_ci_db  = cq->wq_ctrl.db.db;
1142         mcq->arm_db     = cq->wq_ctrl.db.db + 1;
1143         *mcq->set_ci_db = 0;
1144         *mcq->arm_db    = 0;
1145         mcq->vector     = param->eq_ix;
1146         mcq->comp       = mlx5e_completion_event;
1147         mcq->event      = mlx5e_cq_error_event;
1148         mcq->irqn       = irqn;
1149         mcq->uar        = &mdev->mlx5e_res.cq_uar;
1150
1151         for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
1152                 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
1153
1154                 cqe->op_own = 0xf1;
1155         }
1156
1157         cq->channel = c;
1158         cq->priv = priv;
1159
1160         return 0;
1161 }
1162
1163 static void mlx5e_destroy_cq(struct mlx5e_cq *cq)
1164 {
1165         mlx5_wq_destroy(&cq->wq_ctrl);
1166 }
1167
1168 static int mlx5e_enable_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
1169 {
1170         struct mlx5e_priv *priv = cq->priv;
1171         struct mlx5_core_dev *mdev = priv->mdev;
1172         struct mlx5_core_cq *mcq = &cq->mcq;
1173
1174         void *in;
1175         void *cqc;
1176         int inlen;
1177         unsigned int irqn_not_used;
1178         int eqn;
1179         int err;
1180
1181         inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
1182                 sizeof(u64) * cq->wq_ctrl.buf.npages;
1183         in = mlx5_vzalloc(inlen);
1184         if (!in)
1185                 return -ENOMEM;
1186
1187         cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
1188
1189         memcpy(cqc, param->cqc, sizeof(param->cqc));
1190
1191         mlx5_fill_page_array(&cq->wq_ctrl.buf,
1192                              (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));
1193
1194         mlx5_vector2eqn(mdev, param->eq_ix, &eqn, &irqn_not_used);
1195
1196         MLX5_SET(cqc,   cqc, cq_period_mode, param->cq_period_mode);
1197         MLX5_SET(cqc,   cqc, c_eqn,         eqn);
1198         MLX5_SET(cqc,   cqc, uar_page,      mcq->uar->index);
1199         MLX5_SET(cqc,   cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
1200                                             MLX5_ADAPTER_PAGE_SHIFT);
1201         MLX5_SET64(cqc, cqc, dbr_addr,      cq->wq_ctrl.db.dma);
1202
1203         err = mlx5_core_create_cq(mdev, mcq, in, inlen);
1204
1205         kvfree(in);
1206
1207         if (err)
1208                 return err;
1209
1210         mlx5e_cq_arm(cq);
1211
1212         return 0;
1213 }
1214
1215 static void mlx5e_disable_cq(struct mlx5e_cq *cq)
1216 {
1217         struct mlx5e_priv *priv = cq->priv;
1218         struct mlx5_core_dev *mdev = priv->mdev;
1219
1220         mlx5_core_destroy_cq(mdev, &cq->mcq);
1221 }
1222
1223 static int mlx5e_open_cq(struct mlx5e_channel *c,
1224                          struct mlx5e_cq_param *param,
1225                          struct mlx5e_cq *cq,
1226                          struct mlx5e_cq_moder moderation)
1227 {
1228         int err;
1229         struct mlx5e_priv *priv = c->priv;
1230         struct mlx5_core_dev *mdev = priv->mdev;
1231
1232         err = mlx5e_create_cq(c, param, cq);
1233         if (err)
1234                 return err;
1235
1236         err = mlx5e_enable_cq(cq, param);
1237         if (err)
1238                 goto err_destroy_cq;
1239
1240         if (MLX5_CAP_GEN(mdev, cq_moderation))
1241                 mlx5_core_modify_cq_moderation(mdev, &cq->mcq,
1242                                                moderation.usec,
1243                                                moderation.pkts);
1244         return 0;
1245
1246 err_destroy_cq:
1247         mlx5e_destroy_cq(cq);
1248
1249         return err;
1250 }
1251
1252 static void mlx5e_close_cq(struct mlx5e_cq *cq)
1253 {
1254         mlx5e_disable_cq(cq);
1255         mlx5e_destroy_cq(cq);
1256 }
1257
1258 static int mlx5e_get_cpu(struct mlx5e_priv *priv, int ix)
1259 {
1260         return cpumask_first(priv->mdev->priv.irq_info[ix].mask);
1261 }
1262
1263 static int mlx5e_open_tx_cqs(struct mlx5e_channel *c,
1264                              struct mlx5e_channel_param *cparam)
1265 {
1266         struct mlx5e_priv *priv = c->priv;
1267         int err;
1268         int tc;
1269
1270         for (tc = 0; tc < c->num_tc; tc++) {
1271                 err = mlx5e_open_cq(c, &cparam->tx_cq, &c->sq[tc].cq,
1272                                     priv->params.tx_cq_moderation);
1273                 if (err)
1274                         goto err_close_tx_cqs;
1275         }
1276
1277         return 0;
1278
1279 err_close_tx_cqs:
1280         for (tc--; tc >= 0; tc--)
1281                 mlx5e_close_cq(&c->sq[tc].cq);
1282
1283         return err;
1284 }
1285
1286 static void mlx5e_close_tx_cqs(struct mlx5e_channel *c)
1287 {
1288         int tc;
1289
1290         for (tc = 0; tc < c->num_tc; tc++)
1291                 mlx5e_close_cq(&c->sq[tc].cq);
1292 }
1293
1294 static int mlx5e_open_sqs(struct mlx5e_channel *c,
1295                           struct mlx5e_channel_param *cparam)
1296 {
1297         int err;
1298         int tc;
1299
1300         for (tc = 0; tc < c->num_tc; tc++) {
1301                 err = mlx5e_open_sq(c, tc, &cparam->sq, &c->sq[tc]);
1302                 if (err)
1303                         goto err_close_sqs;
1304         }
1305
1306         return 0;
1307
1308 err_close_sqs:
1309         for (tc--; tc >= 0; tc--)
1310                 mlx5e_close_sq(&c->sq[tc]);
1311
1312         return err;
1313 }
1314
1315 static void mlx5e_close_sqs(struct mlx5e_channel *c)
1316 {
1317         int tc;
1318
1319         for (tc = 0; tc < c->num_tc; tc++)
1320                 mlx5e_close_sq(&c->sq[tc]);
1321 }
1322
1323 static void mlx5e_build_channeltc_to_txq_map(struct mlx5e_priv *priv, int ix)
1324 {
1325         int i;
1326
1327         for (i = 0; i < priv->profile->max_tc; i++)
1328                 priv->channeltc_to_txq_map[ix][i] =
1329                         ix + i * priv->params.num_channels;
1330 }
1331
1332 static int mlx5e_set_sq_maxrate(struct net_device *dev,
1333                                 struct mlx5e_sq *sq, u32 rate)
1334 {
1335         struct mlx5e_priv *priv = netdev_priv(dev);
1336         struct mlx5_core_dev *mdev = priv->mdev;
1337         u16 rl_index = 0;
1338         int err;
1339
1340         if (rate == sq->rate_limit)
1341                 /* nothing to do */
1342                 return 0;
1343
1344         if (sq->rate_limit)
1345                 /* remove current rl index to free space to next ones */
1346                 mlx5_rl_remove_rate(mdev, sq->rate_limit);
1347
1348         sq->rate_limit = 0;
1349
1350         if (rate) {
1351                 err = mlx5_rl_add_rate(mdev, rate, &rl_index);
1352                 if (err) {
1353                         netdev_err(dev, "Failed configuring rate %u: %d\n",
1354                                    rate, err);
1355                         return err;
1356                 }
1357         }
1358
1359         err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY,
1360                               MLX5_SQC_STATE_RDY, true, rl_index);
1361         if (err) {
1362                 netdev_err(dev, "Failed configuring rate %u: %d\n",
1363                            rate, err);
1364                 /* remove the rate from the table */
1365                 if (rate)
1366                         mlx5_rl_remove_rate(mdev, rate);
1367                 return err;
1368         }
1369
1370         sq->rate_limit = rate;
1371         return 0;
1372 }
1373
1374 static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
1375 {
1376         struct mlx5e_priv *priv = netdev_priv(dev);
1377         struct mlx5_core_dev *mdev = priv->mdev;
1378         struct mlx5e_sq *sq = priv->txq_to_sq_map[index];
1379         int err = 0;
1380
1381         if (!mlx5_rl_is_supported(mdev)) {
1382                 netdev_err(dev, "Rate limiting is not supported on this device\n");
1383                 return -EINVAL;
1384         }
1385
1386         /* rate is given in Mb/sec, HW config is in Kb/sec */
1387         rate = rate << 10;
1388
1389         /* Check whether rate in valid range, 0 is always valid */
1390         if (rate && !mlx5_rl_is_in_range(mdev, rate)) {
1391                 netdev_err(dev, "TX rate %u, is not in range\n", rate);
1392                 return -ERANGE;
1393         }
1394
1395         mutex_lock(&priv->state_lock);
1396         if (test_bit(MLX5E_STATE_OPENED, &priv->state))
1397                 err = mlx5e_set_sq_maxrate(dev, sq, rate);
1398         if (!err)
1399                 priv->tx_rates[index] = rate;
1400         mutex_unlock(&priv->state_lock);
1401
1402         return err;
1403 }
1404
1405 static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
1406                               struct mlx5e_channel_param *cparam,
1407                               struct mlx5e_channel **cp)
1408 {
1409         struct mlx5e_cq_moder icosq_cq_moder = {0, 0};
1410         struct net_device *netdev = priv->netdev;
1411         struct mlx5e_cq_moder rx_cq_profile;
1412         int cpu = mlx5e_get_cpu(priv, ix);
1413         struct mlx5e_channel *c;
1414         struct mlx5e_sq *sq;
1415         int err;
1416         int i;
1417
1418         c = kzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu));
1419         if (!c)
1420                 return -ENOMEM;
1421
1422         c->priv     = priv;
1423         c->ix       = ix;
1424         c->cpu      = cpu;
1425         c->pdev     = &priv->mdev->pdev->dev;
1426         c->netdev   = priv->netdev;
1427         c->mkey_be  = cpu_to_be32(priv->mdev->mlx5e_res.mkey.key);
1428         c->num_tc   = priv->params.num_tc;
1429
1430         if (priv->params.rx_am_enabled)
1431                 rx_cq_profile = mlx5e_am_get_def_profile(priv->params.rx_cq_period_mode);
1432         else
1433                 rx_cq_profile = priv->params.rx_cq_moderation;
1434
1435         mlx5e_build_channeltc_to_txq_map(priv, ix);
1436
1437         netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
1438
1439         err = mlx5e_open_cq(c, &cparam->icosq_cq, &c->icosq.cq, icosq_cq_moder);
1440         if (err)
1441                 goto err_napi_del;
1442
1443         err = mlx5e_open_tx_cqs(c, cparam);
1444         if (err)
1445                 goto err_close_icosq_cq;
1446
1447         err = mlx5e_open_cq(c, &cparam->rx_cq, &c->rq.cq,
1448                             rx_cq_profile);
1449         if (err)
1450                 goto err_close_tx_cqs;
1451
1452         napi_enable(&c->napi);
1453
1454         err = mlx5e_open_sq(c, 0, &cparam->icosq, &c->icosq);
1455         if (err)
1456                 goto err_disable_napi;
1457
1458         err = mlx5e_open_sqs(c, cparam);
1459         if (err)
1460                 goto err_close_icosq;
1461
1462         for (i = 0; i < priv->params.num_tc; i++) {
1463                 u32 txq_ix = priv->channeltc_to_txq_map[ix][i];
1464
1465                 if (priv->tx_rates[txq_ix]) {
1466                         sq = priv->txq_to_sq_map[txq_ix];
1467                         mlx5e_set_sq_maxrate(priv->netdev, sq,
1468                                              priv->tx_rates[txq_ix]);
1469                 }
1470         }
1471
1472         if (priv->xdp_prog) {
1473                 /* XDP SQ CQ params are same as normal TXQ sq CQ params */
1474                 err = mlx5e_open_cq(c, &cparam->tx_cq, &c->xdp_sq.cq,
1475                                     priv->params.tx_cq_moderation);
1476                 if (err)
1477                         goto err_close_sqs;
1478
1479                 err = mlx5e_open_sq(c, 0, &cparam->xdp_sq, &c->xdp_sq);
1480                 if (err) {
1481                         mlx5e_close_cq(&c->xdp_sq.cq);
1482                         goto err_close_sqs;
1483                 }
1484         }
1485
1486         c->xdp = !!priv->xdp_prog;
1487         err = mlx5e_open_rq(c, &cparam->rq, &c->rq);
1488         if (err)
1489                 goto err_close_xdp_sq;
1490
1491         netif_set_xps_queue(netdev, get_cpu_mask(c->cpu), ix);
1492         *cp = c;
1493
1494         return 0;
1495 err_close_xdp_sq:
1496         mlx5e_close_sq(&c->xdp_sq);
1497
1498 err_close_sqs:
1499         mlx5e_close_sqs(c);
1500
1501 err_close_icosq:
1502         mlx5e_close_sq(&c->icosq);
1503
1504 err_disable_napi:
1505         napi_disable(&c->napi);
1506         mlx5e_close_cq(&c->rq.cq);
1507
1508 err_close_tx_cqs:
1509         mlx5e_close_tx_cqs(c);
1510
1511 err_close_icosq_cq:
1512         mlx5e_close_cq(&c->icosq.cq);
1513
1514 err_napi_del:
1515         netif_napi_del(&c->napi);
1516         napi_hash_del(&c->napi);
1517         kfree(c);
1518
1519         return err;
1520 }
1521
1522 static void mlx5e_close_channel(struct mlx5e_channel *c)
1523 {
1524         mlx5e_close_rq(&c->rq);
1525         if (c->xdp)
1526                 mlx5e_close_sq(&c->xdp_sq);
1527         mlx5e_close_sqs(c);
1528         mlx5e_close_sq(&c->icosq);
1529         napi_disable(&c->napi);
1530         if (c->xdp)
1531                 mlx5e_close_cq(&c->xdp_sq.cq);
1532         mlx5e_close_cq(&c->rq.cq);
1533         mlx5e_close_tx_cqs(c);
1534         mlx5e_close_cq(&c->icosq.cq);
1535         netif_napi_del(&c->napi);
1536
1537         napi_hash_del(&c->napi);
1538         synchronize_rcu();
1539
1540         kfree(c);
1541 }
1542
1543 static void mlx5e_build_rq_param(struct mlx5e_priv *priv,
1544                                  struct mlx5e_rq_param *param)
1545 {
1546         void *rqc = param->rqc;
1547         void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
1548
1549         switch (priv->params.rq_wq_type) {
1550         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
1551                 MLX5_SET(wq, wq, log_wqe_num_of_strides,
1552                          priv->params.mpwqe_log_num_strides - 9);
1553                 MLX5_SET(wq, wq, log_wqe_stride_size,
1554                          priv->params.mpwqe_log_stride_sz - 6);
1555                 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ);
1556                 break;
1557         default: /* MLX5_WQ_TYPE_LINKED_LIST */
1558                 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
1559         }
1560
1561         MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
1562         MLX5_SET(wq, wq, log_wq_stride,    ilog2(sizeof(struct mlx5e_rx_wqe)));
1563         MLX5_SET(wq, wq, log_wq_sz,        priv->params.log_rq_size);
1564         MLX5_SET(wq, wq, pd,               priv->mdev->mlx5e_res.pdn);
1565         MLX5_SET(rqc, rqc, counter_set_id, priv->q_counter);
1566
1567         param->wq.buf_numa_node = dev_to_node(&priv->mdev->pdev->dev);
1568         param->wq.linear = 1;
1569
1570         param->am_enabled = priv->params.rx_am_enabled;
1571 }
1572
1573 static void mlx5e_build_drop_rq_param(struct mlx5e_rq_param *param)
1574 {
1575         void *rqc = param->rqc;
1576         void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
1577
1578         MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
1579         MLX5_SET(wq, wq, log_wq_stride,    ilog2(sizeof(struct mlx5e_rx_wqe)));
1580 }
1581
1582 static void mlx5e_build_sq_param_common(struct mlx5e_priv *priv,
1583                                         struct mlx5e_sq_param *param)
1584 {
1585         void *sqc = param->sqc;
1586         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1587
1588         MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
1589         MLX5_SET(wq, wq, pd,            priv->mdev->mlx5e_res.pdn);
1590
1591         param->wq.buf_numa_node = dev_to_node(&priv->mdev->pdev->dev);
1592 }
1593
1594 static void mlx5e_build_sq_param(struct mlx5e_priv *priv,
1595                                  struct mlx5e_sq_param *param)
1596 {
1597         void *sqc = param->sqc;
1598         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1599
1600         mlx5e_build_sq_param_common(priv, param);
1601         MLX5_SET(wq, wq, log_wq_sz,     priv->params.log_sq_size);
1602
1603         param->max_inline = priv->params.tx_max_inline;
1604         param->min_inline_mode = priv->params.tx_min_inline_mode;
1605         param->type = MLX5E_SQ_TXQ;
1606 }
1607
1608 static void mlx5e_build_common_cq_param(struct mlx5e_priv *priv,
1609                                         struct mlx5e_cq_param *param)
1610 {
1611         void *cqc = param->cqc;
1612
1613         MLX5_SET(cqc, cqc, uar_page, priv->mdev->mlx5e_res.cq_uar.index);
1614 }
1615
1616 static void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
1617                                     struct mlx5e_cq_param *param)
1618 {
1619         void *cqc = param->cqc;
1620         u8 log_cq_size;
1621
1622         switch (priv->params.rq_wq_type) {
1623         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
1624                 log_cq_size = priv->params.log_rq_size +
1625                         priv->params.mpwqe_log_num_strides;
1626                 break;
1627         default: /* MLX5_WQ_TYPE_LINKED_LIST */
1628                 log_cq_size = priv->params.log_rq_size;
1629         }
1630
1631         MLX5_SET(cqc, cqc, log_cq_size, log_cq_size);
1632         if (priv->params.rx_cqe_compress) {
1633                 MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_CSUM);
1634                 MLX5_SET(cqc, cqc, cqe_comp_en, 1);
1635         }
1636
1637         mlx5e_build_common_cq_param(priv, param);
1638
1639         param->cq_period_mode = priv->params.rx_cq_period_mode;
1640 }
1641
1642 static void mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
1643                                     struct mlx5e_cq_param *param)
1644 {
1645         void *cqc = param->cqc;
1646
1647         MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_sq_size);
1648
1649         mlx5e_build_common_cq_param(priv, param);
1650
1651         param->cq_period_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
1652 }
1653
1654 static void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv,
1655                                      struct mlx5e_cq_param *param,
1656                                      u8 log_wq_size)
1657 {
1658         void *cqc = param->cqc;
1659
1660         MLX5_SET(cqc, cqc, log_cq_size, log_wq_size);
1661
1662         mlx5e_build_common_cq_param(priv, param);
1663
1664         param->cq_period_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
1665 }
1666
1667 static void mlx5e_build_icosq_param(struct mlx5e_priv *priv,
1668                                     struct mlx5e_sq_param *param,
1669                                     u8 log_wq_size)
1670 {
1671         void *sqc = param->sqc;
1672         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1673
1674         mlx5e_build_sq_param_common(priv, param);
1675
1676         MLX5_SET(wq, wq, log_wq_sz, log_wq_size);
1677         MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(priv->mdev, reg_umr_sq));
1678
1679         param->type = MLX5E_SQ_ICO;
1680 }
1681
1682 static void mlx5e_build_xdpsq_param(struct mlx5e_priv *priv,
1683                                     struct mlx5e_sq_param *param)
1684 {
1685         void *sqc = param->sqc;
1686         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1687
1688         mlx5e_build_sq_param_common(priv, param);
1689         MLX5_SET(wq, wq, log_wq_sz,     priv->params.log_sq_size);
1690
1691         param->max_inline = priv->params.tx_max_inline;
1692         /* FOR XDP SQs will support only L2 inline mode */
1693         param->min_inline_mode = MLX5_INLINE_MODE_NONE;
1694         param->type = MLX5E_SQ_XDP;
1695 }
1696
1697 static void mlx5e_build_channel_param(struct mlx5e_priv *priv, struct mlx5e_channel_param *cparam)
1698 {
1699         u8 icosq_log_wq_sz = MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
1700
1701         mlx5e_build_rq_param(priv, &cparam->rq);
1702         mlx5e_build_sq_param(priv, &cparam->sq);
1703         mlx5e_build_xdpsq_param(priv, &cparam->xdp_sq);
1704         mlx5e_build_icosq_param(priv, &cparam->icosq, icosq_log_wq_sz);
1705         mlx5e_build_rx_cq_param(priv, &cparam->rx_cq);
1706         mlx5e_build_tx_cq_param(priv, &cparam->tx_cq);
1707         mlx5e_build_ico_cq_param(priv, &cparam->icosq_cq, icosq_log_wq_sz);
1708 }
1709
1710 static int mlx5e_open_channels(struct mlx5e_priv *priv)
1711 {
1712         struct mlx5e_channel_param *cparam;
1713         int nch = priv->params.num_channels;
1714         int err = -ENOMEM;
1715         int i;
1716         int j;
1717
1718         priv->channel = kcalloc(nch, sizeof(struct mlx5e_channel *),
1719                                 GFP_KERNEL);
1720
1721         priv->txq_to_sq_map = kcalloc(nch * priv->params.num_tc,
1722                                       sizeof(struct mlx5e_sq *), GFP_KERNEL);
1723
1724         cparam = kzalloc(sizeof(struct mlx5e_channel_param), GFP_KERNEL);
1725
1726         if (!priv->channel || !priv->txq_to_sq_map || !cparam)
1727                 goto err_free_txq_to_sq_map;
1728
1729         mlx5e_build_channel_param(priv, cparam);
1730
1731         for (i = 0; i < nch; i++) {
1732                 err = mlx5e_open_channel(priv, i, cparam, &priv->channel[i]);
1733                 if (err)
1734                         goto err_close_channels;
1735         }
1736
1737         for (j = 0; j < nch; j++) {
1738                 err = mlx5e_wait_for_min_rx_wqes(&priv->channel[j]->rq);
1739                 if (err)
1740                         goto err_close_channels;
1741         }
1742
1743         /* FIXME: This is a W/A for tx timeout watch dog false alarm when
1744          * polling for inactive tx queues.
1745          */
1746         netif_tx_start_all_queues(priv->netdev);
1747
1748         kfree(cparam);
1749         return 0;
1750
1751 err_close_channels:
1752         for (i--; i >= 0; i--)
1753                 mlx5e_close_channel(priv->channel[i]);
1754
1755 err_free_txq_to_sq_map:
1756         kfree(priv->txq_to_sq_map);
1757         kfree(priv->channel);
1758         kfree(cparam);
1759
1760         return err;
1761 }
1762
1763 static void mlx5e_close_channels(struct mlx5e_priv *priv)
1764 {
1765         int i;
1766
1767         /* FIXME: This is a W/A only for tx timeout watch dog false alarm when
1768          * polling for inactive tx queues.
1769          */
1770         netif_tx_stop_all_queues(priv->netdev);
1771         netif_tx_disable(priv->netdev);
1772
1773         for (i = 0; i < priv->params.num_channels; i++)
1774                 mlx5e_close_channel(priv->channel[i]);
1775
1776         kfree(priv->txq_to_sq_map);
1777         kfree(priv->channel);
1778 }
1779
1780 static int mlx5e_rx_hash_fn(int hfunc)
1781 {
1782         return (hfunc == ETH_RSS_HASH_TOP) ?
1783                MLX5_RX_HASH_FN_TOEPLITZ :
1784                MLX5_RX_HASH_FN_INVERTED_XOR8;
1785 }
1786
1787 static int mlx5e_bits_invert(unsigned long a, int size)
1788 {
1789         int inv = 0;
1790         int i;
1791
1792         for (i = 0; i < size; i++)
1793                 inv |= (test_bit(size - i - 1, &a) ? 1 : 0) << i;
1794
1795         return inv;
1796 }
1797
1798 static void mlx5e_fill_indir_rqt_rqns(struct mlx5e_priv *priv, void *rqtc)
1799 {
1800         int i;
1801
1802         for (i = 0; i < MLX5E_INDIR_RQT_SIZE; i++) {
1803                 int ix = i;
1804                 u32 rqn;
1805
1806                 if (priv->params.rss_hfunc == ETH_RSS_HASH_XOR)
1807                         ix = mlx5e_bits_invert(i, MLX5E_LOG_INDIR_RQT_SIZE);
1808
1809                 ix = priv->params.indirection_rqt[ix];
1810                 rqn = test_bit(MLX5E_STATE_OPENED, &priv->state) ?
1811                                 priv->channel[ix]->rq.rqn :
1812                                 priv->drop_rq.rqn;
1813                 MLX5_SET(rqtc, rqtc, rq_num[i], rqn);
1814         }
1815 }
1816
1817 static void mlx5e_fill_direct_rqt_rqn(struct mlx5e_priv *priv, void *rqtc,
1818                                       int ix)
1819 {
1820         u32 rqn = test_bit(MLX5E_STATE_OPENED, &priv->state) ?
1821                         priv->channel[ix]->rq.rqn :
1822                         priv->drop_rq.rqn;
1823
1824         MLX5_SET(rqtc, rqtc, rq_num[0], rqn);
1825 }
1826
1827 static int mlx5e_create_rqt(struct mlx5e_priv *priv, int sz,
1828                             int ix, struct mlx5e_rqt *rqt)
1829 {
1830         struct mlx5_core_dev *mdev = priv->mdev;
1831         void *rqtc;
1832         int inlen;
1833         int err;
1834         u32 *in;
1835
1836         inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
1837         in = mlx5_vzalloc(inlen);
1838         if (!in)
1839                 return -ENOMEM;
1840
1841         rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
1842
1843         MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
1844         MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
1845
1846         if (sz > 1) /* RSS */
1847                 mlx5e_fill_indir_rqt_rqns(priv, rqtc);
1848         else
1849                 mlx5e_fill_direct_rqt_rqn(priv, rqtc, ix);
1850
1851         err = mlx5_core_create_rqt(mdev, in, inlen, &rqt->rqtn);
1852         if (!err)
1853                 rqt->enabled = true;
1854
1855         kvfree(in);
1856         return err;
1857 }
1858
1859 void mlx5e_destroy_rqt(struct mlx5e_priv *priv, struct mlx5e_rqt *rqt)
1860 {
1861         rqt->enabled = false;
1862         mlx5_core_destroy_rqt(priv->mdev, rqt->rqtn);
1863 }
1864
1865 static int mlx5e_create_indirect_rqts(struct mlx5e_priv *priv)
1866 {
1867         struct mlx5e_rqt *rqt = &priv->indir_rqt;
1868
1869         return mlx5e_create_rqt(priv, MLX5E_INDIR_RQT_SIZE, 0, rqt);
1870 }
1871
1872 int mlx5e_create_direct_rqts(struct mlx5e_priv *priv)
1873 {
1874         struct mlx5e_rqt *rqt;
1875         int err;
1876         int ix;
1877
1878         for (ix = 0; ix < priv->profile->max_nch(priv->mdev); ix++) {
1879                 rqt = &priv->direct_tir[ix].rqt;
1880                 err = mlx5e_create_rqt(priv, 1 /*size */, ix, rqt);
1881                 if (err)
1882                         goto err_destroy_rqts;
1883         }
1884
1885         return 0;
1886
1887 err_destroy_rqts:
1888         for (ix--; ix >= 0; ix--)
1889                 mlx5e_destroy_rqt(priv, &priv->direct_tir[ix].rqt);
1890
1891         return err;
1892 }
1893
1894 int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz, int ix)
1895 {
1896         struct mlx5_core_dev *mdev = priv->mdev;
1897         void *rqtc;
1898         int inlen;
1899         u32 *in;
1900         int err;
1901
1902         inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32) * sz;
1903         in = mlx5_vzalloc(inlen);
1904         if (!in)
1905                 return -ENOMEM;
1906
1907         rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx);
1908
1909         MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
1910         if (sz > 1) /* RSS */
1911                 mlx5e_fill_indir_rqt_rqns(priv, rqtc);
1912         else
1913                 mlx5e_fill_direct_rqt_rqn(priv, rqtc, ix);
1914
1915         MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1);
1916
1917         err = mlx5_core_modify_rqt(mdev, rqtn, in, inlen);
1918
1919         kvfree(in);
1920
1921         return err;
1922 }
1923
1924 static void mlx5e_redirect_rqts(struct mlx5e_priv *priv)
1925 {
1926         u32 rqtn;
1927         int ix;
1928
1929         if (priv->indir_rqt.enabled) {
1930                 rqtn = priv->indir_rqt.rqtn;
1931                 mlx5e_redirect_rqt(priv, rqtn, MLX5E_INDIR_RQT_SIZE, 0);
1932         }
1933
1934         for (ix = 0; ix < priv->params.num_channels; ix++) {
1935                 if (!priv->direct_tir[ix].rqt.enabled)
1936                         continue;
1937                 rqtn = priv->direct_tir[ix].rqt.rqtn;
1938                 mlx5e_redirect_rqt(priv, rqtn, 1, ix);
1939         }
1940 }
1941
1942 static void mlx5e_build_tir_ctx_lro(void *tirc, struct mlx5e_priv *priv)
1943 {
1944         if (!priv->params.lro_en)
1945                 return;
1946
1947 #define ROUGH_MAX_L2_L3_HDR_SZ 256
1948
1949         MLX5_SET(tirc, tirc, lro_enable_mask,
1950                  MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO |
1951                  MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO);
1952         MLX5_SET(tirc, tirc, lro_max_ip_payload_size,
1953                  (priv->params.lro_wqe_sz -
1954                   ROUGH_MAX_L2_L3_HDR_SZ) >> 8);
1955         MLX5_SET(tirc, tirc, lro_timeout_period_usecs,
1956                  MLX5_CAP_ETH(priv->mdev,
1957                               lro_timer_supported_periods[2]));
1958 }
1959
1960 void mlx5e_build_tir_ctx_hash(void *tirc, struct mlx5e_priv *priv)
1961 {
1962         MLX5_SET(tirc, tirc, rx_hash_fn,
1963                  mlx5e_rx_hash_fn(priv->params.rss_hfunc));
1964         if (priv->params.rss_hfunc == ETH_RSS_HASH_TOP) {
1965                 void *rss_key = MLX5_ADDR_OF(tirc, tirc,
1966                                              rx_hash_toeplitz_key);
1967                 size_t len = MLX5_FLD_SZ_BYTES(tirc,
1968                                                rx_hash_toeplitz_key);
1969
1970                 MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
1971                 memcpy(rss_key, priv->params.toeplitz_hash_key, len);
1972         }
1973 }
1974
1975 static int mlx5e_modify_tirs_lro(struct mlx5e_priv *priv)
1976 {
1977         struct mlx5_core_dev *mdev = priv->mdev;
1978
1979         void *in;
1980         void *tirc;
1981         int inlen;
1982         int err;
1983         int tt;
1984         int ix;
1985
1986         inlen = MLX5_ST_SZ_BYTES(modify_tir_in);
1987         in = mlx5_vzalloc(inlen);
1988         if (!in)
1989                 return -ENOMEM;
1990
1991         MLX5_SET(modify_tir_in, in, bitmask.lro, 1);
1992         tirc = MLX5_ADDR_OF(modify_tir_in, in, ctx);
1993
1994         mlx5e_build_tir_ctx_lro(tirc, priv);
1995
1996         for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
1997                 err = mlx5_core_modify_tir(mdev, priv->indir_tir[tt].tirn, in,
1998                                            inlen);
1999                 if (err)
2000                         goto free_in;
2001         }
2002
2003         for (ix = 0; ix < priv->profile->max_nch(priv->mdev); ix++) {
2004                 err = mlx5_core_modify_tir(mdev, priv->direct_tir[ix].tirn,
2005                                            in, inlen);
2006                 if (err)
2007                         goto free_in;
2008         }
2009
2010 free_in:
2011         kvfree(in);
2012
2013         return err;
2014 }
2015
2016 static int mlx5e_set_mtu(struct mlx5e_priv *priv, u16 mtu)
2017 {
2018         struct mlx5_core_dev *mdev = priv->mdev;
2019         u16 hw_mtu = MLX5E_SW2HW_MTU(mtu);
2020         int err;
2021
2022         err = mlx5_set_port_mtu(mdev, hw_mtu, 1);
2023         if (err)
2024                 return err;
2025
2026         /* Update vport context MTU */
2027         mlx5_modify_nic_vport_mtu(mdev, hw_mtu);
2028         return 0;
2029 }
2030
2031 static void mlx5e_query_mtu(struct mlx5e_priv *priv, u16 *mtu)
2032 {
2033         struct mlx5_core_dev *mdev = priv->mdev;
2034         u16 hw_mtu = 0;
2035         int err;
2036
2037         err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu);
2038         if (err || !hw_mtu) /* fallback to port oper mtu */
2039                 mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1);
2040
2041         *mtu = MLX5E_HW2SW_MTU(hw_mtu);
2042 }
2043
2044 static int mlx5e_set_dev_port_mtu(struct net_device *netdev)
2045 {
2046         struct mlx5e_priv *priv = netdev_priv(netdev);
2047         u16 mtu;
2048         int err;
2049
2050         err = mlx5e_set_mtu(priv, netdev->mtu);
2051         if (err)
2052                 return err;
2053
2054         mlx5e_query_mtu(priv, &mtu);
2055         if (mtu != netdev->mtu)
2056                 netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n",
2057                             __func__, mtu, netdev->mtu);
2058
2059         netdev->mtu = mtu;
2060         return 0;
2061 }
2062
2063 static void mlx5e_netdev_set_tcs(struct net_device *netdev)
2064 {
2065         struct mlx5e_priv *priv = netdev_priv(netdev);
2066         int nch = priv->params.num_channels;
2067         int ntc = priv->params.num_tc;
2068         int tc;
2069
2070         netdev_reset_tc(netdev);
2071
2072         if (ntc == 1)
2073                 return;
2074
2075         netdev_set_num_tc(netdev, ntc);
2076
2077         /* Map netdev TCs to offset 0
2078          * We have our own UP to TXQ mapping for QoS
2079          */
2080         for (tc = 0; tc < ntc; tc++)
2081                 netdev_set_tc_queue(netdev, tc, nch, 0);
2082 }
2083
2084 int mlx5e_open_locked(struct net_device *netdev)
2085 {
2086         struct mlx5e_priv *priv = netdev_priv(netdev);
2087         struct mlx5_core_dev *mdev = priv->mdev;
2088         int num_txqs;
2089         int err;
2090
2091         set_bit(MLX5E_STATE_OPENED, &priv->state);
2092
2093         mlx5e_netdev_set_tcs(netdev);
2094
2095         num_txqs = priv->params.num_channels * priv->params.num_tc;
2096         netif_set_real_num_tx_queues(netdev, num_txqs);
2097         netif_set_real_num_rx_queues(netdev, priv->params.num_channels);
2098
2099         err = mlx5e_open_channels(priv);
2100         if (err) {
2101                 netdev_err(netdev, "%s: mlx5e_open_channels failed, %d\n",
2102                            __func__, err);
2103                 goto err_clear_state_opened_flag;
2104         }
2105
2106         err = mlx5e_refresh_tirs_self_loopback_enable(priv->mdev);
2107         if (err) {
2108                 netdev_err(netdev, "%s: mlx5e_refresh_tirs_self_loopback_enable failed, %d\n",
2109                            __func__, err);
2110                 goto err_close_channels;
2111         }
2112
2113         mlx5e_redirect_rqts(priv);
2114         mlx5e_update_carrier(priv);
2115         mlx5e_timestamp_init(priv);
2116 #ifdef CONFIG_RFS_ACCEL
2117         priv->netdev->rx_cpu_rmap = priv->mdev->rmap;
2118 #endif
2119         if (priv->profile->update_stats)
2120                 queue_delayed_work(priv->wq, &priv->update_stats_work, 0);
2121
2122         if (MLX5_CAP_GEN(mdev, vport_group_manager)) {
2123                 err = mlx5e_add_sqs_fwd_rules(priv);
2124                 if (err)
2125                         goto err_close_channels;
2126         }
2127         return 0;
2128
2129 err_close_channels:
2130         mlx5e_close_channels(priv);
2131 err_clear_state_opened_flag:
2132         clear_bit(MLX5E_STATE_OPENED, &priv->state);
2133         return err;
2134 }
2135
2136 int mlx5e_open(struct net_device *netdev)
2137 {
2138         struct mlx5e_priv *priv = netdev_priv(netdev);
2139         int err;
2140
2141         mutex_lock(&priv->state_lock);
2142         err = mlx5e_open_locked(netdev);
2143         mutex_unlock(&priv->state_lock);
2144
2145         return err;
2146 }
2147
2148 int mlx5e_close_locked(struct net_device *netdev)
2149 {
2150         struct mlx5e_priv *priv = netdev_priv(netdev);
2151         struct mlx5_core_dev *mdev = priv->mdev;
2152
2153         /* May already be CLOSED in case a previous configuration operation
2154          * (e.g RX/TX queue size change) that involves close&open failed.
2155          */
2156         if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
2157                 return 0;
2158
2159         clear_bit(MLX5E_STATE_OPENED, &priv->state);
2160
2161         if (MLX5_CAP_GEN(mdev, vport_group_manager))
2162                 mlx5e_remove_sqs_fwd_rules(priv);
2163
2164         mlx5e_timestamp_cleanup(priv);
2165         netif_carrier_off(priv->netdev);
2166         mlx5e_redirect_rqts(priv);
2167         mlx5e_close_channels(priv);
2168
2169         return 0;
2170 }
2171
2172 int mlx5e_close(struct net_device *netdev)
2173 {
2174         struct mlx5e_priv *priv = netdev_priv(netdev);
2175         int err;
2176
2177         if (!netif_device_present(netdev))
2178                 return -ENODEV;
2179
2180         mutex_lock(&priv->state_lock);
2181         err = mlx5e_close_locked(netdev);
2182         mutex_unlock(&priv->state_lock);
2183
2184         return err;
2185 }
2186
2187 static int mlx5e_create_drop_rq(struct mlx5e_priv *priv,
2188                                 struct mlx5e_rq *rq,
2189                                 struct mlx5e_rq_param *param)
2190 {
2191         struct mlx5_core_dev *mdev = priv->mdev;
2192         void *rqc = param->rqc;
2193         void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
2194         int err;
2195
2196         param->wq.db_numa_node = param->wq.buf_numa_node;
2197
2198         err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
2199                                 &rq->wq_ctrl);
2200         if (err)
2201                 return err;
2202
2203         rq->priv = priv;
2204
2205         return 0;
2206 }
2207
2208 static int mlx5e_create_drop_cq(struct mlx5e_priv *priv,
2209                                 struct mlx5e_cq *cq,
2210                                 struct mlx5e_cq_param *param)
2211 {
2212         struct mlx5_core_dev *mdev = priv->mdev;
2213         struct mlx5_core_cq *mcq = &cq->mcq;
2214         int eqn_not_used;
2215         unsigned int irqn;
2216         int err;
2217
2218         err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
2219                                &cq->wq_ctrl);
2220         if (err)
2221                 return err;
2222
2223         mlx5_vector2eqn(mdev, param->eq_ix, &eqn_not_used, &irqn);
2224
2225         mcq->cqe_sz     = 64;
2226         mcq->set_ci_db  = cq->wq_ctrl.db.db;
2227         mcq->arm_db     = cq->wq_ctrl.db.db + 1;
2228         *mcq->set_ci_db = 0;
2229         *mcq->arm_db    = 0;
2230         mcq->vector     = param->eq_ix;
2231         mcq->comp       = mlx5e_completion_event;
2232         mcq->event      = mlx5e_cq_error_event;
2233         mcq->irqn       = irqn;
2234         mcq->uar        = &mdev->mlx5e_res.cq_uar;
2235
2236         cq->priv = priv;
2237
2238         return 0;
2239 }
2240
2241 static int mlx5e_open_drop_rq(struct mlx5e_priv *priv)
2242 {
2243         struct mlx5e_cq_param cq_param;
2244         struct mlx5e_rq_param rq_param;
2245         struct mlx5e_rq *rq = &priv->drop_rq;
2246         struct mlx5e_cq *cq = &priv->drop_rq.cq;
2247         int err;
2248
2249         memset(&cq_param, 0, sizeof(cq_param));
2250         memset(&rq_param, 0, sizeof(rq_param));
2251         mlx5e_build_drop_rq_param(&rq_param);
2252
2253         err = mlx5e_create_drop_cq(priv, cq, &cq_param);
2254         if (err)
2255                 return err;
2256
2257         err = mlx5e_enable_cq(cq, &cq_param);
2258         if (err)
2259                 goto err_destroy_cq;
2260
2261         err = mlx5e_create_drop_rq(priv, rq, &rq_param);
2262         if (err)
2263                 goto err_disable_cq;
2264
2265         err = mlx5e_enable_rq(rq, &rq_param);
2266         if (err)
2267                 goto err_destroy_rq;
2268
2269         return 0;
2270
2271 err_destroy_rq:
2272         mlx5e_destroy_rq(&priv->drop_rq);
2273
2274 err_disable_cq:
2275         mlx5e_disable_cq(&priv->drop_rq.cq);
2276
2277 err_destroy_cq:
2278         mlx5e_destroy_cq(&priv->drop_rq.cq);
2279
2280         return err;
2281 }
2282
2283 static void mlx5e_close_drop_rq(struct mlx5e_priv *priv)
2284 {
2285         mlx5e_disable_rq(&priv->drop_rq);
2286         mlx5e_destroy_rq(&priv->drop_rq);
2287         mlx5e_disable_cq(&priv->drop_rq.cq);
2288         mlx5e_destroy_cq(&priv->drop_rq.cq);
2289 }
2290
2291 static int mlx5e_create_tis(struct mlx5e_priv *priv, int tc)
2292 {
2293         struct mlx5_core_dev *mdev = priv->mdev;
2294         u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {0};
2295         void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
2296
2297         MLX5_SET(tisc, tisc, prio, tc << 1);
2298         MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.td.tdn);
2299
2300         if (mlx5_lag_is_lacp_owner(mdev))
2301                 MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1);
2302
2303         return mlx5_core_create_tis(mdev, in, sizeof(in), &priv->tisn[tc]);
2304 }
2305
2306 static void mlx5e_destroy_tis(struct mlx5e_priv *priv, int tc)
2307 {
2308         mlx5_core_destroy_tis(priv->mdev, priv->tisn[tc]);
2309 }
2310
2311 int mlx5e_create_tises(struct mlx5e_priv *priv)
2312 {
2313         int err;
2314         int tc;
2315
2316         for (tc = 0; tc < priv->profile->max_tc; tc++) {
2317                 err = mlx5e_create_tis(priv, tc);
2318                 if (err)
2319                         goto err_close_tises;
2320         }
2321
2322         return 0;
2323
2324 err_close_tises:
2325         for (tc--; tc >= 0; tc--)
2326                 mlx5e_destroy_tis(priv, tc);
2327
2328         return err;
2329 }
2330
2331 void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv)
2332 {
2333         int tc;
2334
2335         for (tc = 0; tc < priv->profile->max_tc; tc++)
2336                 mlx5e_destroy_tis(priv, tc);
2337 }
2338
2339 static void mlx5e_build_indir_tir_ctx(struct mlx5e_priv *priv, u32 *tirc,
2340                                       enum mlx5e_traffic_types tt)
2341 {
2342         void *hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
2343
2344         MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn);
2345
2346 #define MLX5_HASH_IP            (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2347                                  MLX5_HASH_FIELD_SEL_DST_IP)
2348
2349 #define MLX5_HASH_IP_L4PORTS    (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2350                                  MLX5_HASH_FIELD_SEL_DST_IP   |\
2351                                  MLX5_HASH_FIELD_SEL_L4_SPORT |\
2352                                  MLX5_HASH_FIELD_SEL_L4_DPORT)
2353
2354 #define MLX5_HASH_IP_IPSEC_SPI  (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2355                                  MLX5_HASH_FIELD_SEL_DST_IP   |\
2356                                  MLX5_HASH_FIELD_SEL_IPSEC_SPI)
2357
2358         mlx5e_build_tir_ctx_lro(tirc, priv);
2359
2360         MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
2361         MLX5_SET(tirc, tirc, indirect_table, priv->indir_rqt.rqtn);
2362         mlx5e_build_tir_ctx_hash(tirc, priv);
2363
2364         switch (tt) {
2365         case MLX5E_TT_IPV4_TCP:
2366                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2367                          MLX5_L3_PROT_TYPE_IPV4);
2368                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2369                          MLX5_L4_PROT_TYPE_TCP);
2370                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2371                          MLX5_HASH_IP_L4PORTS);
2372                 break;
2373
2374         case MLX5E_TT_IPV6_TCP:
2375                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2376                          MLX5_L3_PROT_TYPE_IPV6);
2377                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2378                          MLX5_L4_PROT_TYPE_TCP);
2379                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2380                          MLX5_HASH_IP_L4PORTS);
2381                 break;
2382
2383         case MLX5E_TT_IPV4_UDP:
2384                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2385                          MLX5_L3_PROT_TYPE_IPV4);
2386                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2387                          MLX5_L4_PROT_TYPE_UDP);
2388                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2389                          MLX5_HASH_IP_L4PORTS);
2390                 break;
2391
2392         case MLX5E_TT_IPV6_UDP:
2393                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2394                          MLX5_L3_PROT_TYPE_IPV6);
2395                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2396                          MLX5_L4_PROT_TYPE_UDP);
2397                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2398                          MLX5_HASH_IP_L4PORTS);
2399                 break;
2400
2401         case MLX5E_TT_IPV4_IPSEC_AH:
2402                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2403                          MLX5_L3_PROT_TYPE_IPV4);
2404                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2405                          MLX5_HASH_IP_IPSEC_SPI);
2406                 break;
2407
2408         case MLX5E_TT_IPV6_IPSEC_AH:
2409                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2410                          MLX5_L3_PROT_TYPE_IPV6);
2411                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2412                          MLX5_HASH_IP_IPSEC_SPI);
2413                 break;
2414
2415         case MLX5E_TT_IPV4_IPSEC_ESP:
2416                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2417                          MLX5_L3_PROT_TYPE_IPV4);
2418                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2419                          MLX5_HASH_IP_IPSEC_SPI);
2420                 break;
2421
2422         case MLX5E_TT_IPV6_IPSEC_ESP:
2423                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2424                          MLX5_L3_PROT_TYPE_IPV6);
2425                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2426                          MLX5_HASH_IP_IPSEC_SPI);
2427                 break;
2428
2429         case MLX5E_TT_IPV4:
2430                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2431                          MLX5_L3_PROT_TYPE_IPV4);
2432                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2433                          MLX5_HASH_IP);
2434                 break;
2435
2436         case MLX5E_TT_IPV6:
2437                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2438                          MLX5_L3_PROT_TYPE_IPV6);
2439                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2440                          MLX5_HASH_IP);
2441                 break;
2442         default:
2443                 WARN_ONCE(true,
2444                           "mlx5e_build_indir_tir_ctx: bad traffic type!\n");
2445         }
2446 }
2447
2448 static void mlx5e_build_direct_tir_ctx(struct mlx5e_priv *priv, u32 *tirc,
2449                                        u32 rqtn)
2450 {
2451         MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn);
2452
2453         mlx5e_build_tir_ctx_lro(tirc, priv);
2454
2455         MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
2456         MLX5_SET(tirc, tirc, indirect_table, rqtn);
2457         MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8);
2458 }
2459
2460 static int mlx5e_create_indirect_tirs(struct mlx5e_priv *priv)
2461 {
2462         struct mlx5e_tir *tir;
2463         void *tirc;
2464         int inlen;
2465         int err;
2466         u32 *in;
2467         int tt;
2468
2469         inlen = MLX5_ST_SZ_BYTES(create_tir_in);
2470         in = mlx5_vzalloc(inlen);
2471         if (!in)
2472                 return -ENOMEM;
2473
2474         for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
2475                 memset(in, 0, inlen);
2476                 tir = &priv->indir_tir[tt];
2477                 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
2478                 mlx5e_build_indir_tir_ctx(priv, tirc, tt);
2479                 err = mlx5e_create_tir(priv->mdev, tir, in, inlen);
2480                 if (err)
2481                         goto err_destroy_tirs;
2482         }
2483
2484         kvfree(in);
2485
2486         return 0;
2487
2488 err_destroy_tirs:
2489         for (tt--; tt >= 0; tt--)
2490                 mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[tt]);
2491
2492         kvfree(in);
2493
2494         return err;
2495 }
2496
2497 int mlx5e_create_direct_tirs(struct mlx5e_priv *priv)
2498 {
2499         int nch = priv->profile->max_nch(priv->mdev);
2500         struct mlx5e_tir *tir;
2501         void *tirc;
2502         int inlen;
2503         int err;
2504         u32 *in;
2505         int ix;
2506
2507         inlen = MLX5_ST_SZ_BYTES(create_tir_in);
2508         in = mlx5_vzalloc(inlen);
2509         if (!in)
2510                 return -ENOMEM;
2511
2512         for (ix = 0; ix < nch; ix++) {
2513                 memset(in, 0, inlen);
2514                 tir = &priv->direct_tir[ix];
2515                 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
2516                 mlx5e_build_direct_tir_ctx(priv, tirc,
2517                                            priv->direct_tir[ix].rqt.rqtn);
2518                 err = mlx5e_create_tir(priv->mdev, tir, in, inlen);
2519                 if (err)
2520                         goto err_destroy_ch_tirs;
2521         }
2522
2523         kvfree(in);
2524
2525         return 0;
2526
2527 err_destroy_ch_tirs:
2528         for (ix--; ix >= 0; ix--)
2529                 mlx5e_destroy_tir(priv->mdev, &priv->direct_tir[ix]);
2530
2531         kvfree(in);
2532
2533         return err;
2534 }
2535
2536 static void mlx5e_destroy_indirect_tirs(struct mlx5e_priv *priv)
2537 {
2538         int i;
2539
2540         for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++)
2541                 mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[i]);
2542 }
2543
2544 void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv)
2545 {
2546         int nch = priv->profile->max_nch(priv->mdev);
2547         int i;
2548
2549         for (i = 0; i < nch; i++)
2550                 mlx5e_destroy_tir(priv->mdev, &priv->direct_tir[i]);
2551 }
2552
2553 int mlx5e_modify_rqs_vsd(struct mlx5e_priv *priv, bool vsd)
2554 {
2555         int err = 0;
2556         int i;
2557
2558         if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
2559                 return 0;
2560
2561         for (i = 0; i < priv->params.num_channels; i++) {
2562                 err = mlx5e_modify_rq_vsd(&priv->channel[i]->rq, vsd);
2563                 if (err)
2564                         return err;
2565         }
2566
2567         return 0;
2568 }
2569
2570 static int mlx5e_setup_tc(struct net_device *netdev, u8 tc)
2571 {
2572         struct mlx5e_priv *priv = netdev_priv(netdev);
2573         bool was_opened;
2574         int err = 0;
2575
2576         if (tc && tc != MLX5E_MAX_NUM_TC)
2577                 return -EINVAL;
2578
2579         mutex_lock(&priv->state_lock);
2580
2581         was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
2582         if (was_opened)
2583                 mlx5e_close_locked(priv->netdev);
2584
2585         priv->params.num_tc = tc ? tc : 1;
2586
2587         if (was_opened)
2588                 err = mlx5e_open_locked(priv->netdev);
2589
2590         mutex_unlock(&priv->state_lock);
2591
2592         return err;
2593 }
2594
2595 static int mlx5e_ndo_setup_tc(struct net_device *dev, u32 handle,
2596                               __be16 proto, struct tc_to_netdev *tc)
2597 {
2598         struct mlx5e_priv *priv = netdev_priv(dev);
2599
2600         if (TC_H_MAJ(handle) != TC_H_MAJ(TC_H_INGRESS))
2601                 goto mqprio;
2602
2603         switch (tc->type) {
2604         case TC_SETUP_CLSFLOWER:
2605                 switch (tc->cls_flower->command) {
2606                 case TC_CLSFLOWER_REPLACE:
2607                         return mlx5e_configure_flower(priv, proto, tc->cls_flower);
2608                 case TC_CLSFLOWER_DESTROY:
2609                         return mlx5e_delete_flower(priv, tc->cls_flower);
2610                 case TC_CLSFLOWER_STATS:
2611                         return mlx5e_stats_flower(priv, tc->cls_flower);
2612                 }
2613         default:
2614                 return -EOPNOTSUPP;
2615         }
2616
2617 mqprio:
2618         if (tc->type != TC_SETUP_MQPRIO)
2619                 return -EINVAL;
2620
2621         return mlx5e_setup_tc(dev, tc->tc);
2622 }
2623
2624 struct rtnl_link_stats64 *
2625 mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats)
2626 {
2627         struct mlx5e_priv *priv = netdev_priv(dev);
2628         struct mlx5e_sw_stats *sstats = &priv->stats.sw;
2629         struct mlx5e_vport_stats *vstats = &priv->stats.vport;
2630         struct mlx5e_pport_stats *pstats = &priv->stats.pport;
2631
2632         stats->rx_packets = sstats->rx_packets;
2633         stats->rx_bytes   = sstats->rx_bytes;
2634         stats->tx_packets = sstats->tx_packets;
2635         stats->tx_bytes   = sstats->tx_bytes;
2636
2637         stats->rx_dropped = priv->stats.qcnt.rx_out_of_buffer;
2638         stats->tx_dropped = sstats->tx_queue_dropped;
2639
2640         stats->rx_length_errors =
2641                 PPORT_802_3_GET(pstats, a_in_range_length_errors) +
2642                 PPORT_802_3_GET(pstats, a_out_of_range_length_field) +
2643                 PPORT_802_3_GET(pstats, a_frame_too_long_errors);
2644         stats->rx_crc_errors =
2645                 PPORT_802_3_GET(pstats, a_frame_check_sequence_errors);
2646         stats->rx_frame_errors = PPORT_802_3_GET(pstats, a_alignment_errors);
2647         stats->tx_aborted_errors = PPORT_2863_GET(pstats, if_out_discards);
2648         stats->tx_carrier_errors =
2649                 PPORT_802_3_GET(pstats, a_symbol_error_during_carrier);
2650         stats->rx_errors = stats->rx_length_errors + stats->rx_crc_errors +
2651                            stats->rx_frame_errors;
2652         stats->tx_errors = stats->tx_aborted_errors + stats->tx_carrier_errors;
2653
2654         /* vport multicast also counts packets that are dropped due to steering
2655          * or rx out of buffer
2656          */
2657         stats->multicast =
2658                 VPORT_COUNTER_GET(vstats, received_eth_multicast.packets);
2659
2660         return stats;
2661 }
2662
2663 static void mlx5e_set_rx_mode(struct net_device *dev)
2664 {
2665         struct mlx5e_priv *priv = netdev_priv(dev);
2666
2667         queue_work(priv->wq, &priv->set_rx_mode_work);
2668 }
2669
2670 static int mlx5e_set_mac(struct net_device *netdev, void *addr)
2671 {
2672         struct mlx5e_priv *priv = netdev_priv(netdev);
2673         struct sockaddr *saddr = addr;
2674
2675         if (!is_valid_ether_addr(saddr->sa_data))
2676                 return -EADDRNOTAVAIL;
2677
2678         netif_addr_lock_bh(netdev);
2679         ether_addr_copy(netdev->dev_addr, saddr->sa_data);
2680         netif_addr_unlock_bh(netdev);
2681
2682         queue_work(priv->wq, &priv->set_rx_mode_work);
2683
2684         return 0;
2685 }
2686
2687 #define MLX5E_SET_FEATURE(netdev, feature, enable)      \
2688         do {                                            \
2689                 if (enable)                             \
2690                         netdev->features |= feature;    \
2691                 else                                    \
2692                         netdev->features &= ~feature;   \
2693         } while (0)
2694
2695 typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable);
2696
2697 static int set_feature_lro(struct net_device *netdev, bool enable)
2698 {
2699         struct mlx5e_priv *priv = netdev_priv(netdev);
2700         bool was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
2701         int err;
2702
2703         mutex_lock(&priv->state_lock);
2704
2705         if (was_opened && (priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST))
2706                 mlx5e_close_locked(priv->netdev);
2707
2708         priv->params.lro_en = enable;
2709         err = mlx5e_modify_tirs_lro(priv);
2710         if (err) {
2711                 netdev_err(netdev, "lro modify failed, %d\n", err);
2712                 priv->params.lro_en = !enable;
2713         }
2714
2715         if (was_opened && (priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST))
2716                 mlx5e_open_locked(priv->netdev);
2717
2718         mutex_unlock(&priv->state_lock);
2719
2720         return err;
2721 }
2722
2723 static int set_feature_vlan_filter(struct net_device *netdev, bool enable)
2724 {
2725         struct mlx5e_priv *priv = netdev_priv(netdev);
2726
2727         if (enable)
2728                 mlx5e_enable_vlan_filter(priv);
2729         else
2730                 mlx5e_disable_vlan_filter(priv);
2731
2732         return 0;
2733 }
2734
2735 static int set_feature_tc_num_filters(struct net_device *netdev, bool enable)
2736 {
2737         struct mlx5e_priv *priv = netdev_priv(netdev);
2738
2739         if (!enable && mlx5e_tc_num_filters(priv)) {
2740                 netdev_err(netdev,
2741                            "Active offloaded tc filters, can't turn hw_tc_offload off\n");
2742                 return -EINVAL;
2743         }
2744
2745         return 0;
2746 }
2747
2748 static int set_feature_rx_all(struct net_device *netdev, bool enable)
2749 {
2750         struct mlx5e_priv *priv = netdev_priv(netdev);
2751         struct mlx5_core_dev *mdev = priv->mdev;
2752
2753         return mlx5_set_port_fcs(mdev, !enable);
2754 }
2755
2756 static int set_feature_rx_vlan(struct net_device *netdev, bool enable)
2757 {
2758         struct mlx5e_priv *priv = netdev_priv(netdev);
2759         int err;
2760
2761         mutex_lock(&priv->state_lock);
2762
2763         priv->params.vlan_strip_disable = !enable;
2764         err = mlx5e_modify_rqs_vsd(priv, !enable);
2765         if (err)
2766                 priv->params.vlan_strip_disable = enable;
2767
2768         mutex_unlock(&priv->state_lock);
2769
2770         return err;
2771 }
2772
2773 #ifdef CONFIG_RFS_ACCEL
2774 static int set_feature_arfs(struct net_device *netdev, bool enable)
2775 {
2776         struct mlx5e_priv *priv = netdev_priv(netdev);
2777         int err;
2778
2779         if (enable)
2780                 err = mlx5e_arfs_enable(priv);
2781         else
2782                 err = mlx5e_arfs_disable(priv);
2783
2784         return err;
2785 }
2786 #endif
2787
2788 static int mlx5e_handle_feature(struct net_device *netdev,
2789                                 netdev_features_t wanted_features,
2790                                 netdev_features_t feature,
2791                                 mlx5e_feature_handler feature_handler)
2792 {
2793         netdev_features_t changes = wanted_features ^ netdev->features;
2794         bool enable = !!(wanted_features & feature);
2795         int err;
2796
2797         if (!(changes & feature))
2798                 return 0;
2799
2800         err = feature_handler(netdev, enable);
2801         if (err) {
2802                 netdev_err(netdev, "%s feature 0x%llx failed err %d\n",
2803                            enable ? "Enable" : "Disable", feature, err);
2804                 return err;
2805         }
2806
2807         MLX5E_SET_FEATURE(netdev, feature, enable);
2808         return 0;
2809 }
2810
2811 static int mlx5e_set_features(struct net_device *netdev,
2812                               netdev_features_t features)
2813 {
2814         int err;
2815
2816         err  = mlx5e_handle_feature(netdev, features, NETIF_F_LRO,
2817                                     set_feature_lro);
2818         err |= mlx5e_handle_feature(netdev, features,
2819                                     NETIF_F_HW_VLAN_CTAG_FILTER,
2820                                     set_feature_vlan_filter);
2821         err |= mlx5e_handle_feature(netdev, features, NETIF_F_HW_TC,
2822                                     set_feature_tc_num_filters);
2823         err |= mlx5e_handle_feature(netdev, features, NETIF_F_RXALL,
2824                                     set_feature_rx_all);
2825         err |= mlx5e_handle_feature(netdev, features, NETIF_F_HW_VLAN_CTAG_RX,
2826                                     set_feature_rx_vlan);
2827 #ifdef CONFIG_RFS_ACCEL
2828         err |= mlx5e_handle_feature(netdev, features, NETIF_F_NTUPLE,
2829                                     set_feature_arfs);
2830 #endif
2831
2832         return err ? -EINVAL : 0;
2833 }
2834
2835 #define MXL5_HW_MIN_MTU 64
2836 #define MXL5E_MIN_MTU (MXL5_HW_MIN_MTU + ETH_FCS_LEN)
2837
2838 static int mlx5e_change_mtu(struct net_device *netdev, int new_mtu)
2839 {
2840         struct mlx5e_priv *priv = netdev_priv(netdev);
2841         struct mlx5_core_dev *mdev = priv->mdev;
2842         bool was_opened;
2843         u16 max_mtu;
2844         u16 min_mtu;
2845         int err = 0;
2846         bool reset;
2847
2848         mlx5_query_port_max_mtu(mdev, &max_mtu, 1);
2849
2850         max_mtu = MLX5E_HW2SW_MTU(max_mtu);
2851         min_mtu = MLX5E_HW2SW_MTU(MXL5E_MIN_MTU);
2852
2853         if (new_mtu > max_mtu || new_mtu < min_mtu) {
2854                 netdev_err(netdev,
2855                            "%s: Bad MTU (%d), valid range is: [%d..%d]\n",
2856                            __func__, new_mtu, min_mtu, max_mtu);
2857                 return -EINVAL;
2858         }
2859
2860         mutex_lock(&priv->state_lock);
2861
2862         reset = !priv->params.lro_en &&
2863                 (priv->params.rq_wq_type !=
2864                  MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ);
2865
2866         was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
2867         if (was_opened && reset)
2868                 mlx5e_close_locked(netdev);
2869
2870         netdev->mtu = new_mtu;
2871         mlx5e_set_dev_port_mtu(netdev);
2872
2873         if (was_opened && reset)
2874                 err = mlx5e_open_locked(netdev);
2875
2876         mutex_unlock(&priv->state_lock);
2877
2878         return err;
2879 }
2880
2881 static int mlx5e_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
2882 {
2883         switch (cmd) {
2884         case SIOCSHWTSTAMP:
2885                 return mlx5e_hwstamp_set(dev, ifr);
2886         case SIOCGHWTSTAMP:
2887                 return mlx5e_hwstamp_get(dev, ifr);
2888         default:
2889                 return -EOPNOTSUPP;
2890         }
2891 }
2892
2893 static int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
2894 {
2895         struct mlx5e_priv *priv = netdev_priv(dev);
2896         struct mlx5_core_dev *mdev = priv->mdev;
2897
2898         return mlx5_eswitch_set_vport_mac(mdev->priv.eswitch, vf + 1, mac);
2899 }
2900
2901 static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos)
2902 {
2903         struct mlx5e_priv *priv = netdev_priv(dev);
2904         struct mlx5_core_dev *mdev = priv->mdev;
2905
2906         return mlx5_eswitch_set_vport_vlan(mdev->priv.eswitch, vf + 1,
2907                                            vlan, qos);
2908 }
2909
2910 static int mlx5e_set_vf_spoofchk(struct net_device *dev, int vf, bool setting)
2911 {
2912         struct mlx5e_priv *priv = netdev_priv(dev);
2913         struct mlx5_core_dev *mdev = priv->mdev;
2914
2915         return mlx5_eswitch_set_vport_spoofchk(mdev->priv.eswitch, vf + 1, setting);
2916 }
2917
2918 static int mlx5e_set_vf_trust(struct net_device *dev, int vf, bool setting)
2919 {
2920         struct mlx5e_priv *priv = netdev_priv(dev);
2921         struct mlx5_core_dev *mdev = priv->mdev;
2922
2923         return mlx5_eswitch_set_vport_trust(mdev->priv.eswitch, vf + 1, setting);
2924 }
2925 static int mlx5_vport_link2ifla(u8 esw_link)
2926 {
2927         switch (esw_link) {
2928         case MLX5_ESW_VPORT_ADMIN_STATE_DOWN:
2929                 return IFLA_VF_LINK_STATE_DISABLE;
2930         case MLX5_ESW_VPORT_ADMIN_STATE_UP:
2931                 return IFLA_VF_LINK_STATE_ENABLE;
2932         }
2933         return IFLA_VF_LINK_STATE_AUTO;
2934 }
2935
2936 static int mlx5_ifla_link2vport(u8 ifla_link)
2937 {
2938         switch (ifla_link) {
2939         case IFLA_VF_LINK_STATE_DISABLE:
2940                 return MLX5_ESW_VPORT_ADMIN_STATE_DOWN;
2941         case IFLA_VF_LINK_STATE_ENABLE:
2942                 return MLX5_ESW_VPORT_ADMIN_STATE_UP;
2943         }
2944         return MLX5_ESW_VPORT_ADMIN_STATE_AUTO;
2945 }
2946
2947 static int mlx5e_set_vf_link_state(struct net_device *dev, int vf,
2948                                    int link_state)
2949 {
2950         struct mlx5e_priv *priv = netdev_priv(dev);
2951         struct mlx5_core_dev *mdev = priv->mdev;
2952
2953         return mlx5_eswitch_set_vport_state(mdev->priv.eswitch, vf + 1,
2954                                             mlx5_ifla_link2vport(link_state));
2955 }
2956
2957 static int mlx5e_get_vf_config(struct net_device *dev,
2958                                int vf, struct ifla_vf_info *ivi)
2959 {
2960         struct mlx5e_priv *priv = netdev_priv(dev);
2961         struct mlx5_core_dev *mdev = priv->mdev;
2962         int err;
2963
2964         err = mlx5_eswitch_get_vport_config(mdev->priv.eswitch, vf + 1, ivi);
2965         if (err)
2966                 return err;
2967         ivi->linkstate = mlx5_vport_link2ifla(ivi->linkstate);
2968         return 0;
2969 }
2970
2971 static int mlx5e_get_vf_stats(struct net_device *dev,
2972                               int vf, struct ifla_vf_stats *vf_stats)
2973 {
2974         struct mlx5e_priv *priv = netdev_priv(dev);
2975         struct mlx5_core_dev *mdev = priv->mdev;
2976
2977         return mlx5_eswitch_get_vport_stats(mdev->priv.eswitch, vf + 1,
2978                                             vf_stats);
2979 }
2980
2981 static void mlx5e_add_vxlan_port(struct net_device *netdev,
2982                                  struct udp_tunnel_info *ti)
2983 {
2984         struct mlx5e_priv *priv = netdev_priv(netdev);
2985
2986         if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
2987                 return;
2988
2989         if (!mlx5e_vxlan_allowed(priv->mdev))
2990                 return;
2991
2992         mlx5e_vxlan_queue_work(priv, ti->sa_family, be16_to_cpu(ti->port), 1);
2993 }
2994
2995 static void mlx5e_del_vxlan_port(struct net_device *netdev,
2996                                  struct udp_tunnel_info *ti)
2997 {
2998         struct mlx5e_priv *priv = netdev_priv(netdev);
2999
3000         if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
3001                 return;
3002
3003         if (!mlx5e_vxlan_allowed(priv->mdev))
3004                 return;
3005
3006         mlx5e_vxlan_queue_work(priv, ti->sa_family, be16_to_cpu(ti->port), 0);
3007 }
3008
3009 static netdev_features_t mlx5e_vxlan_features_check(struct mlx5e_priv *priv,
3010                                                     struct sk_buff *skb,
3011                                                     netdev_features_t features)
3012 {
3013         struct udphdr *udph;
3014         u16 proto;
3015         u16 port = 0;
3016
3017         switch (vlan_get_protocol(skb)) {
3018         case htons(ETH_P_IP):
3019                 proto = ip_hdr(skb)->protocol;
3020                 break;
3021         case htons(ETH_P_IPV6):
3022                 proto = ipv6_hdr(skb)->nexthdr;
3023                 break;
3024         default:
3025                 goto out;
3026         }
3027
3028         if (proto == IPPROTO_UDP) {
3029                 udph = udp_hdr(skb);
3030                 port = be16_to_cpu(udph->dest);
3031         }
3032
3033         /* Verify if UDP port is being offloaded by HW */
3034         if (port && mlx5e_vxlan_lookup_port(priv, port))
3035                 return features;
3036
3037 out:
3038         /* Disable CSUM and GSO if the udp dport is not offloaded by HW */
3039         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
3040 }
3041
3042 static netdev_features_t mlx5e_features_check(struct sk_buff *skb,
3043                                               struct net_device *netdev,
3044                                               netdev_features_t features)
3045 {
3046         struct mlx5e_priv *priv = netdev_priv(netdev);
3047
3048         features = vlan_features_check(skb, features);
3049         features = vxlan_features_check(skb, features);
3050
3051         /* Validate if the tunneled packet is being offloaded by HW */
3052         if (skb->encapsulation &&
3053             (features & NETIF_F_CSUM_MASK || features & NETIF_F_GSO_MASK))
3054                 return mlx5e_vxlan_features_check(priv, skb, features);
3055
3056         return features;
3057 }
3058
3059 static void mlx5e_tx_timeout(struct net_device *dev)
3060 {
3061         struct mlx5e_priv *priv = netdev_priv(dev);
3062         bool sched_work = false;
3063         int i;
3064
3065         netdev_err(dev, "TX timeout detected\n");
3066
3067         for (i = 0; i < priv->params.num_channels * priv->params.num_tc; i++) {
3068                 struct mlx5e_sq *sq = priv->txq_to_sq_map[i];
3069
3070                 if (!netif_xmit_stopped(netdev_get_tx_queue(dev, i)))
3071                         continue;
3072                 sched_work = true;
3073                 set_bit(MLX5E_SQ_STATE_FLUSH, &sq->state);
3074                 netdev_err(dev, "TX timeout on queue: %d, SQ: 0x%x, CQ: 0x%x, SQ Cons: 0x%x SQ Prod: 0x%x\n",
3075                            i, sq->sqn, sq->cq.mcq.cqn, sq->cc, sq->pc);
3076         }
3077
3078         if (sched_work && test_bit(MLX5E_STATE_OPENED, &priv->state))
3079                 schedule_work(&priv->tx_timeout_work);
3080 }
3081
3082 static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)
3083 {
3084         struct mlx5e_priv *priv = netdev_priv(netdev);
3085         struct bpf_prog *old_prog;
3086         int err = 0;
3087         bool reset, was_opened;
3088         int i;
3089
3090         mutex_lock(&priv->state_lock);
3091
3092         if ((netdev->features & NETIF_F_LRO) && prog) {
3093                 netdev_warn(netdev, "can't set XDP while LRO is on, disable LRO first\n");
3094                 err = -EINVAL;
3095                 goto unlock;
3096         }
3097
3098         was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
3099         /* no need for full reset when exchanging programs */
3100         reset = (!priv->xdp_prog || !prog);
3101
3102         if (was_opened && reset)
3103                 mlx5e_close_locked(netdev);
3104
3105         /* exchange programs */
3106         old_prog = xchg(&priv->xdp_prog, prog);
3107         if (prog)
3108                 bpf_prog_add(prog, 1);
3109         if (old_prog)
3110                 bpf_prog_put(old_prog);
3111
3112         if (reset) /* change RQ type according to priv->xdp_prog */
3113                 mlx5e_set_rq_priv_params(priv);
3114
3115         if (was_opened && reset)
3116                 mlx5e_open_locked(netdev);
3117
3118         if (!test_bit(MLX5E_STATE_OPENED, &priv->state) || reset)
3119                 goto unlock;
3120
3121         /* exchanging programs w/o reset, we update ref counts on behalf
3122          * of the channels RQs here.
3123          */
3124         bpf_prog_add(prog, priv->params.num_channels);
3125         for (i = 0; i < priv->params.num_channels; i++) {
3126                 struct mlx5e_channel *c = priv->channel[i];
3127
3128                 set_bit(MLX5E_RQ_STATE_FLUSH, &c->rq.state);
3129                 napi_synchronize(&c->napi);
3130                 /* prevent mlx5e_poll_rx_cq from accessing rq->xdp_prog */
3131
3132                 old_prog = xchg(&c->rq.xdp_prog, prog);
3133
3134                 clear_bit(MLX5E_RQ_STATE_FLUSH, &c->rq.state);
3135                 /* napi_schedule in case we have missed anything */
3136                 set_bit(MLX5E_CHANNEL_NAPI_SCHED, &c->flags);
3137                 napi_schedule(&c->napi);
3138
3139                 if (old_prog)
3140                         bpf_prog_put(old_prog);
3141         }
3142
3143 unlock:
3144         mutex_unlock(&priv->state_lock);
3145         return err;
3146 }
3147
3148 static bool mlx5e_xdp_attached(struct net_device *dev)
3149 {
3150         struct mlx5e_priv *priv = netdev_priv(dev);
3151
3152         return !!priv->xdp_prog;
3153 }
3154
3155 static int mlx5e_xdp(struct net_device *dev, struct netdev_xdp *xdp)
3156 {
3157         switch (xdp->command) {
3158         case XDP_SETUP_PROG:
3159                 return mlx5e_xdp_set(dev, xdp->prog);
3160         case XDP_QUERY_PROG:
3161                 xdp->prog_attached = mlx5e_xdp_attached(dev);
3162                 return 0;
3163         default:
3164                 return -EINVAL;
3165         }
3166 }
3167
3168 static const struct net_device_ops mlx5e_netdev_ops_basic = {
3169         .ndo_open                = mlx5e_open,
3170         .ndo_stop                = mlx5e_close,
3171         .ndo_start_xmit          = mlx5e_xmit,
3172         .ndo_setup_tc            = mlx5e_ndo_setup_tc,
3173         .ndo_select_queue        = mlx5e_select_queue,
3174         .ndo_get_stats64         = mlx5e_get_stats,
3175         .ndo_set_rx_mode         = mlx5e_set_rx_mode,
3176         .ndo_set_mac_address     = mlx5e_set_mac,
3177         .ndo_vlan_rx_add_vid     = mlx5e_vlan_rx_add_vid,
3178         .ndo_vlan_rx_kill_vid    = mlx5e_vlan_rx_kill_vid,
3179         .ndo_set_features        = mlx5e_set_features,
3180         .ndo_change_mtu          = mlx5e_change_mtu,
3181         .ndo_do_ioctl            = mlx5e_ioctl,
3182         .ndo_set_tx_maxrate      = mlx5e_set_tx_maxrate,
3183 #ifdef CONFIG_RFS_ACCEL
3184         .ndo_rx_flow_steer       = mlx5e_rx_flow_steer,
3185 #endif
3186         .ndo_tx_timeout          = mlx5e_tx_timeout,
3187         .ndo_xdp                 = mlx5e_xdp,
3188 };
3189
3190 static const struct net_device_ops mlx5e_netdev_ops_sriov = {
3191         .ndo_open                = mlx5e_open,
3192         .ndo_stop                = mlx5e_close,
3193         .ndo_start_xmit          = mlx5e_xmit,
3194         .ndo_setup_tc            = mlx5e_ndo_setup_tc,
3195         .ndo_select_queue        = mlx5e_select_queue,
3196         .ndo_get_stats64         = mlx5e_get_stats,
3197         .ndo_set_rx_mode         = mlx5e_set_rx_mode,
3198         .ndo_set_mac_address     = mlx5e_set_mac,
3199         .ndo_vlan_rx_add_vid     = mlx5e_vlan_rx_add_vid,
3200         .ndo_vlan_rx_kill_vid    = mlx5e_vlan_rx_kill_vid,
3201         .ndo_set_features        = mlx5e_set_features,
3202         .ndo_change_mtu          = mlx5e_change_mtu,
3203         .ndo_do_ioctl            = mlx5e_ioctl,
3204         .ndo_udp_tunnel_add      = mlx5e_add_vxlan_port,
3205         .ndo_udp_tunnel_del      = mlx5e_del_vxlan_port,
3206         .ndo_set_tx_maxrate      = mlx5e_set_tx_maxrate,
3207         .ndo_features_check      = mlx5e_features_check,
3208 #ifdef CONFIG_RFS_ACCEL
3209         .ndo_rx_flow_steer       = mlx5e_rx_flow_steer,
3210 #endif
3211         .ndo_set_vf_mac          = mlx5e_set_vf_mac,
3212         .ndo_set_vf_vlan         = mlx5e_set_vf_vlan,
3213         .ndo_set_vf_spoofchk     = mlx5e_set_vf_spoofchk,
3214         .ndo_set_vf_trust        = mlx5e_set_vf_trust,
3215         .ndo_get_vf_config       = mlx5e_get_vf_config,
3216         .ndo_set_vf_link_state   = mlx5e_set_vf_link_state,
3217         .ndo_get_vf_stats        = mlx5e_get_vf_stats,
3218         .ndo_tx_timeout          = mlx5e_tx_timeout,
3219         .ndo_xdp                 = mlx5e_xdp,
3220 };
3221
3222 static int mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev)
3223 {
3224         if (MLX5_CAP_GEN(mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
3225                 return -ENOTSUPP;
3226         if (!MLX5_CAP_GEN(mdev, eth_net_offloads) ||
3227             !MLX5_CAP_GEN(mdev, nic_flow_table) ||
3228             !MLX5_CAP_ETH(mdev, csum_cap) ||
3229             !MLX5_CAP_ETH(mdev, max_lso_cap) ||
3230             !MLX5_CAP_ETH(mdev, vlan_cap) ||
3231             !MLX5_CAP_ETH(mdev, rss_ind_tbl_cap) ||
3232             MLX5_CAP_FLOWTABLE(mdev,
3233                                flow_table_properties_nic_receive.max_ft_level)
3234                                < 3) {
3235                 mlx5_core_warn(mdev,
3236                                "Not creating net device, some required device capabilities are missing\n");
3237                 return -ENOTSUPP;
3238         }
3239         if (!MLX5_CAP_ETH(mdev, self_lb_en_modifiable))
3240                 mlx5_core_warn(mdev, "Self loop back prevention is not supported\n");
3241         if (!MLX5_CAP_GEN(mdev, cq_moderation))
3242                 mlx5_core_warn(mdev, "CQ modiration is not supported\n");
3243
3244         return 0;
3245 }
3246
3247 u16 mlx5e_get_max_inline_cap(struct mlx5_core_dev *mdev)
3248 {
3249         int bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2;
3250
3251         return bf_buf_size -
3252                sizeof(struct mlx5e_tx_wqe) +
3253                2 /*sizeof(mlx5e_tx_wqe.inline_hdr_start)*/;
3254 }
3255
3256 #ifdef CONFIG_MLX5_CORE_EN_DCB
3257 static void mlx5e_ets_init(struct mlx5e_priv *priv)
3258 {
3259         int i;
3260
3261         priv->params.ets.ets_cap = mlx5_max_tc(priv->mdev) + 1;
3262         for (i = 0; i < priv->params.ets.ets_cap; i++) {
3263                 priv->params.ets.tc_tx_bw[i] = MLX5E_MAX_BW_ALLOC;
3264                 priv->params.ets.tc_tsa[i] = IEEE_8021QAZ_TSA_VENDOR;
3265                 priv->params.ets.prio_tc[i] = i;
3266         }
3267
3268         /* tclass[prio=0]=1, tclass[prio=1]=0, tclass[prio=i]=i (for i>1) */
3269         priv->params.ets.prio_tc[0] = 1;
3270         priv->params.ets.prio_tc[1] = 0;
3271 }
3272 #endif
3273
3274 void mlx5e_build_default_indir_rqt(struct mlx5_core_dev *mdev,
3275                                    u32 *indirection_rqt, int len,
3276                                    int num_channels)
3277 {
3278         int node = mdev->priv.numa_node;
3279         int node_num_of_cores;
3280         int i;
3281
3282         if (node == -1)
3283                 node = first_online_node;
3284
3285         node_num_of_cores = cpumask_weight(cpumask_of_node(node));
3286
3287         if (node_num_of_cores)
3288                 num_channels = min_t(int, num_channels, node_num_of_cores);
3289
3290         for (i = 0; i < len; i++)
3291                 indirection_rqt[i] = i % num_channels;
3292 }
3293
3294 static int mlx5e_get_pci_bw(struct mlx5_core_dev *mdev, u32 *pci_bw)
3295 {
3296         enum pcie_link_width width;
3297         enum pci_bus_speed speed;
3298         int err = 0;
3299
3300         err = pcie_get_minimum_link(mdev->pdev, &speed, &width);
3301         if (err)
3302                 return err;
3303
3304         if (speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN)
3305                 return -EINVAL;
3306
3307         switch (speed) {
3308         case PCIE_SPEED_2_5GT:
3309                 *pci_bw = 2500 * width;
3310                 break;
3311         case PCIE_SPEED_5_0GT:
3312                 *pci_bw = 5000 * width;
3313                 break;
3314         case PCIE_SPEED_8_0GT:
3315                 *pci_bw = 8000 * width;
3316                 break;
3317         default:
3318                 return -EINVAL;
3319         }
3320
3321         return 0;
3322 }
3323
3324 static bool cqe_compress_heuristic(u32 link_speed, u32 pci_bw)
3325 {
3326         return (link_speed && pci_bw &&
3327                 (pci_bw < 40000) && (pci_bw < link_speed));
3328 }
3329
3330 void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
3331 {
3332         params->rx_cq_period_mode = cq_period_mode;
3333
3334         params->rx_cq_moderation.pkts =
3335                 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
3336         params->rx_cq_moderation.usec =
3337                         MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
3338
3339         if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
3340                 params->rx_cq_moderation.usec =
3341                         MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE;
3342 }
3343
3344 static void mlx5e_query_min_inline(struct mlx5_core_dev *mdev,
3345                                    u8 *min_inline_mode)
3346 {
3347         switch (MLX5_CAP_ETH(mdev, wqe_inline_mode)) {
3348         case MLX5E_INLINE_MODE_L2:
3349                 *min_inline_mode = MLX5_INLINE_MODE_L2;
3350                 break;
3351         case MLX5E_INLINE_MODE_VPORT_CONTEXT:
3352                 mlx5_query_nic_vport_min_inline(mdev,
3353                                                 min_inline_mode);
3354                 break;
3355         case MLX5_INLINE_MODE_NOT_REQUIRED:
3356                 *min_inline_mode = MLX5_INLINE_MODE_NONE;
3357                 break;
3358         }
3359 }
3360
3361 static void mlx5e_build_nic_netdev_priv(struct mlx5_core_dev *mdev,
3362                                         struct net_device *netdev,
3363                                         const struct mlx5e_profile *profile,
3364                                         void *ppriv)
3365 {
3366         struct mlx5e_priv *priv = netdev_priv(netdev);
3367         u32 link_speed = 0;
3368         u32 pci_bw = 0;
3369         u8 cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
3370                                          MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
3371                                          MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
3372
3373         priv->mdev                         = mdev;
3374         priv->netdev                       = netdev;
3375         priv->params.num_channels          = profile->max_nch(mdev);
3376         priv->profile                      = profile;
3377         priv->ppriv                        = ppriv;
3378
3379         priv->params.log_sq_size = MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
3380
3381         /* set CQE compression */
3382         priv->params.rx_cqe_compress_admin = false;
3383         if (MLX5_CAP_GEN(mdev, cqe_compression) &&
3384             MLX5_CAP_GEN(mdev, vport_group_manager)) {
3385                 mlx5e_get_max_linkspeed(mdev, &link_speed);
3386                 mlx5e_get_pci_bw(mdev, &pci_bw);
3387                 mlx5_core_dbg(mdev, "Max link speed = %d, PCI BW = %d\n",
3388                               link_speed, pci_bw);
3389                 priv->params.rx_cqe_compress_admin =
3390                         cqe_compress_heuristic(link_speed, pci_bw);
3391         }
3392         priv->params.rx_cqe_compress = priv->params.rx_cqe_compress_admin;
3393
3394         mlx5e_set_rq_priv_params(priv);
3395         if (priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
3396                 priv->params.lro_en = true;
3397
3398         priv->params.rx_am_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
3399         mlx5e_set_rx_cq_mode_params(&priv->params, cq_period_mode);
3400
3401         priv->params.tx_cq_moderation.usec =
3402                 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
3403         priv->params.tx_cq_moderation.pkts =
3404                 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
3405         priv->params.tx_max_inline         = mlx5e_get_max_inline_cap(mdev);
3406         mlx5e_query_min_inline(mdev, &priv->params.tx_min_inline_mode);
3407         priv->params.num_tc                = 1;
3408         priv->params.rss_hfunc             = ETH_RSS_HASH_XOR;
3409
3410         netdev_rss_key_fill(priv->params.toeplitz_hash_key,
3411                             sizeof(priv->params.toeplitz_hash_key));
3412
3413         mlx5e_build_default_indir_rqt(mdev, priv->params.indirection_rqt,
3414                                       MLX5E_INDIR_RQT_SIZE, profile->max_nch(mdev));
3415
3416         priv->params.lro_wqe_sz =
3417                 MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ -
3418                 /* Extra room needed for build_skb */
3419                 MLX5_RX_HEADROOM -
3420                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3421
3422         /* Initialize pflags */
3423         MLX5E_SET_PRIV_FLAG(priv, MLX5E_PFLAG_RX_CQE_BASED_MODER,
3424                             priv->params.rx_cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
3425
3426 #ifdef CONFIG_MLX5_CORE_EN_DCB
3427         mlx5e_ets_init(priv);
3428 #endif
3429
3430         mutex_init(&priv->state_lock);
3431
3432         INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
3433         INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
3434         INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work);
3435         INIT_DELAYED_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
3436 }
3437
3438 static void mlx5e_set_netdev_dev_addr(struct net_device *netdev)
3439 {
3440         struct mlx5e_priv *priv = netdev_priv(netdev);
3441
3442         mlx5_query_nic_vport_mac_address(priv->mdev, 0, netdev->dev_addr);
3443         if (is_zero_ether_addr(netdev->dev_addr) &&
3444             !MLX5_CAP_GEN(priv->mdev, vport_group_manager)) {
3445                 eth_hw_addr_random(netdev);
3446                 mlx5_core_info(priv->mdev, "Assigned random MAC address %pM\n", netdev->dev_addr);
3447         }
3448 }
3449
3450 static const struct switchdev_ops mlx5e_switchdev_ops = {
3451         .switchdev_port_attr_get        = mlx5e_attr_get,
3452 };
3453
3454 static void mlx5e_build_nic_netdev(struct net_device *netdev)
3455 {
3456         struct mlx5e_priv *priv = netdev_priv(netdev);
3457         struct mlx5_core_dev *mdev = priv->mdev;
3458         bool fcs_supported;
3459         bool fcs_enabled;
3460
3461         SET_NETDEV_DEV(netdev, &mdev->pdev->dev);
3462
3463         if (MLX5_CAP_GEN(mdev, vport_group_manager)) {
3464                 netdev->netdev_ops = &mlx5e_netdev_ops_sriov;
3465 #ifdef CONFIG_MLX5_CORE_EN_DCB
3466                 netdev->dcbnl_ops = &mlx5e_dcbnl_ops;
3467 #endif
3468         } else {
3469                 netdev->netdev_ops = &mlx5e_netdev_ops_basic;
3470         }
3471
3472         netdev->watchdog_timeo    = 15 * HZ;
3473
3474         netdev->ethtool_ops       = &mlx5e_ethtool_ops;
3475
3476         netdev->vlan_features    |= NETIF_F_SG;
3477         netdev->vlan_features    |= NETIF_F_IP_CSUM;
3478         netdev->vlan_features    |= NETIF_F_IPV6_CSUM;
3479         netdev->vlan_features    |= NETIF_F_GRO;
3480         netdev->vlan_features    |= NETIF_F_TSO;
3481         netdev->vlan_features    |= NETIF_F_TSO6;
3482         netdev->vlan_features    |= NETIF_F_RXCSUM;
3483         netdev->vlan_features    |= NETIF_F_RXHASH;
3484
3485         if (!!MLX5_CAP_ETH(mdev, lro_cap))
3486                 netdev->vlan_features    |= NETIF_F_LRO;
3487
3488         netdev->hw_features       = netdev->vlan_features;
3489         netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_TX;
3490         netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_RX;
3491         netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_FILTER;
3492
3493         if (mlx5e_vxlan_allowed(mdev)) {
3494                 netdev->hw_features     |= NETIF_F_GSO_UDP_TUNNEL |
3495                                            NETIF_F_GSO_UDP_TUNNEL_CSUM |
3496                                            NETIF_F_GSO_PARTIAL;
3497                 netdev->hw_enc_features |= NETIF_F_IP_CSUM;
3498                 netdev->hw_enc_features |= NETIF_F_IPV6_CSUM;
3499                 netdev->hw_enc_features |= NETIF_F_TSO;
3500                 netdev->hw_enc_features |= NETIF_F_TSO6;
3501                 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
3502                 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM |
3503                                            NETIF_F_GSO_PARTIAL;
3504                 netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM;
3505         }
3506
3507         mlx5_query_port_fcs(mdev, &fcs_supported, &fcs_enabled);
3508
3509         if (fcs_supported)
3510                 netdev->hw_features |= NETIF_F_RXALL;
3511
3512         netdev->features          = netdev->hw_features;
3513         if (!priv->params.lro_en)
3514                 netdev->features  &= ~NETIF_F_LRO;
3515
3516         if (fcs_enabled)
3517                 netdev->features  &= ~NETIF_F_RXALL;
3518
3519 #define FT_CAP(f) MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive.f)
3520         if (FT_CAP(flow_modify_en) &&
3521             FT_CAP(modify_root) &&
3522             FT_CAP(identified_miss_table_mode) &&
3523             FT_CAP(flow_table_modify)) {
3524                 netdev->hw_features      |= NETIF_F_HW_TC;
3525 #ifdef CONFIG_RFS_ACCEL
3526                 netdev->hw_features      |= NETIF_F_NTUPLE;
3527 #endif
3528         }
3529
3530         netdev->features         |= NETIF_F_HIGHDMA;
3531
3532         netdev->priv_flags       |= IFF_UNICAST_FLT;
3533
3534         mlx5e_set_netdev_dev_addr(netdev);
3535
3536 #ifdef CONFIG_NET_SWITCHDEV
3537         if (MLX5_CAP_GEN(mdev, vport_group_manager))
3538                 netdev->switchdev_ops = &mlx5e_switchdev_ops;
3539 #endif
3540 }
3541
3542 static void mlx5e_create_q_counter(struct mlx5e_priv *priv)
3543 {
3544         struct mlx5_core_dev *mdev = priv->mdev;
3545         int err;
3546
3547         err = mlx5_core_alloc_q_counter(mdev, &priv->q_counter);
3548         if (err) {
3549                 mlx5_core_warn(mdev, "alloc queue counter failed, %d\n", err);
3550                 priv->q_counter = 0;
3551         }
3552 }
3553
3554 static void mlx5e_destroy_q_counter(struct mlx5e_priv *priv)
3555 {
3556         if (!priv->q_counter)
3557                 return;
3558
3559         mlx5_core_dealloc_q_counter(priv->mdev, priv->q_counter);
3560 }
3561
3562 static int mlx5e_create_umr_mkey(struct mlx5e_priv *priv)
3563 {
3564         struct mlx5_core_dev *mdev = priv->mdev;
3565         u64 npages = MLX5E_REQUIRED_MTTS(priv->profile->max_nch(mdev),
3566                                          BIT(MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW));
3567         int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
3568         void *mkc;
3569         u32 *in;
3570         int err;
3571
3572         in = mlx5_vzalloc(inlen);
3573         if (!in)
3574                 return -ENOMEM;
3575
3576         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
3577
3578         npages = min_t(u32, ALIGN(U16_MAX, 4) * 2, npages);
3579
3580         MLX5_SET(mkc, mkc, free, 1);
3581         MLX5_SET(mkc, mkc, umr_en, 1);
3582         MLX5_SET(mkc, mkc, lw, 1);
3583         MLX5_SET(mkc, mkc, lr, 1);
3584         MLX5_SET(mkc, mkc, access_mode, MLX5_MKC_ACCESS_MODE_MTT);
3585
3586         MLX5_SET(mkc, mkc, qpn, 0xffffff);
3587         MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.pdn);
3588         MLX5_SET64(mkc, mkc, len, npages << PAGE_SHIFT);
3589         MLX5_SET(mkc, mkc, translations_octword_size,
3590                  MLX5_MTT_OCTW(npages));
3591         MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
3592
3593         err = mlx5_core_create_mkey(mdev, &priv->umr_mkey, in, inlen);
3594
3595         kvfree(in);
3596         return err;
3597 }
3598
3599 static void mlx5e_nic_init(struct mlx5_core_dev *mdev,
3600                            struct net_device *netdev,
3601                            const struct mlx5e_profile *profile,
3602                            void *ppriv)
3603 {
3604         struct mlx5e_priv *priv = netdev_priv(netdev);
3605
3606         mlx5e_build_nic_netdev_priv(mdev, netdev, profile, ppriv);
3607         mlx5e_build_nic_netdev(netdev);
3608         mlx5e_vxlan_init(priv);
3609 }
3610
3611 static void mlx5e_nic_cleanup(struct mlx5e_priv *priv)
3612 {
3613         struct mlx5_core_dev *mdev = priv->mdev;
3614         struct mlx5_eswitch *esw = mdev->priv.eswitch;
3615
3616         mlx5e_vxlan_cleanup(priv);
3617
3618         if (MLX5_CAP_GEN(mdev, vport_group_manager))
3619                 mlx5_eswitch_unregister_vport_rep(esw, 0);
3620 }
3621
3622 static int mlx5e_init_nic_rx(struct mlx5e_priv *priv)
3623 {
3624         struct mlx5_core_dev *mdev = priv->mdev;
3625         int err;
3626         int i;
3627
3628         err = mlx5e_create_indirect_rqts(priv);
3629         if (err) {
3630                 mlx5_core_warn(mdev, "create indirect rqts failed, %d\n", err);
3631                 return err;
3632         }
3633
3634         err = mlx5e_create_direct_rqts(priv);
3635         if (err) {
3636                 mlx5_core_warn(mdev, "create direct rqts failed, %d\n", err);
3637                 goto err_destroy_indirect_rqts;
3638         }
3639
3640         err = mlx5e_create_indirect_tirs(priv);
3641         if (err) {
3642                 mlx5_core_warn(mdev, "create indirect tirs failed, %d\n", err);
3643                 goto err_destroy_direct_rqts;
3644         }
3645
3646         err = mlx5e_create_direct_tirs(priv);
3647         if (err) {
3648                 mlx5_core_warn(mdev, "create direct tirs failed, %d\n", err);
3649                 goto err_destroy_indirect_tirs;
3650         }
3651
3652         err = mlx5e_create_flow_steering(priv);
3653         if (err) {
3654                 mlx5_core_warn(mdev, "create flow steering failed, %d\n", err);
3655                 goto err_destroy_direct_tirs;
3656         }
3657
3658         err = mlx5e_tc_init(priv);
3659         if (err)
3660                 goto err_destroy_flow_steering;
3661
3662         return 0;
3663
3664 err_destroy_flow_steering:
3665         mlx5e_destroy_flow_steering(priv);
3666 err_destroy_direct_tirs:
3667         mlx5e_destroy_direct_tirs(priv);
3668 err_destroy_indirect_tirs:
3669         mlx5e_destroy_indirect_tirs(priv);
3670 err_destroy_direct_rqts:
3671         for (i = 0; i < priv->profile->max_nch(mdev); i++)
3672                 mlx5e_destroy_rqt(priv, &priv->direct_tir[i].rqt);
3673 err_destroy_indirect_rqts:
3674         mlx5e_destroy_rqt(priv, &priv->indir_rqt);
3675         return err;
3676 }
3677
3678 static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv)
3679 {
3680         int i;
3681
3682         mlx5e_tc_cleanup(priv);
3683         mlx5e_destroy_flow_steering(priv);
3684         mlx5e_destroy_direct_tirs(priv);
3685         mlx5e_destroy_indirect_tirs(priv);
3686         for (i = 0; i < priv->profile->max_nch(priv->mdev); i++)
3687                 mlx5e_destroy_rqt(priv, &priv->direct_tir[i].rqt);
3688         mlx5e_destroy_rqt(priv, &priv->indir_rqt);
3689 }
3690
3691 static int mlx5e_init_nic_tx(struct mlx5e_priv *priv)
3692 {
3693         int err;
3694
3695         err = mlx5e_create_tises(priv);
3696         if (err) {
3697                 mlx5_core_warn(priv->mdev, "create tises failed, %d\n", err);
3698                 return err;
3699         }
3700
3701 #ifdef CONFIG_MLX5_CORE_EN_DCB
3702         mlx5e_dcbnl_ieee_setets_core(priv, &priv->params.ets);
3703 #endif
3704         return 0;
3705 }
3706
3707 static void mlx5e_nic_enable(struct mlx5e_priv *priv)
3708 {
3709         struct net_device *netdev = priv->netdev;
3710         struct mlx5_core_dev *mdev = priv->mdev;
3711         struct mlx5_eswitch *esw = mdev->priv.eswitch;
3712         struct mlx5_eswitch_rep rep;
3713
3714         mlx5_lag_add(mdev, netdev);
3715
3716         if (mlx5e_vxlan_allowed(mdev)) {
3717                 rtnl_lock();
3718                 udp_tunnel_get_rx_info(netdev);
3719                 rtnl_unlock();
3720         }
3721
3722         mlx5e_enable_async_events(priv);
3723         queue_work(priv->wq, &priv->set_rx_mode_work);
3724
3725         if (MLX5_CAP_GEN(mdev, vport_group_manager)) {
3726                 mlx5_query_nic_vport_mac_address(mdev, 0, rep.hw_id);
3727                 rep.load = mlx5e_nic_rep_load;
3728                 rep.unload = mlx5e_nic_rep_unload;
3729                 rep.vport = 0;
3730                 rep.priv_data = priv;
3731                 mlx5_eswitch_register_vport_rep(esw, &rep);
3732         }
3733 }
3734
3735 static void mlx5e_nic_disable(struct mlx5e_priv *priv)
3736 {
3737         queue_work(priv->wq, &priv->set_rx_mode_work);
3738         mlx5e_disable_async_events(priv);
3739         mlx5_lag_remove(priv->mdev);
3740 }
3741
3742 static const struct mlx5e_profile mlx5e_nic_profile = {
3743         .init              = mlx5e_nic_init,
3744         .cleanup           = mlx5e_nic_cleanup,
3745         .init_rx           = mlx5e_init_nic_rx,
3746         .cleanup_rx        = mlx5e_cleanup_nic_rx,
3747         .init_tx           = mlx5e_init_nic_tx,
3748         .cleanup_tx        = mlx5e_cleanup_nic_tx,
3749         .enable            = mlx5e_nic_enable,
3750         .disable           = mlx5e_nic_disable,
3751         .update_stats      = mlx5e_update_stats,
3752         .max_nch           = mlx5e_get_max_num_channels,
3753         .max_tc            = MLX5E_MAX_NUM_TC,
3754 };
3755
3756 struct net_device *mlx5e_create_netdev(struct mlx5_core_dev *mdev,
3757                                        const struct mlx5e_profile *profile,
3758                                        void *ppriv)
3759 {
3760         int nch = profile->max_nch(mdev);
3761         struct net_device *netdev;
3762         struct mlx5e_priv *priv;
3763
3764         netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv),
3765                                     nch * profile->max_tc,
3766                                     nch);
3767         if (!netdev) {
3768                 mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n");
3769                 return NULL;
3770         }
3771
3772         profile->init(mdev, netdev, profile, ppriv);
3773
3774         netif_carrier_off(netdev);
3775
3776         priv = netdev_priv(netdev);
3777
3778         priv->wq = create_singlethread_workqueue("mlx5e");
3779         if (!priv->wq)
3780                 goto err_cleanup_nic;
3781
3782         return netdev;
3783
3784 err_cleanup_nic:
3785         profile->cleanup(priv);
3786         free_netdev(netdev);
3787
3788         return NULL;
3789 }
3790
3791 int mlx5e_attach_netdev(struct mlx5_core_dev *mdev, struct net_device *netdev)
3792 {
3793         const struct mlx5e_profile *profile;
3794         struct mlx5e_priv *priv;
3795         int err;
3796
3797         priv = netdev_priv(netdev);
3798         profile = priv->profile;
3799         clear_bit(MLX5E_STATE_DESTROYING, &priv->state);
3800
3801         err = mlx5e_create_umr_mkey(priv);
3802         if (err) {
3803                 mlx5_core_err(mdev, "create umr mkey failed, %d\n", err);
3804                 goto out;
3805         }
3806
3807         err = profile->init_tx(priv);
3808         if (err)
3809                 goto err_destroy_umr_mkey;
3810
3811         err = mlx5e_open_drop_rq(priv);
3812         if (err) {
3813                 mlx5_core_err(mdev, "open drop rq failed, %d\n", err);
3814                 goto err_cleanup_tx;
3815         }
3816
3817         err = profile->init_rx(priv);
3818         if (err)
3819                 goto err_close_drop_rq;
3820
3821         mlx5e_create_q_counter(priv);
3822
3823         mlx5e_init_l2_addr(priv);
3824
3825         mlx5e_set_dev_port_mtu(netdev);
3826
3827         if (profile->enable)
3828                 profile->enable(priv);
3829
3830         rtnl_lock();
3831         if (netif_running(netdev))
3832                 mlx5e_open(netdev);
3833         netif_device_attach(netdev);
3834         rtnl_unlock();
3835
3836         return 0;
3837
3838 err_close_drop_rq:
3839         mlx5e_close_drop_rq(priv);
3840
3841 err_cleanup_tx:
3842         profile->cleanup_tx(priv);
3843
3844 err_destroy_umr_mkey:
3845         mlx5_core_destroy_mkey(mdev, &priv->umr_mkey);
3846
3847 out:
3848         return err;
3849 }
3850
3851 static void mlx5e_register_vport_rep(struct mlx5_core_dev *mdev)
3852 {
3853         struct mlx5_eswitch *esw = mdev->priv.eswitch;
3854         int total_vfs = MLX5_TOTAL_VPORTS(mdev);
3855         int vport;
3856         u8 mac[ETH_ALEN];
3857
3858         if (!MLX5_CAP_GEN(mdev, vport_group_manager))
3859                 return;
3860
3861         mlx5_query_nic_vport_mac_address(mdev, 0, mac);
3862
3863         for (vport = 1; vport < total_vfs; vport++) {
3864                 struct mlx5_eswitch_rep rep;
3865
3866                 rep.load = mlx5e_vport_rep_load;
3867                 rep.unload = mlx5e_vport_rep_unload;
3868                 rep.vport = vport;
3869                 ether_addr_copy(rep.hw_id, mac);
3870                 mlx5_eswitch_register_vport_rep(esw, &rep);
3871         }
3872 }
3873
3874 void mlx5e_detach_netdev(struct mlx5_core_dev *mdev, struct net_device *netdev)
3875 {
3876         struct mlx5e_priv *priv = netdev_priv(netdev);
3877         const struct mlx5e_profile *profile = priv->profile;
3878
3879         set_bit(MLX5E_STATE_DESTROYING, &priv->state);
3880         if (profile->disable)
3881                 profile->disable(priv);
3882
3883         flush_workqueue(priv->wq);
3884
3885         rtnl_lock();
3886         if (netif_running(netdev))
3887                 mlx5e_close(netdev);
3888         netif_device_detach(netdev);
3889         rtnl_unlock();
3890
3891         mlx5e_destroy_q_counter(priv);
3892         profile->cleanup_rx(priv);
3893         mlx5e_close_drop_rq(priv);
3894         profile->cleanup_tx(priv);
3895         mlx5_core_destroy_mkey(priv->mdev, &priv->umr_mkey);
3896         cancel_delayed_work_sync(&priv->update_stats_work);
3897 }
3898
3899 /* mlx5e_attach and mlx5e_detach scope should be only creating/destroying
3900  * hardware contexts and to connect it to the current netdev.
3901  */
3902 static int mlx5e_attach(struct mlx5_core_dev *mdev, void *vpriv)
3903 {
3904         struct mlx5e_priv *priv = vpriv;
3905         struct net_device *netdev = priv->netdev;
3906         int err;
3907
3908         if (netif_device_present(netdev))
3909                 return 0;
3910
3911         err = mlx5e_create_mdev_resources(mdev);
3912         if (err)
3913                 return err;
3914
3915         err = mlx5e_attach_netdev(mdev, netdev);
3916         if (err) {
3917                 mlx5e_destroy_mdev_resources(mdev);
3918                 return err;
3919         }
3920
3921         return 0;
3922 }
3923
3924 static void mlx5e_detach(struct mlx5_core_dev *mdev, void *vpriv)
3925 {
3926         struct mlx5e_priv *priv = vpriv;
3927         struct net_device *netdev = priv->netdev;
3928
3929         if (!netif_device_present(netdev))
3930                 return;
3931
3932         mlx5e_detach_netdev(mdev, netdev);
3933         mlx5e_destroy_mdev_resources(mdev);
3934 }
3935
3936 static void *mlx5e_add(struct mlx5_core_dev *mdev)
3937 {
3938         struct mlx5_eswitch *esw = mdev->priv.eswitch;
3939         int total_vfs = MLX5_TOTAL_VPORTS(mdev);
3940         void *ppriv = NULL;
3941         void *priv;
3942         int vport;
3943         int err;
3944         struct net_device *netdev;
3945
3946         err = mlx5e_check_required_hca_cap(mdev);
3947         if (err)
3948                 return NULL;
3949
3950         mlx5e_register_vport_rep(mdev);
3951
3952         if (MLX5_CAP_GEN(mdev, vport_group_manager))
3953                 ppriv = &esw->offloads.vport_reps[0];
3954
3955         netdev = mlx5e_create_netdev(mdev, &mlx5e_nic_profile, ppriv);
3956         if (!netdev) {
3957                 mlx5_core_err(mdev, "mlx5e_create_netdev failed\n");
3958                 goto err_unregister_reps;
3959         }
3960
3961         priv = netdev_priv(netdev);
3962
3963         err = mlx5e_attach(mdev, priv);
3964         if (err) {
3965                 mlx5_core_err(mdev, "mlx5e_attach failed, %d\n", err);
3966                 goto err_destroy_netdev;
3967         }
3968
3969         err = register_netdev(netdev);
3970         if (err) {
3971                 mlx5_core_err(mdev, "register_netdev failed, %d\n", err);
3972                 goto err_detach;
3973         }
3974
3975         return priv;
3976
3977 err_detach:
3978         mlx5e_detach(mdev, priv);
3979
3980 err_destroy_netdev:
3981         mlx5e_destroy_netdev(mdev, priv);
3982
3983 err_unregister_reps:
3984         for (vport = 1; vport < total_vfs; vport++)
3985                 mlx5_eswitch_unregister_vport_rep(esw, vport);
3986
3987         return NULL;
3988 }
3989
3990 void mlx5e_destroy_netdev(struct mlx5_core_dev *mdev, struct mlx5e_priv *priv)
3991 {
3992         const struct mlx5e_profile *profile = priv->profile;
3993         struct net_device *netdev = priv->netdev;
3994
3995         unregister_netdev(netdev);
3996         destroy_workqueue(priv->wq);
3997         if (profile->cleanup)
3998                 profile->cleanup(priv);
3999         free_netdev(netdev);
4000 }
4001
4002 static void mlx5e_remove(struct mlx5_core_dev *mdev, void *vpriv)
4003 {
4004         struct mlx5_eswitch *esw = mdev->priv.eswitch;
4005         int total_vfs = MLX5_TOTAL_VPORTS(mdev);
4006         struct mlx5e_priv *priv = vpriv;
4007         int vport;
4008
4009         for (vport = 1; vport < total_vfs; vport++)
4010                 mlx5_eswitch_unregister_vport_rep(esw, vport);
4011
4012         mlx5e_detach(mdev, vpriv);
4013         mlx5e_destroy_netdev(mdev, priv);
4014 }
4015
4016 static void *mlx5e_get_netdev(void *vpriv)
4017 {
4018         struct mlx5e_priv *priv = vpriv;
4019
4020         return priv->netdev;
4021 }
4022
4023 static struct mlx5_interface mlx5e_interface = {
4024         .add       = mlx5e_add,
4025         .remove    = mlx5e_remove,
4026         .attach    = mlx5e_attach,
4027         .detach    = mlx5e_detach,
4028         .event     = mlx5e_async_event,
4029         .protocol  = MLX5_INTERFACE_PROTOCOL_ETH,
4030         .get_dev   = mlx5e_get_netdev,
4031 };
4032
4033 void mlx5e_init(void)
4034 {
4035         mlx5e_build_ptys2ethtool_map();
4036         mlx5_register_interface(&mlx5e_interface);
4037 }
4038
4039 void mlx5e_cleanup(void)
4040 {
4041         mlx5_unregister_interface(&mlx5e_interface);
4042 }