drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c

   1 /*
   2  * cxgb4_uld.c:Chelsio Upper Layer Driver Interface for T4/T5/T6 SGE management
   3  *
   4  * Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved.
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenIB.org BSD license below:
  11  *
  12  *     Redistribution and use in source and binary forms, with or
  13  *     without modification, are permitted provided that the following
  14  *     conditions are met:
  15  *
  16  *      - Redistributions of source code must retain the above
  17  *        copyright notice, this list of conditions and the following
  18  *        disclaimer.
  19  *
  20  *      - Redistributions in binary form must reproduce the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer in the documentation and/or other materials
  23  *        provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  *
  34  *  Written by: Atul Gupta (atul.gupta@chelsio.com)
  35  *  Written by: Hariprasad Shenai (hariprasad@chelsio.com)
  36  */
  37
  38 #include <linux/kernel.h>
  39 #include <linux/version.h>
  40 #include <linux/module.h>
  41 #include <linux/errno.h>
  42 #include <linux/types.h>
  43 #include <linux/debugfs.h>
  44 #include <linux/export.h>
  45 #include <linux/list.h>
  46 #include <linux/skbuff.h>
  47 #include <linux/pci.h>
  48
  49 #include "cxgb4.h"
  50 #include "cxgb4_uld.h"
  51 #include "t4_regs.h"
  52 #include "t4fw_api.h"
  53 #include "t4_msg.h"
  54
  55 #define for_each_uldrxq(m, i) for (i = 0; i < ((m)->nrxq + (m)->nciq); i++)
  56
  57 static int get_msix_idx_from_bmap(struct adapter *adap)
  58 {
  59         struct uld_msix_bmap *bmap = &adap->msix_bmap_ulds;
  60         unsigned long flags;
  61         unsigned int msix_idx;
  62
  63         spin_lock_irqsave(&bmap->lock, flags);
  64         msix_idx = find_first_zero_bit(bmap->msix_bmap, bmap->mapsize);
  65         if (msix_idx < bmap->mapsize) {
  66                 __set_bit(msix_idx, bmap->msix_bmap);
  67         } else {
  68                 spin_unlock_irqrestore(&bmap->lock, flags);
  69                 return -ENOSPC;
  70         }
  71
  72         spin_unlock_irqrestore(&bmap->lock, flags);
  73         return msix_idx;
  74 }
  75
  76 static void free_msix_idx_in_bmap(struct adapter *adap, unsigned int msix_idx)
  77 {
  78         struct uld_msix_bmap *bmap = &adap->msix_bmap_ulds;
  79         unsigned long flags;
  80
  81         spin_lock_irqsave(&bmap->lock, flags);
  82          __clear_bit(msix_idx, bmap->msix_bmap);
  83         spin_unlock_irqrestore(&bmap->lock, flags);
  84 }
  85
  86 static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
  87                          const struct pkt_gl *gl)
  88 {
  89         struct adapter *adap = q->adap;
  90         struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
  91         int ret;
  92
  93         /* FW can send CPLs encapsulated in a CPL_FW4_MSG */
  94         if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG &&
  95             ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL)
  96                 rsp += 2;
  97
  98         if (q->flush_handler)
  99                 ret = adap->uld[q->uld].lro_rx_handler(adap->uld[q->uld].handle,
 100                                 rsp, gl, &q->lro_mgr,
 101                                 &q->napi);
 102         else
 103                 ret = adap->uld[q->uld].rx_handler(adap->uld[q->uld].handle,
 104                                 rsp, gl);
 105
 106         if (ret) {
 107                 rxq->stats.nomem++;
 108                 return -1;
 109         }
 110
 111         if (!gl)
 112                 rxq->stats.imm++;
 113         else if (gl == CXGB4_MSG_AN)
 114                 rxq->stats.an++;
 115         else
 116                 rxq->stats.pkts++;
 117         return 0;
 118 }
 119
 120 static int alloc_uld_rxqs(struct adapter *adap,
 121                           struct sge_uld_rxq_info *rxq_info,
 122                           unsigned int nq, unsigned int offset, bool lro)
 123 {
 124         struct sge *s = &adap->sge;
 125         struct sge_ofld_rxq *q = rxq_info->uldrxq + offset;
 126         unsigned short *ids = rxq_info->rspq_id + offset;
 127         unsigned int per_chan = nq / adap->params.nports;
 128         unsigned int msi_idx, bmap_idx;
 129         int i, err;
 130
 131         if (adap->flags & USING_MSIX)
 132                 msi_idx = 1;
 133         else
 134                 msi_idx = -((int)s->intrq.abs_id + 1);
 135
 136         for (i = 0; i < nq; i++, q++) {
 137                 if (msi_idx >= 0) {
 138                         bmap_idx = get_msix_idx_from_bmap(adap);
 139                         adap->msi_idx++;
 140                 }
 141                 err = t4_sge_alloc_rxq(adap, &q->rspq, false,
 142                                        adap->port[i / per_chan],
 143                                        adap->msi_idx,
 144                                        q->fl.size ? &q->fl : NULL,
 145                                        uldrx_handler,
 146                                        NULL,
 147                                        0);
 148                 if (err)
 149                         goto freeout;
 150                 if (msi_idx >= 0)
 151                         rxq_info->msix_tbl[i + offset] = bmap_idx;
 152                 memset(&q->stats, 0, sizeof(q->stats));
 153                 if (ids)
 154                         ids[i] = q->rspq.abs_id;
 155         }
 156         return 0;
 157 freeout:
 158         q = rxq_info->uldrxq + offset;
 159         for ( ; i; i--, q++) {
 160                 if (q->rspq.desc)
 161                         free_rspq_fl(adap, &q->rspq,
 162                                      q->fl.size ? &q->fl : NULL);
 163                 adap->msi_idx--;
 164         }
 165
 166         /* We need to free rxq also in case of ciq allocation failure */
 167         if (offset) {
 168                 q = rxq_info->uldrxq + offset;
 169                 for ( ; i; i--, q++) {
 170                         if (q->rspq.desc)
 171                                 free_rspq_fl(adap, &q->rspq,
 172                                              q->fl.size ? &q->fl : NULL);
 173                         adap->msi_idx--;
 174                 }
 175         }
 176         return err;
 177 }
 178
 179 int setup_sge_queues_uld(struct adapter *adap, unsigned int uld_type, bool lro)
 180 {
 181         struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 182
 183         if (adap->flags & USING_MSIX) {
 184                 rxq_info->msix_tbl = kzalloc(rxq_info->nrxq + rxq_info->nciq,
 185                                              GFP_KERNEL);
 186                 if (!rxq_info->msix_tbl)
 187                         return -ENOMEM;
 188         }
 189
 190         return !(!alloc_uld_rxqs(adap, rxq_info, rxq_info->nrxq, 0, lro) &&
 191                  !alloc_uld_rxqs(adap, rxq_info, rxq_info->nciq,
 192                                  rxq_info->nrxq, lro));
 193 }
 194
 195 static void t4_free_uld_rxqs(struct adapter *adap, int n,
 196                              struct sge_ofld_rxq *q)
 197 {
 198         for ( ; n; n--, q++) {
 199                 if (q->rspq.desc)
 200                         free_rspq_fl(adap, &q->rspq,
 201                                      q->fl.size ? &q->fl : NULL);
 202                 adap->msi_idx--;
 203         }
 204 }
 205
 206 void free_sge_queues_uld(struct adapter *adap, unsigned int uld_type)
 207 {
 208         struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 209
 210         if (rxq_info->nciq)
 211                 t4_free_uld_rxqs(adap, rxq_info->nciq,
 212                                  rxq_info->uldrxq + rxq_info->nrxq);
 213         t4_free_uld_rxqs(adap, rxq_info->nrxq, rxq_info->uldrxq);
 214         if (adap->flags & USING_MSIX)
 215                 kfree(rxq_info->msix_tbl);
 216 }
 217
 218 int cfg_queues_uld(struct adapter *adap, unsigned int uld_type,
 219                    const struct cxgb4_pci_uld_info *uld_info)
 220 {
 221         struct sge *s = &adap->sge;
 222         struct sge_uld_rxq_info *rxq_info;
 223         int i, nrxq;
 224
 225         rxq_info = kzalloc(sizeof(*rxq_info), GFP_KERNEL);
 226         if (!rxq_info)
 227                 return -ENOMEM;
 228
 229         if (uld_info->nrxq > s->nqs_per_uld)
 230                 rxq_info->nrxq = s->nqs_per_uld;
 231         else
 232                 rxq_info->nrxq = uld_info->nrxq;
 233         if (!uld_info->nciq)
 234                 rxq_info->nciq = 0;
 235         else if (uld_info->nciq && uld_info->nciq > s->nqs_per_uld)
 236                 rxq_info->nciq = s->nqs_per_uld;
 237         else
 238                 rxq_info->nciq = uld_info->nciq;
 239
 240         nrxq = rxq_info->nrxq + rxq_info->nciq; /* total rxq's */
 241         rxq_info->uldrxq = kcalloc(nrxq, sizeof(struct sge_ofld_rxq),
 242                                    GFP_KERNEL);
 243         if (!rxq_info->uldrxq) {
 244                 kfree(rxq_info);
 245                 return -ENOMEM;
 246         }
 247
 248         rxq_info->rspq_id = kcalloc(nrxq, sizeof(unsigned short), GFP_KERNEL);
 249         if (!rxq_info->uldrxq) {
 250                 kfree(rxq_info->uldrxq);
 251                 kfree(rxq_info);
 252                 return -ENOMEM;
 253         }
 254
 255         for (i = 0; i < rxq_info->nrxq; i++) {
 256                 struct sge_ofld_rxq *r = &rxq_info->uldrxq[i];
 257
 258                 init_rspq(adap, &r->rspq, 5, 1, uld_info->rxq_size, 64);
 259                 r->rspq.uld = uld_type;
 260                 r->fl.size = 72;
 261         }
 262
 263         for (i = rxq_info->nrxq; i < nrxq; i++) {
 264                 struct sge_ofld_rxq *r = &rxq_info->uldrxq[i];
 265
 266                 init_rspq(adap, &r->rspq, 5, 1, uld_info->ciq_size, 64);
 267                 r->rspq.uld = uld_type;
 268                 r->fl.size = 72;
 269         }
 270
 271         memcpy(rxq_info->name, uld_info->name, IFNAMSIZ);
 272         adap->sge.uld_rxq_info[uld_type] = rxq_info;
 273
 274         return 0;
 275 }
 276
 277 void free_queues_uld(struct adapter *adap, unsigned int uld_type)
 278 {
 279         struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 280
 281         kfree(rxq_info->rspq_id);
 282         kfree(rxq_info->uldrxq);
 283         kfree(rxq_info);
 284 }
 285
 286 int request_msix_queue_irqs_uld(struct adapter *adap, unsigned int uld_type)
 287 {
 288         struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 289         int idx, bmap_idx, err = 0;
 290
 291         for_each_uldrxq(rxq_info, idx) {
 292                 bmap_idx = rxq_info->msix_tbl[idx];
 293                 err = request_irq(adap->msix_info_ulds[bmap_idx].vec,
 294                                   t4_sge_intr_msix, 0,
 295                                   adap->msix_info_ulds[bmap_idx].desc,
 296                                   &rxq_info->uldrxq[idx].rspq);
 297                 if (err)
 298                         goto unwind;
 299         }
 300         return 0;
 301 unwind:
 302         while (--idx >= 0) {
 303                 bmap_idx = rxq_info->msix_tbl[idx];
 304                 free_msix_idx_in_bmap(adap, bmap_idx);
 305                 free_irq(adap->msix_info_ulds[bmap_idx].vec,
 306                          &rxq_info->uldrxq[idx].rspq);
 307         }
 308         return err;
 309 }
 310
 311 void free_msix_queue_irqs_uld(struct adapter *adap, unsigned int uld_type)
 312 {
 313         struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 314         int idx;
 315
 316         for_each_uldrxq(rxq_info, idx) {
 317                 unsigned int bmap_idx = rxq_info->msix_tbl[idx];
 318
 319                 free_msix_idx_in_bmap(adap, bmap_idx);
 320                 free_irq(adap->msix_info_ulds[bmap_idx].vec,
 321                          &rxq_info->uldrxq[idx].rspq);
 322         }
 323 }
 324
 325 void name_msix_vecs_uld(struct adapter *adap, unsigned int uld_type)
 326 {
 327         struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 328         int n = sizeof(adap->msix_info_ulds[0].desc);
 329         int idx;
 330
 331         for_each_uldrxq(rxq_info, idx) {
 332                 unsigned int bmap_idx = rxq_info->msix_tbl[idx];
 333
 334                 snprintf(adap->msix_info_ulds[bmap_idx].desc, n, "%s-%s%d",
 335                          adap->port[0]->name, rxq_info->name, idx);
 336         }
 337 }
 338
 339 static void enable_rx(struct adapter *adap, struct sge_rspq *q)
 340 {
 341         if (!q)
 342                 return;
 343
 344         if (q->handler) {
 345                 cxgb_busy_poll_init_lock(q);
 346                 napi_enable(&q->napi);
 347         }
 348         /* 0-increment GTS to start the timer and enable interrupts */
 349         t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A),
 350                      SEINTARM_V(q->intr_params) |
 351                      INGRESSQID_V(q->cntxt_id));
 352 }
 353
 354 static void quiesce_rx(struct adapter *adap, struct sge_rspq *q)
 355 {
 356         if (q && q->handler) {
 357                 napi_disable(&q->napi);
 358                 local_bh_disable();
 359                 while (!cxgb_poll_lock_napi(q))
 360                         mdelay(1);
 361                 local_bh_enable();
 362         }
 363 }
 364
 365 void enable_rx_uld(struct adapter *adap, unsigned int uld_type)
 366 {
 367         struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 368         int idx;
 369
 370         for_each_uldrxq(rxq_info, idx)
 371                 enable_rx(adap, &rxq_info->uldrxq[idx].rspq);
 372 }
 373
 374 void quiesce_rx_uld(struct adapter *adap, unsigned int uld_type)
 375 {
 376         struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 377         int idx;
 378
 379         for_each_uldrxq(rxq_info, idx)
 380                 quiesce_rx(adap, &rxq_info->uldrxq[idx].rspq);
 381 }
 382
 383 static void uld_queue_init(struct adapter *adap, unsigned int uld_type,
 384                            struct cxgb4_lld_info *lli)
 385 {
 386         struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 387
 388         lli->rxq_ids = rxq_info->rspq_id;
 389         lli->nrxq = rxq_info->nrxq;
 390         lli->ciq_ids = rxq_info->rspq_id + rxq_info->nrxq;
 391         lli->nciq = rxq_info->nciq;
 392 }
 393
 394 int uld_mem_alloc(struct adapter *adap)
 395 {
 396         struct sge *s = &adap->sge;
 397
 398         adap->uld = kcalloc(adap->num_uld, sizeof(*adap->uld), GFP_KERNEL);
 399         if (!adap->uld)
 400                 return -ENOMEM;
 401
 402         s->uld_rxq_info = kzalloc(adap->num_uld *
 403                                   sizeof(struct sge_uld_rxq_info *),
 404                                   GFP_KERNEL);
 405         if (!s->uld_rxq_info)
 406                 goto err_uld;
 407
 408         return 0;
 409 err_uld:
 410         kfree(adap->uld);
 411         return -ENOMEM;
 412 }
 413
 414 void uld_mem_free(struct adapter *adap)
 415 {
 416         struct sge *s = &adap->sge;
 417
 418         kfree(s->uld_rxq_info);
 419         kfree(adap->uld);
 420 }
 421
 422 static void uld_init(struct adapter *adap, struct cxgb4_lld_info *lld)
 423 {
 424         int i;
 425
 426         lld->pdev = adap->pdev;
 427         lld->pf = adap->pf;
 428         lld->l2t = adap->l2t;
 429         lld->tids = &adap->tids;
 430         lld->ports = adap->port;
 431         lld->vr = &adap->vres;
 432         lld->mtus = adap->params.mtus;
 433         lld->ntxq = adap->sge.iscsiqsets;
 434         lld->nchan = adap->params.nports;
 435         lld->nports = adap->params.nports;
 436         lld->wr_cred = adap->params.ofldq_wr_cred;
 437         lld->adapter_type = adap->params.chip;
 438         lld->cclk_ps = 1000000000 / adap->params.vpd.cclk;
 439         lld->udb_density = 1 << adap->params.sge.eq_qpp;
 440         lld->ucq_density = 1 << adap->params.sge.iq_qpp;
 441         lld->filt_mode = adap->params.tp.vlan_pri_map;
 442         /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */
 443         for (i = 0; i < NCHAN; i++)
 444                 lld->tx_modq[i] = i;
 445         lld->gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS_A);
 446         lld->db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL_A);
 447         lld->fw_vers = adap->params.fw_vers;
 448         lld->dbfifo_int_thresh = dbfifo_int_thresh;
 449         lld->sge_ingpadboundary = adap->sge.fl_align;
 450         lld->sge_egrstatuspagesize = adap->sge.stat_len;
 451         lld->sge_pktshift = adap->sge.pktshift;
 452         lld->enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN;
 453         lld->max_ordird_qp = adap->params.max_ordird_qp;
 454         lld->max_ird_adapter = adap->params.max_ird_adapter;
 455         lld->ulptx_memwrite_dsgl = adap->params.ulptx_memwrite_dsgl;
 456         lld->nodeid = dev_to_node(adap->pdev_dev);
 457 }
 458
 459 static void uld_attach(struct adapter *adap, unsigned int uld)
 460 {
 461         void *handle;
 462         struct cxgb4_lld_info lli;
 463
 464         uld_init(adap, &lli);
 465         uld_queue_init(adap, uld, &lli);
 466
 467         handle = adap->uld[uld].add(&lli);
 468         if (IS_ERR(handle)) {
 469                 dev_warn(adap->pdev_dev,
 470                          "could not attach to the %s driver, error %ld\n",
 471                          adap->uld[uld].name, PTR_ERR(handle));
 472                 return;
 473         }
 474
 475         adap->uld[uld].handle = handle;
 476
 477         if (adap->flags & FULL_INIT_DONE)
 478                 adap->uld[uld].state_change(handle, CXGB4_STATE_UP);
 479 }
 480
 481 int cxgb4_register_pci_uld(enum cxgb4_pci_uld type,
 482                            struct cxgb4_pci_uld_info *p)
 483 {
 484         int ret = 0;
 485         struct adapter *adap;
 486
 487         if (type >= CXGB4_PCI_ULD_MAX)
 488                 return -EINVAL;
 489
 490         mutex_lock(&uld_mutex);
 491         list_for_each_entry(adap, &adapter_list, list_node) {
 492                 if (!is_pci_uld(adap))
 493                         continue;
 494                 ret = cfg_queues_uld(adap, type, p);
 495                 if (ret)
 496                         goto out;
 497                 ret = setup_sge_queues_uld(adap, type, p->lro);
 498                 if (ret)
 499                         goto free_queues;
 500                 if (adap->flags & USING_MSIX) {
 501                         name_msix_vecs_uld(adap, type);
 502                         ret = request_msix_queue_irqs_uld(adap, type);
 503                         if (ret)
 504                                 goto free_rxq;
 505                 }
 506                 if (adap->flags & FULL_INIT_DONE)
 507                         enable_rx_uld(adap, type);
 508                 if (adap->uld[type].add) {
 509                         ret = -EBUSY;
 510                         goto free_irq;
 511                 }
 512                 adap->uld[type] = *p;
 513                 uld_attach(adap, type);
 514         }
 515         mutex_unlock(&uld_mutex);
 516         return 0;
 517
 518 free_irq:
 519         if (adap->flags & USING_MSIX)
 520                 free_msix_queue_irqs_uld(adap, type);
 521 free_rxq:
 522         free_sge_queues_uld(adap, type);
 523 free_queues:
 524         free_queues_uld(adap, type);
 525 out:
 526         mutex_unlock(&uld_mutex);
 527         return ret;
 528 }
 529 EXPORT_SYMBOL(cxgb4_register_pci_uld);
 530
 531 int cxgb4_unregister_pci_uld(enum cxgb4_pci_uld type)
 532 {
 533         struct adapter *adap;
 534
 535         if (type >= CXGB4_PCI_ULD_MAX)
 536                 return -EINVAL;
 537
 538         mutex_lock(&uld_mutex);
 539         list_for_each_entry(adap, &adapter_list, list_node) {
 540                 if (!is_pci_uld(adap))
 541                         continue;
 542                 adap->uld[type].handle = NULL;
 543                 adap->uld[type].add = NULL;
 544                 if (adap->flags & FULL_INIT_DONE)
 545                         quiesce_rx_uld(adap, type);
 546                 if (adap->flags & USING_MSIX)
 547                         free_msix_queue_irqs_uld(adap, type);
 548                 free_sge_queues_uld(adap, type);
 549                 free_queues_uld(adap, type);
 550         }
 551         mutex_unlock(&uld_mutex);
 552
 553         return 0;
 554 }
 555 EXPORT_SYMBOL(cxgb4_unregister_pci_uld);