drivers/infiniband/hw/cxgb3/iwch_provider.c

   1 /*
   2  * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/module.h>
  33 #include <linux/moduleparam.h>
  34 #include <linux/device.h>
  35 #include <linux/netdevice.h>
  36 #include <linux/etherdevice.h>
  37 #include <linux/delay.h>
  38 #include <linux/errno.h>
  39 #include <linux/list.h>
  40 #include <linux/sched.h>
  41 #include <linux/spinlock.h>
  42 #include <linux/ethtool.h>
  43 #include <linux/rtnetlink.h>
  44 #include <linux/inetdevice.h>
  45 #include <linux/slab.h>
  46
  47 #include <asm/io.h>
  48 #include <asm/irq.h>
  49 #include <asm/byteorder.h>
  50
  51 #include <rdma/iw_cm.h>
  52 #include <rdma/ib_verbs.h>
  53 #include <rdma/ib_smi.h>
  54 #include <rdma/ib_umem.h>
  55 #include <rdma/ib_user_verbs.h>
  56
  57 #include "cxio_hal.h"
  58 #include "iwch.h"
  59 #include "iwch_provider.h"
  60 #include "iwch_cm.h"
  61 #include "iwch_user.h"
  62 #include "common.h"
  63
  64 static struct ib_ah *iwch_ah_create(struct ib_pd *pd,
  65                                     struct ib_ah_attr *ah_attr)
  66 {
  67         return ERR_PTR(-ENOSYS);
  68 }
  69
  70 static int iwch_ah_destroy(struct ib_ah *ah)
  71 {
  72         return -ENOSYS;
  73 }
  74
  75 static int iwch_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
  76 {
  77         return -ENOSYS;
  78 }
  79
  80 static int iwch_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
  81 {
  82         return -ENOSYS;
  83 }
  84
  85 static int iwch_process_mad(struct ib_device *ibdev,
  86                             int mad_flags,
  87                             u8 port_num,
  88                             const struct ib_wc *in_wc,
  89                             const struct ib_grh *in_grh,
  90                             const struct ib_mad_hdr *in_mad,
  91                             size_t in_mad_size,
  92                             struct ib_mad_hdr *out_mad,
  93                             size_t *out_mad_size,
  94                             u16 *out_mad_pkey_index)
  95 {
  96         return -ENOSYS;
  97 }
  98
  99 static int iwch_dealloc_ucontext(struct ib_ucontext *context)
 100 {
 101         struct iwch_dev *rhp = to_iwch_dev(context->device);
 102         struct iwch_ucontext *ucontext = to_iwch_ucontext(context);
 103         struct iwch_mm_entry *mm, *tmp;
 104
 105         PDBG("%s context %p\n", __func__, context);
 106         list_for_each_entry_safe(mm, tmp, &ucontext->mmaps, entry)
 107                 kfree(mm);
 108         cxio_release_ucontext(&rhp->rdev, &ucontext->uctx);
 109         kfree(ucontext);
 110         return 0;
 111 }
 112
 113 static struct ib_ucontext *iwch_alloc_ucontext(struct ib_device *ibdev,
 114                                         struct ib_udata *udata)
 115 {
 116         struct iwch_ucontext *context;
 117         struct iwch_dev *rhp = to_iwch_dev(ibdev);
 118
 119         PDBG("%s ibdev %p\n", __func__, ibdev);
 120         context = kzalloc(sizeof(*context), GFP_KERNEL);
 121         if (!context)
 122                 return ERR_PTR(-ENOMEM);
 123         cxio_init_ucontext(&rhp->rdev, &context->uctx);
 124         INIT_LIST_HEAD(&context->mmaps);
 125         spin_lock_init(&context->mmap_lock);
 126         return &context->ibucontext;
 127 }
 128
 129 static int iwch_destroy_cq(struct ib_cq *ib_cq)
 130 {
 131         struct iwch_cq *chp;
 132
 133         PDBG("%s ib_cq %p\n", __func__, ib_cq);
 134         chp = to_iwch_cq(ib_cq);
 135
 136         remove_handle(chp->rhp, &chp->rhp->cqidr, chp->cq.cqid);
 137         atomic_dec(&chp->refcnt);
 138         wait_event(chp->wait, !atomic_read(&chp->refcnt));
 139
 140         cxio_destroy_cq(&chp->rhp->rdev, &chp->cq);
 141         kfree(chp);
 142         return 0;
 143 }
 144
 145 static struct ib_cq *iwch_create_cq(struct ib_device *ibdev,
 146                                     const struct ib_cq_init_attr *attr,
 147                                     struct ib_ucontext *ib_context,
 148                                     struct ib_udata *udata)
 149 {
 150         int entries = attr->cqe;
 151         struct iwch_dev *rhp;
 152         struct iwch_cq *chp;
 153         struct iwch_create_cq_resp uresp;
 154         struct iwch_create_cq_req ureq;
 155         struct iwch_ucontext *ucontext = NULL;
 156         static int warned;
 157         size_t resplen;
 158
 159         PDBG("%s ib_dev %p entries %d\n", __func__, ibdev, entries);
 160         if (attr->flags)
 161                 return ERR_PTR(-EINVAL);
 162
 163         rhp = to_iwch_dev(ibdev);
 164         chp = kzalloc(sizeof(*chp), GFP_KERNEL);
 165         if (!chp)
 166                 return ERR_PTR(-ENOMEM);
 167
 168         if (ib_context) {
 169                 ucontext = to_iwch_ucontext(ib_context);
 170                 if (!t3a_device(rhp)) {
 171                         if (ib_copy_from_udata(&ureq, udata, sizeof (ureq))) {
 172                                 kfree(chp);
 173                                 return ERR_PTR(-EFAULT);
 174                         }
 175                         chp->user_rptr_addr = (u32 __user *)(unsigned long)ureq.user_rptr_addr;
 176                 }
 177         }
 178
 179         if (t3a_device(rhp)) {
 180
 181                 /*
 182                  * T3A: Add some fluff to handle extra CQEs inserted
 183                  * for various errors.
 184                  * Additional CQE possibilities:
 185                  *      TERMINATE,
 186                  *      incoming RDMA WRITE Failures
 187                  *      incoming RDMA READ REQUEST FAILUREs
 188                  * NOTE: We cannot ensure the CQ won't overflow.
 189                  */
 190                 entries += 16;
 191         }
 192         entries = roundup_pow_of_two(entries);
 193         chp->cq.size_log2 = ilog2(entries);
 194
 195         if (cxio_create_cq(&rhp->rdev, &chp->cq, !ucontext)) {
 196                 kfree(chp);
 197                 return ERR_PTR(-ENOMEM);
 198         }
 199         chp->rhp = rhp;
 200         chp->ibcq.cqe = 1 << chp->cq.size_log2;
 201         spin_lock_init(&chp->lock);
 202         spin_lock_init(&chp->comp_handler_lock);
 203         atomic_set(&chp->refcnt, 1);
 204         init_waitqueue_head(&chp->wait);
 205         if (insert_handle(rhp, &rhp->cqidr, chp, chp->cq.cqid)) {
 206                 cxio_destroy_cq(&chp->rhp->rdev, &chp->cq);
 207                 kfree(chp);
 208                 return ERR_PTR(-ENOMEM);
 209         }
 210
 211         if (ucontext) {
 212                 struct iwch_mm_entry *mm;
 213
 214                 mm = kmalloc(sizeof *mm, GFP_KERNEL);
 215                 if (!mm) {
 216                         iwch_destroy_cq(&chp->ibcq);
 217                         return ERR_PTR(-ENOMEM);
 218                 }
 219                 uresp.cqid = chp->cq.cqid;
 220                 uresp.size_log2 = chp->cq.size_log2;
 221                 spin_lock(&ucontext->mmap_lock);
 222                 uresp.key = ucontext->key;
 223                 ucontext->key += PAGE_SIZE;
 224                 spin_unlock(&ucontext->mmap_lock);
 225                 mm->key = uresp.key;
 226                 mm->addr = virt_to_phys(chp->cq.queue);
 227                 if (udata->outlen < sizeof uresp) {
 228                         if (!warned++)
 229                                 printk(KERN_WARNING MOD "Warning - "
 230                                        "downlevel libcxgb3 (non-fatal).\n");
 231                         mm->len = PAGE_ALIGN((1UL << uresp.size_log2) *
 232                                              sizeof(struct t3_cqe));
 233                         resplen = sizeof(struct iwch_create_cq_resp_v0);
 234                 } else {
 235                         mm->len = PAGE_ALIGN(((1UL << uresp.size_log2) + 1) *
 236                                              sizeof(struct t3_cqe));
 237                         uresp.memsize = mm->len;
 238                         uresp.reserved = 0;
 239                         resplen = sizeof uresp;
 240                 }
 241                 if (ib_copy_to_udata(udata, &uresp, resplen)) {
 242                         kfree(mm);
 243                         iwch_destroy_cq(&chp->ibcq);
 244                         return ERR_PTR(-EFAULT);
 245                 }
 246                 insert_mmap(ucontext, mm);
 247         }
 248         PDBG("created cqid 0x%0x chp %p size 0x%0x, dma_addr 0x%0llx\n",
 249              chp->cq.cqid, chp, (1 << chp->cq.size_log2),
 250              (unsigned long long) chp->cq.dma_addr);
 251         return &chp->ibcq;
 252 }
 253
 254 static int iwch_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)
 255 {
 256 #ifdef notyet
 257         struct iwch_cq *chp = to_iwch_cq(cq);
 258         struct t3_cq oldcq, newcq;
 259         int ret;
 260
 261         PDBG("%s ib_cq %p cqe %d\n", __func__, cq, cqe);
 262
 263         /* We don't downsize... */
 264         if (cqe <= cq->cqe)
 265                 return 0;
 266
 267         /* create new t3_cq with new size */
 268         cqe = roundup_pow_of_two(cqe+1);
 269         newcq.size_log2 = ilog2(cqe);
 270
 271         /* Dont allow resize to less than the current wce count */
 272         if (cqe < Q_COUNT(chp->cq.rptr, chp->cq.wptr)) {
 273                 return -ENOMEM;
 274         }
 275
 276         /* Quiesce all QPs using this CQ */
 277         ret = iwch_quiesce_qps(chp);
 278         if (ret) {
 279                 return ret;
 280         }
 281
 282         ret = cxio_create_cq(&chp->rhp->rdev, &newcq);
 283         if (ret) {
 284                 return ret;
 285         }
 286
 287         /* copy CQEs */
 288         memcpy(newcq.queue, chp->cq.queue, (1 << chp->cq.size_log2) *
 289                                         sizeof(struct t3_cqe));
 290
 291         /* old iwch_qp gets new t3_cq but keeps old cqid */
 292         oldcq = chp->cq;
 293         chp->cq = newcq;
 294         chp->cq.cqid = oldcq.cqid;
 295
 296         /* resize new t3_cq to update the HW context */
 297         ret = cxio_resize_cq(&chp->rhp->rdev, &chp->cq);
 298         if (ret) {
 299                 chp->cq = oldcq;
 300                 return ret;
 301         }
 302         chp->ibcq.cqe = (1<<chp->cq.size_log2) - 1;
 303
 304         /* destroy old t3_cq */
 305         oldcq.cqid = newcq.cqid;
 306         ret = cxio_destroy_cq(&chp->rhp->rdev, &oldcq);
 307         if (ret) {
 308                 printk(KERN_ERR MOD "%s - cxio_destroy_cq failed %d\n",
 309                         __func__, ret);
 310         }
 311
 312         /* add user hooks here */
 313
 314         /* resume qps */
 315         ret = iwch_resume_qps(chp);
 316         return ret;
 317 #else
 318         return -ENOSYS;
 319 #endif
 320 }
 321
 322 static int iwch_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 323 {
 324         struct iwch_dev *rhp;
 325         struct iwch_cq *chp;
 326         enum t3_cq_opcode cq_op;
 327         int err;
 328         unsigned long flag;
 329         u32 rptr;
 330
 331         chp = to_iwch_cq(ibcq);
 332         rhp = chp->rhp;
 333         if ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)
 334                 cq_op = CQ_ARM_SE;
 335         else
 336                 cq_op = CQ_ARM_AN;
 337         if (chp->user_rptr_addr) {
 338                 if (get_user(rptr, chp->user_rptr_addr))
 339                         return -EFAULT;
 340                 spin_lock_irqsave(&chp->lock, flag);
 341                 chp->cq.rptr = rptr;
 342         } else
 343                 spin_lock_irqsave(&chp->lock, flag);
 344         PDBG("%s rptr 0x%x\n", __func__, chp->cq.rptr);
 345         err = cxio_hal_cq_op(&rhp->rdev, &chp->cq, cq_op, 0);
 346         spin_unlock_irqrestore(&chp->lock, flag);
 347         if (err < 0)
 348                 printk(KERN_ERR MOD "Error %d rearming CQID 0x%x\n", err,
 349                        chp->cq.cqid);
 350         if (err > 0 && !(flags & IB_CQ_REPORT_MISSED_EVENTS))
 351                 err = 0;
 352         return err;
 353 }
 354
 355 static int iwch_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 356 {
 357         int len = vma->vm_end - vma->vm_start;
 358         u32 key = vma->vm_pgoff << PAGE_SHIFT;
 359         struct cxio_rdev *rdev_p;
 360         int ret = 0;
 361         struct iwch_mm_entry *mm;
 362         struct iwch_ucontext *ucontext;
 363         u64 addr;
 364
 365         PDBG("%s pgoff 0x%lx key 0x%x len %d\n", __func__, vma->vm_pgoff,
 366              key, len);
 367
 368         if (vma->vm_start & (PAGE_SIZE-1)) {
 369                 return -EINVAL;
 370         }
 371
 372         rdev_p = &(to_iwch_dev(context->device)->rdev);
 373         ucontext = to_iwch_ucontext(context);
 374
 375         mm = remove_mmap(ucontext, key, len);
 376         if (!mm)
 377                 return -EINVAL;
 378         addr = mm->addr;
 379         kfree(mm);
 380
 381         if ((addr >= rdev_p->rnic_info.udbell_physbase) &&
 382             (addr < (rdev_p->rnic_info.udbell_physbase +
 383                        rdev_p->rnic_info.udbell_len))) {
 384
 385                 /*
 386                  * Map T3 DB register.
 387                  */
 388                 if (vma->vm_flags & VM_READ) {
 389                         return -EPERM;
 390                 }
 391
 392                 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 393                 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND;
 394                 vma->vm_flags &= ~VM_MAYREAD;
 395                 ret = io_remap_pfn_range(vma, vma->vm_start,
 396                                          addr >> PAGE_SHIFT,
 397                                          len, vma->vm_page_prot);
 398         } else {
 399
 400                 /*
 401                  * Map WQ or CQ contig dma memory...
 402                  */
 403                 ret = remap_pfn_range(vma, vma->vm_start,
 404                                       addr >> PAGE_SHIFT,
 405                                       len, vma->vm_page_prot);
 406         }
 407
 408         return ret;
 409 }
 410
 411 static int iwch_deallocate_pd(struct ib_pd *pd)
 412 {
 413         struct iwch_dev *rhp;
 414         struct iwch_pd *php;
 415
 416         php = to_iwch_pd(pd);
 417         rhp = php->rhp;
 418         PDBG("%s ibpd %p pdid 0x%x\n", __func__, pd, php->pdid);
 419         cxio_hal_put_pdid(rhp->rdev.rscp, php->pdid);
 420         kfree(php);
 421         return 0;
 422 }
 423
 424 static struct ib_pd *iwch_allocate_pd(struct ib_device *ibdev,
 425                                struct ib_ucontext *context,
 426                                struct ib_udata *udata)
 427 {
 428         struct iwch_pd *php;
 429         u32 pdid;
 430         struct iwch_dev *rhp;
 431
 432         PDBG("%s ibdev %p\n", __func__, ibdev);
 433         rhp = (struct iwch_dev *) ibdev;
 434         pdid = cxio_hal_get_pdid(rhp->rdev.rscp);
 435         if (!pdid)
 436                 return ERR_PTR(-EINVAL);
 437         php = kzalloc(sizeof(*php), GFP_KERNEL);
 438         if (!php) {
 439                 cxio_hal_put_pdid(rhp->rdev.rscp, pdid);
 440                 return ERR_PTR(-ENOMEM);
 441         }
 442         php->pdid = pdid;
 443         php->rhp = rhp;
 444         if (context) {
 445                 if (ib_copy_to_udata(udata, &php->pdid, sizeof (__u32))) {
 446                         iwch_deallocate_pd(&php->ibpd);
 447                         return ERR_PTR(-EFAULT);
 448                 }
 449         }
 450         PDBG("%s pdid 0x%0x ptr 0x%p\n", __func__, pdid, php);
 451         return &php->ibpd;
 452 }
 453
 454 static int iwch_dereg_mr(struct ib_mr *ib_mr)
 455 {
 456         struct iwch_dev *rhp;
 457         struct iwch_mr *mhp;
 458         u32 mmid;
 459
 460         PDBG("%s ib_mr %p\n", __func__, ib_mr);
 461
 462         mhp = to_iwch_mr(ib_mr);
 463         kfree(mhp->pages);
 464         rhp = mhp->rhp;
 465         mmid = mhp->attr.stag >> 8;
 466         cxio_dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size,
 467                        mhp->attr.pbl_addr);
 468         iwch_free_pbl(mhp);
 469         remove_handle(rhp, &rhp->mmidr, mmid);
 470         if (mhp->kva)
 471                 kfree((void *) (unsigned long) mhp->kva);
 472         if (mhp->umem)
 473                 ib_umem_release(mhp->umem);
 474         PDBG("%s mmid 0x%x ptr %p\n", __func__, mmid, mhp);
 475         kfree(mhp);
 476         return 0;
 477 }
 478
 479 static struct ib_mr *iwch_get_dma_mr(struct ib_pd *pd, int acc)
 480 {
 481         const u64 total_size = 0xffffffff;
 482         const u64 mask = (total_size + PAGE_SIZE - 1) & PAGE_MASK;
 483         struct iwch_pd *php = to_iwch_pd(pd);
 484         struct iwch_dev *rhp = php->rhp;
 485         struct iwch_mr *mhp;
 486         __be64 *page_list;
 487         int shift = 26, npages, ret, i;
 488
 489         PDBG("%s ib_pd %p\n", __func__, pd);
 490
 491         /*
 492          * T3 only supports 32 bits of size.
 493          */
 494         if (sizeof(phys_addr_t) > 4) {
 495                 pr_warn_once(MOD "Cannot support dma_mrs on this platform.\n");
 496                 return ERR_PTR(-ENOTSUPP);
 497         }
 498
 499         mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
 500         if (!mhp)
 501                 return ERR_PTR(-ENOMEM);
 502
 503         mhp->rhp = rhp;
 504
 505         npages = (total_size + (1ULL << shift) - 1) >> shift;
 506         if (!npages) {
 507                 ret = -EINVAL;
 508                 goto err;
 509         }
 510
 511         page_list = kmalloc_array(npages, sizeof(u64), GFP_KERNEL);
 512         if (!page_list) {
 513                 ret = -ENOMEM;
 514                 goto err;
 515         }
 516
 517         for (i = 0; i < npages; i++)
 518                 page_list[i] = cpu_to_be64((u64)i << shift);
 519
 520         PDBG("%s mask 0x%llx shift %d len %lld pbl_size %d\n",
 521                 __func__, mask, shift, total_size, npages);
 522
 523         ret = iwch_alloc_pbl(mhp, npages);
 524         if (ret) {
 525                 kfree(page_list);
 526                 goto err_pbl;
 527         }
 528
 529         ret = iwch_write_pbl(mhp, page_list, npages, 0);
 530         kfree(page_list);
 531         if (ret)
 532                 goto err_pbl;
 533
 534         mhp->attr.pdid = php->pdid;
 535         mhp->attr.zbva = 0;
 536
 537         mhp->attr.perms = iwch_ib_to_tpt_access(acc);
 538         mhp->attr.va_fbo = 0;
 539         mhp->attr.page_size = shift - 12;
 540
 541         mhp->attr.len = (u32) total_size;
 542         mhp->attr.pbl_size = npages;
 543         ret = iwch_register_mem(rhp, php, mhp, shift);
 544         if (ret)
 545                 goto err_pbl;
 546
 547         return &mhp->ibmr;
 548
 549 err_pbl:
 550         iwch_free_pbl(mhp);
 551
 552 err:
 553         kfree(mhp);
 554         return ERR_PTR(ret);
 555 }
 556
 557 static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 558                                       u64 virt, int acc, struct ib_udata *udata)
 559 {
 560         __be64 *pages;
 561         int shift, n, len;
 562         int i, k, entry;
 563         int err = 0;
 564         struct iwch_dev *rhp;
 565         struct iwch_pd *php;
 566         struct iwch_mr *mhp;
 567         struct iwch_reg_user_mr_resp uresp;
 568         struct scatterlist *sg;
 569         PDBG("%s ib_pd %p\n", __func__, pd);
 570
 571         php = to_iwch_pd(pd);
 572         rhp = php->rhp;
 573         mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
 574         if (!mhp)
 575                 return ERR_PTR(-ENOMEM);
 576
 577         mhp->rhp = rhp;
 578
 579         mhp->umem = ib_umem_get(pd->uobject->context, start, length, acc, 0);
 580         if (IS_ERR(mhp->umem)) {
 581                 err = PTR_ERR(mhp->umem);
 582                 kfree(mhp);
 583                 return ERR_PTR(err);
 584         }
 585
 586         shift = ffs(mhp->umem->page_size) - 1;
 587
 588         n = mhp->umem->nmap;
 589
 590         err = iwch_alloc_pbl(mhp, n);
 591         if (err)
 592                 goto err;
 593
 594         pages = (__be64 *) __get_free_page(GFP_KERNEL);
 595         if (!pages) {
 596                 err = -ENOMEM;
 597                 goto err_pbl;
 598         }
 599
 600         i = n = 0;
 601
 602         for_each_sg(mhp->umem->sg_head.sgl, sg, mhp->umem->nmap, entry) {
 603                         len = sg_dma_len(sg) >> shift;
 604                         for (k = 0; k < len; ++k) {
 605                                 pages[i++] = cpu_to_be64(sg_dma_address(sg) +
 606                                         mhp->umem->page_size * k);
 607                                 if (i == PAGE_SIZE / sizeof *pages) {
 608                                         err = iwch_write_pbl(mhp, pages, i, n);
 609                                         if (err)
 610                                                 goto pbl_done;
 611                                         n += i;
 612                                         i = 0;
 613                                 }
 614                         }
 615         }
 616
 617         if (i)
 618                 err = iwch_write_pbl(mhp, pages, i, n);
 619
 620 pbl_done:
 621         free_page((unsigned long) pages);
 622         if (err)
 623                 goto err_pbl;
 624
 625         mhp->attr.pdid = php->pdid;
 626         mhp->attr.zbva = 0;
 627         mhp->attr.perms = iwch_ib_to_tpt_access(acc);
 628         mhp->attr.va_fbo = virt;
 629         mhp->attr.page_size = shift - 12;
 630         mhp->attr.len = (u32) length;
 631
 632         err = iwch_register_mem(rhp, php, mhp, shift);
 633         if (err)
 634                 goto err_pbl;
 635
 636         if (udata && !t3a_device(rhp)) {
 637                 uresp.pbl_addr = (mhp->attr.pbl_addr -
 638                                  rhp->rdev.rnic_info.pbl_base) >> 3;
 639                 PDBG("%s user resp pbl_addr 0x%x\n", __func__,
 640                      uresp.pbl_addr);
 641
 642                 if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
 643                         iwch_dereg_mr(&mhp->ibmr);
 644                         err = -EFAULT;
 645                         goto err;
 646                 }
 647         }
 648
 649         return &mhp->ibmr;
 650
 651 err_pbl:
 652         iwch_free_pbl(mhp);
 653
 654 err:
 655         ib_umem_release(mhp->umem);
 656         kfree(mhp);
 657         return ERR_PTR(err);
 658 }
 659
 660 static struct ib_mw *iwch_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
 661                                    struct ib_udata *udata)
 662 {
 663         struct iwch_dev *rhp;
 664         struct iwch_pd *php;
 665         struct iwch_mw *mhp;
 666         u32 mmid;
 667         u32 stag = 0;
 668         int ret;
 669
 670         if (type != IB_MW_TYPE_1)
 671                 return ERR_PTR(-EINVAL);
 672
 673         php = to_iwch_pd(pd);
 674         rhp = php->rhp;
 675         mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
 676         if (!mhp)
 677                 return ERR_PTR(-ENOMEM);
 678         ret = cxio_allocate_window(&rhp->rdev, &stag, php->pdid);
 679         if (ret) {
 680                 kfree(mhp);
 681                 return ERR_PTR(ret);
 682         }
 683         mhp->rhp = rhp;
 684         mhp->attr.pdid = php->pdid;
 685         mhp->attr.type = TPT_MW;
 686         mhp->attr.stag = stag;
 687         mmid = (stag) >> 8;
 688         mhp->ibmw.rkey = stag;
 689         if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) {
 690                 cxio_deallocate_window(&rhp->rdev, mhp->attr.stag);
 691                 kfree(mhp);
 692                 return ERR_PTR(-ENOMEM);
 693         }
 694         PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag);
 695         return &(mhp->ibmw);
 696 }
 697
 698 static int iwch_dealloc_mw(struct ib_mw *mw)
 699 {
 700         struct iwch_dev *rhp;
 701         struct iwch_mw *mhp;
 702         u32 mmid;
 703
 704         mhp = to_iwch_mw(mw);
 705         rhp = mhp->rhp;
 706         mmid = (mw->rkey) >> 8;
 707         cxio_deallocate_window(&rhp->rdev, mhp->attr.stag);
 708         remove_handle(rhp, &rhp->mmidr, mmid);
 709         PDBG("%s ib_mw %p mmid 0x%x ptr %p\n", __func__, mw, mmid, mhp);
 710         kfree(mhp);
 711         return 0;
 712 }
 713
 714 static struct ib_mr *iwch_alloc_mr(struct ib_pd *pd,
 715                                    enum ib_mr_type mr_type,
 716                                    u32 max_num_sg)
 717 {
 718         struct iwch_dev *rhp;
 719         struct iwch_pd *php;
 720         struct iwch_mr *mhp;
 721         u32 mmid;
 722         u32 stag = 0;
 723         int ret = 0;
 724
 725         if (mr_type != IB_MR_TYPE_MEM_REG ||
 726             max_num_sg > T3_MAX_FASTREG_DEPTH)
 727                 return ERR_PTR(-EINVAL);
 728
 729         php = to_iwch_pd(pd);
 730         rhp = php->rhp;
 731         mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
 732         if (!mhp)
 733                 goto err;
 734
 735         mhp->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL);
 736         if (!mhp->pages) {
 737                 ret = -ENOMEM;
 738                 goto pl_err;
 739         }
 740
 741         mhp->rhp = rhp;
 742         ret = iwch_alloc_pbl(mhp, max_num_sg);
 743         if (ret)
 744                 goto err1;
 745         mhp->attr.pbl_size = max_num_sg;
 746         ret = cxio_allocate_stag(&rhp->rdev, &stag, php->pdid,
 747                                  mhp->attr.pbl_size, mhp->attr.pbl_addr);
 748         if (ret)
 749                 goto err2;
 750         mhp->attr.pdid = php->pdid;
 751         mhp->attr.type = TPT_NON_SHARED_MR;
 752         mhp->attr.stag = stag;
 753         mhp->attr.state = 1;
 754         mmid = (stag) >> 8;
 755         mhp->ibmr.rkey = mhp->ibmr.lkey = stag;
 756         if (insert_handle(rhp, &rhp->mmidr, mhp, mmid))
 757                 goto err3;
 758
 759         PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag);
 760         return &(mhp->ibmr);
 761 err3:
 762         cxio_dereg_mem(&rhp->rdev, stag, mhp->attr.pbl_size,
 763                        mhp->attr.pbl_addr);
 764 err2:
 765         iwch_free_pbl(mhp);
 766 err1:
 767         kfree(mhp->pages);
 768 pl_err:
 769         kfree(mhp);
 770 err:
 771         return ERR_PTR(ret);
 772 }
 773
 774 static int iwch_set_page(struct ib_mr *ibmr, u64 addr)
 775 {
 776         struct iwch_mr *mhp = to_iwch_mr(ibmr);
 777
 778         if (unlikely(mhp->npages == mhp->attr.pbl_size))
 779                 return -ENOMEM;
 780
 781         mhp->pages[mhp->npages++] = addr;
 782
 783         return 0;
 784 }
 785
 786 static int iwch_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 787                           int sg_nents, unsigned int *sg_offset)
 788 {
 789         struct iwch_mr *mhp = to_iwch_mr(ibmr);
 790
 791         mhp->npages = 0;
 792
 793         return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, iwch_set_page);
 794 }
 795
 796 static int iwch_destroy_qp(struct ib_qp *ib_qp)
 797 {
 798         struct iwch_dev *rhp;
 799         struct iwch_qp *qhp;
 800         struct iwch_qp_attributes attrs;
 801         struct iwch_ucontext *ucontext;
 802
 803         qhp = to_iwch_qp(ib_qp);
 804         rhp = qhp->rhp;
 805
 806         attrs.next_state = IWCH_QP_STATE_ERROR;
 807         iwch_modify_qp(rhp, qhp, IWCH_QP_ATTR_NEXT_STATE, &attrs, 0);
 808         wait_event(qhp->wait, !qhp->ep);
 809
 810         remove_handle(rhp, &rhp->qpidr, qhp->wq.qpid);
 811
 812         atomic_dec(&qhp->refcnt);
 813         wait_event(qhp->wait, !atomic_read(&qhp->refcnt));
 814
 815         ucontext = ib_qp->uobject ? to_iwch_ucontext(ib_qp->uobject->context)
 816                                   : NULL;
 817         cxio_destroy_qp(&rhp->rdev, &qhp->wq,
 818                         ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
 819
 820         PDBG("%s ib_qp %p qpid 0x%0x qhp %p\n", __func__,
 821              ib_qp, qhp->wq.qpid, qhp);
 822         kfree(qhp);
 823         return 0;
 824 }
 825
 826 static struct ib_qp *iwch_create_qp(struct ib_pd *pd,
 827                              struct ib_qp_init_attr *attrs,
 828                              struct ib_udata *udata)
 829 {
 830         struct iwch_dev *rhp;
 831         struct iwch_qp *qhp;
 832         struct iwch_pd *php;
 833         struct iwch_cq *schp;
 834         struct iwch_cq *rchp;
 835         struct iwch_create_qp_resp uresp;
 836         int wqsize, sqsize, rqsize;
 837         struct iwch_ucontext *ucontext;
 838
 839         PDBG("%s ib_pd %p\n", __func__, pd);
 840         if (attrs->qp_type != IB_QPT_RC)
 841                 return ERR_PTR(-EINVAL);
 842         php = to_iwch_pd(pd);
 843         rhp = php->rhp;
 844         schp = get_chp(rhp, ((struct iwch_cq *) attrs->send_cq)->cq.cqid);
 845         rchp = get_chp(rhp, ((struct iwch_cq *) attrs->recv_cq)->cq.cqid);
 846         if (!schp || !rchp)
 847                 return ERR_PTR(-EINVAL);
 848
 849         /* The RQT size must be # of entries + 1 rounded up to a power of two */
 850         rqsize = roundup_pow_of_two(attrs->cap.max_recv_wr);
 851         if (rqsize == attrs->cap.max_recv_wr)
 852                 rqsize = roundup_pow_of_two(attrs->cap.max_recv_wr+1);
 853
 854         /* T3 doesn't support RQT depth < 16 */
 855         if (rqsize < 16)
 856                 rqsize = 16;
 857
 858         if (rqsize > T3_MAX_RQ_SIZE)
 859                 return ERR_PTR(-EINVAL);
 860
 861         if (attrs->cap.max_inline_data > T3_MAX_INLINE)
 862                 return ERR_PTR(-EINVAL);
 863
 864         /*
 865          * NOTE: The SQ and total WQ sizes don't need to be
 866          * a power of two.  However, all the code assumes
 867          * they are. EG: Q_FREECNT() and friends.
 868          */
 869         sqsize = roundup_pow_of_two(attrs->cap.max_send_wr);
 870         wqsize = roundup_pow_of_two(rqsize + sqsize);
 871
 872         /*
 873          * Kernel users need more wq space for fastreg WRs which can take
 874          * 2 WR fragments.
 875          */
 876         ucontext = pd->uobject ? to_iwch_ucontext(pd->uobject->context) : NULL;
 877         if (!ucontext && wqsize < (rqsize + (2 * sqsize)))
 878                 wqsize = roundup_pow_of_two(rqsize +
 879                                 roundup_pow_of_two(attrs->cap.max_send_wr * 2));
 880         PDBG("%s wqsize %d sqsize %d rqsize %d\n", __func__,
 881              wqsize, sqsize, rqsize);
 882         qhp = kzalloc(sizeof(*qhp), GFP_KERNEL);
 883         if (!qhp)
 884                 return ERR_PTR(-ENOMEM);
 885         qhp->wq.size_log2 = ilog2(wqsize);
 886         qhp->wq.rq_size_log2 = ilog2(rqsize);
 887         qhp->wq.sq_size_log2 = ilog2(sqsize);
 888         if (cxio_create_qp(&rhp->rdev, !udata, &qhp->wq,
 889                            ucontext ? &ucontext->uctx : &rhp->rdev.uctx)) {
 890                 kfree(qhp);
 891                 return ERR_PTR(-ENOMEM);
 892         }
 893
 894         attrs->cap.max_recv_wr = rqsize - 1;
 895         attrs->cap.max_send_wr = sqsize;
 896         attrs->cap.max_inline_data = T3_MAX_INLINE;
 897
 898         qhp->rhp = rhp;
 899         qhp->attr.pd = php->pdid;
 900         qhp->attr.scq = ((struct iwch_cq *) attrs->send_cq)->cq.cqid;
 901         qhp->attr.rcq = ((struct iwch_cq *) attrs->recv_cq)->cq.cqid;
 902         qhp->attr.sq_num_entries = attrs->cap.max_send_wr;
 903         qhp->attr.rq_num_entries = attrs->cap.max_recv_wr;
 904         qhp->attr.sq_max_sges = attrs->cap.max_send_sge;
 905         qhp->attr.sq_max_sges_rdma_write = attrs->cap.max_send_sge;
 906         qhp->attr.rq_max_sges = attrs->cap.max_recv_sge;
 907         qhp->attr.state = IWCH_QP_STATE_IDLE;
 908         qhp->attr.next_state = IWCH_QP_STATE_IDLE;
 909
 910         /*
 911          * XXX - These don't get passed in from the openib user
 912          * at create time.  The CM sets them via a QP modify.
 913          * Need to fix...  I think the CM should
 914          */
 915         qhp->attr.enable_rdma_read = 1;
 916         qhp->attr.enable_rdma_write = 1;
 917         qhp->attr.enable_bind = 1;
 918         qhp->attr.max_ord = 1;
 919         qhp->attr.max_ird = 1;
 920
 921         spin_lock_init(&qhp->lock);
 922         init_waitqueue_head(&qhp->wait);
 923         atomic_set(&qhp->refcnt, 1);
 924
 925         if (insert_handle(rhp, &rhp->qpidr, qhp, qhp->wq.qpid)) {
 926                 cxio_destroy_qp(&rhp->rdev, &qhp->wq,
 927                         ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
 928                 kfree(qhp);
 929                 return ERR_PTR(-ENOMEM);
 930         }
 931
 932         if (udata) {
 933
 934                 struct iwch_mm_entry *mm1, *mm2;
 935
 936                 mm1 = kmalloc(sizeof *mm1, GFP_KERNEL);
 937                 if (!mm1) {
 938                         iwch_destroy_qp(&qhp->ibqp);
 939                         return ERR_PTR(-ENOMEM);
 940                 }
 941
 942                 mm2 = kmalloc(sizeof *mm2, GFP_KERNEL);
 943                 if (!mm2) {
 944                         kfree(mm1);
 945                         iwch_destroy_qp(&qhp->ibqp);
 946                         return ERR_PTR(-ENOMEM);
 947                 }
 948
 949                 uresp.qpid = qhp->wq.qpid;
 950                 uresp.size_log2 = qhp->wq.size_log2;
 951                 uresp.sq_size_log2 = qhp->wq.sq_size_log2;
 952                 uresp.rq_size_log2 = qhp->wq.rq_size_log2;
 953                 spin_lock(&ucontext->mmap_lock);
 954                 uresp.key = ucontext->key;
 955                 ucontext->key += PAGE_SIZE;
 956                 uresp.db_key = ucontext->key;
 957                 ucontext->key += PAGE_SIZE;
 958                 spin_unlock(&ucontext->mmap_lock);
 959                 if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
 960                         kfree(mm1);
 961                         kfree(mm2);
 962                         iwch_destroy_qp(&qhp->ibqp);
 963                         return ERR_PTR(-EFAULT);
 964                 }
 965                 mm1->key = uresp.key;
 966                 mm1->addr = virt_to_phys(qhp->wq.queue);
 967                 mm1->len = PAGE_ALIGN(wqsize * sizeof (union t3_wr));
 968                 insert_mmap(ucontext, mm1);
 969                 mm2->key = uresp.db_key;
 970                 mm2->addr = qhp->wq.udb & PAGE_MASK;
 971                 mm2->len = PAGE_SIZE;
 972                 insert_mmap(ucontext, mm2);
 973         }
 974         qhp->ibqp.qp_num = qhp->wq.qpid;
 975         init_timer(&(qhp->timer));
 976         PDBG("%s sq_num_entries %d, rq_num_entries %d "
 977              "qpid 0x%0x qhp %p dma_addr 0x%llx size %d rq_addr 0x%x\n",
 978              __func__, qhp->attr.sq_num_entries, qhp->attr.rq_num_entries,
 979              qhp->wq.qpid, qhp, (unsigned long long) qhp->wq.dma_addr,
 980              1 << qhp->wq.size_log2, qhp->wq.rq_addr);
 981         return &qhp->ibqp;
 982 }
 983
 984 static int iwch_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 985                       int attr_mask, struct ib_udata *udata)
 986 {
 987         struct iwch_dev *rhp;
 988         struct iwch_qp *qhp;
 989         enum iwch_qp_attr_mask mask = 0;
 990         struct iwch_qp_attributes attrs;
 991
 992         PDBG("%s ib_qp %p\n", __func__, ibqp);
 993
 994         /* iwarp does not support the RTR state */
 995         if ((attr_mask & IB_QP_STATE) && (attr->qp_state == IB_QPS_RTR))
 996                 attr_mask &= ~IB_QP_STATE;
 997
 998         /* Make sure we still have something left to do */
 999         if (!attr_mask)
1000                 return 0;
1001
1002         memset(&attrs, 0, sizeof attrs);
1003         qhp = to_iwch_qp(ibqp);
1004         rhp = qhp->rhp;
1005
1006         attrs.next_state = iwch_convert_state(attr->qp_state);
1007         attrs.enable_rdma_read = (attr->qp_access_flags &
1008                                IB_ACCESS_REMOTE_READ) ?  1 : 0;
1009         attrs.enable_rdma_write = (attr->qp_access_flags &
1010                                 IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
1011         attrs.enable_bind = (attr->qp_access_flags & IB_ACCESS_MW_BIND) ? 1 : 0;
1012
1013
1014         mask |= (attr_mask & IB_QP_STATE) ? IWCH_QP_ATTR_NEXT_STATE : 0;
1015         mask |= (attr_mask & IB_QP_ACCESS_FLAGS) ?
1016                         (IWCH_QP_ATTR_ENABLE_RDMA_READ |
1017                          IWCH_QP_ATTR_ENABLE_RDMA_WRITE |
1018                          IWCH_QP_ATTR_ENABLE_RDMA_BIND) : 0;
1019
1020         return iwch_modify_qp(rhp, qhp, mask, &attrs, 0);
1021 }
1022
1023 void iwch_qp_add_ref(struct ib_qp *qp)
1024 {
1025         PDBG("%s ib_qp %p\n", __func__, qp);
1026         atomic_inc(&(to_iwch_qp(qp)->refcnt));
1027 }
1028
1029 void iwch_qp_rem_ref(struct ib_qp *qp)
1030 {
1031         PDBG("%s ib_qp %p\n", __func__, qp);
1032         if (atomic_dec_and_test(&(to_iwch_qp(qp)->refcnt)))
1033                 wake_up(&(to_iwch_qp(qp)->wait));
1034 }
1035
1036 static struct ib_qp *iwch_get_qp(struct ib_device *dev, int qpn)
1037 {
1038         PDBG("%s ib_dev %p qpn 0x%x\n", __func__, dev, qpn);
1039         return (struct ib_qp *)get_qhp(to_iwch_dev(dev), qpn);
1040 }
1041
1042
1043 static int iwch_query_pkey(struct ib_device *ibdev,
1044                            u8 port, u16 index, u16 * pkey)
1045 {
1046         PDBG("%s ibdev %p\n", __func__, ibdev);
1047         *pkey = 0;
1048         return 0;
1049 }
1050
1051 static int iwch_query_gid(struct ib_device *ibdev, u8 port,
1052                           int index, union ib_gid *gid)
1053 {
1054         struct iwch_dev *dev;
1055
1056         PDBG("%s ibdev %p, port %d, index %d, gid %p\n",
1057                __func__, ibdev, port, index, gid);
1058         dev = to_iwch_dev(ibdev);
1059         BUG_ON(port == 0 || port > 2);
1060         memset(&(gid->raw[0]), 0, sizeof(gid->raw));
1061         memcpy(&(gid->raw[0]), dev->rdev.port_info.lldevs[port-1]->dev_addr, 6);
1062         return 0;
1063 }
1064
1065 static u64 fw_vers_string_to_u64(struct iwch_dev *iwch_dev)
1066 {
1067         struct ethtool_drvinfo info;
1068         struct net_device *lldev = iwch_dev->rdev.t3cdev_p->lldev;
1069         char *cp, *next;
1070         unsigned fw_maj, fw_min, fw_mic;
1071
1072         lldev->ethtool_ops->get_drvinfo(lldev, &info);
1073
1074         next = info.fw_version + 1;
1075         cp = strsep(&next, ".");
1076         sscanf(cp, "%i", &fw_maj);
1077         cp = strsep(&next, ".");
1078         sscanf(cp, "%i", &fw_min);
1079         cp = strsep(&next, ".");
1080         sscanf(cp, "%i", &fw_mic);
1081
1082         return (((u64)fw_maj & 0xffff) << 32) | ((fw_min & 0xffff) << 16) |
1083                (fw_mic & 0xffff);
1084 }
1085
1086 static int iwch_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
1087                              struct ib_udata *uhw)
1088 {
1089
1090         struct iwch_dev *dev;
1091
1092         PDBG("%s ibdev %p\n", __func__, ibdev);
1093
1094         if (uhw->inlen || uhw->outlen)
1095                 return -EINVAL;
1096
1097         dev = to_iwch_dev(ibdev);
1098         memset(props, 0, sizeof *props);
1099         memcpy(&props->sys_image_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6);
1100         props->hw_ver = dev->rdev.t3cdev_p->type;
1101         props->fw_ver = fw_vers_string_to_u64(dev);
1102         props->device_cap_flags = dev->device_cap_flags;
1103         props->page_size_cap = dev->attr.mem_pgsizes_bitmask;
1104         props->vendor_id = (u32)dev->rdev.rnic_info.pdev->vendor;
1105         props->vendor_part_id = (u32)dev->rdev.rnic_info.pdev->device;
1106         props->max_mr_size = dev->attr.max_mr_size;
1107         props->max_qp = dev->attr.max_qps;
1108         props->max_qp_wr = dev->attr.max_wrs;
1109         props->max_sge = dev->attr.max_sge_per_wr;
1110         props->max_sge_rd = 1;
1111         props->max_qp_rd_atom = dev->attr.max_rdma_reads_per_qp;
1112         props->max_qp_init_rd_atom = dev->attr.max_rdma_reads_per_qp;
1113         props->max_cq = dev->attr.max_cqs;
1114         props->max_cqe = dev->attr.max_cqes_per_cq;
1115         props->max_mr = dev->attr.max_mem_regs;
1116         props->max_pd = dev->attr.max_pds;
1117         props->local_ca_ack_delay = 0;
1118         props->max_fast_reg_page_list_len = T3_MAX_FASTREG_DEPTH;
1119
1120         return 0;
1121 }
1122
1123 static int iwch_query_port(struct ib_device *ibdev,
1124                            u8 port, struct ib_port_attr *props)
1125 {
1126         struct iwch_dev *dev;
1127         struct net_device *netdev;
1128         struct in_device *inetdev;
1129
1130         PDBG("%s ibdev %p\n", __func__, ibdev);
1131
1132         dev = to_iwch_dev(ibdev);
1133         netdev = dev->rdev.port_info.lldevs[port-1];
1134
1135         memset(props, 0, sizeof(struct ib_port_attr));
1136         props->max_mtu = IB_MTU_4096;
1137         if (netdev->mtu >= 4096)
1138                 props->active_mtu = IB_MTU_4096;
1139         else if (netdev->mtu >= 2048)
1140                 props->active_mtu = IB_MTU_2048;
1141         else if (netdev->mtu >= 1024)
1142                 props->active_mtu = IB_MTU_1024;
1143         else if (netdev->mtu >= 512)
1144                 props->active_mtu = IB_MTU_512;
1145         else
1146                 props->active_mtu = IB_MTU_256;
1147
1148         if (!netif_carrier_ok(netdev))
1149                 props->state = IB_PORT_DOWN;
1150         else {
1151                 inetdev = in_dev_get(netdev);
1152                 if (inetdev) {
1153                         if (inetdev->ifa_list)
1154                                 props->state = IB_PORT_ACTIVE;
1155                         else
1156                                 props->state = IB_PORT_INIT;
1157                         in_dev_put(inetdev);
1158                 } else
1159                         props->state = IB_PORT_INIT;
1160         }
1161
1162         props->port_cap_flags =
1163             IB_PORT_CM_SUP |
1164             IB_PORT_SNMP_TUNNEL_SUP |
1165             IB_PORT_REINIT_SUP |
1166             IB_PORT_DEVICE_MGMT_SUP |
1167             IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;
1168         props->gid_tbl_len = 1;
1169         props->pkey_tbl_len = 1;
1170         props->active_width = 2;
1171         props->active_speed = IB_SPEED_DDR;
1172         props->max_msg_sz = -1;
1173
1174         return 0;
1175 }
1176
1177 static ssize_t show_rev(struct device *dev, struct device_attribute *attr,
1178                         char *buf)
1179 {
1180         struct iwch_dev *iwch_dev = container_of(dev, struct iwch_dev,
1181                                                  ibdev.dev);
1182         PDBG("%s dev 0x%p\n", __func__, dev);
1183         return sprintf(buf, "%d\n", iwch_dev->rdev.t3cdev_p->type);
1184 }
1185
1186 static ssize_t show_fw_ver(struct device *dev, struct device_attribute *attr, char *buf)
1187 {
1188         struct iwch_dev *iwch_dev = container_of(dev, struct iwch_dev,
1189                                                  ibdev.dev);
1190         struct ethtool_drvinfo info;
1191         struct net_device *lldev = iwch_dev->rdev.t3cdev_p->lldev;
1192
1193         PDBG("%s dev 0x%p\n", __func__, dev);
1194         lldev->ethtool_ops->get_drvinfo(lldev, &info);
1195         return sprintf(buf, "%s\n", info.fw_version);
1196 }
1197
1198 static ssize_t show_hca(struct device *dev, struct device_attribute *attr,
1199                         char *buf)
1200 {
1201         struct iwch_dev *iwch_dev = container_of(dev, struct iwch_dev,
1202                                                  ibdev.dev);
1203         struct ethtool_drvinfo info;
1204         struct net_device *lldev = iwch_dev->rdev.t3cdev_p->lldev;
1205
1206         PDBG("%s dev 0x%p\n", __func__, dev);
1207         lldev->ethtool_ops->get_drvinfo(lldev, &info);
1208         return sprintf(buf, "%s\n", info.driver);
1209 }
1210
1211 static ssize_t show_board(struct device *dev, struct device_attribute *attr,
1212                           char *buf)
1213 {
1214         struct iwch_dev *iwch_dev = container_of(dev, struct iwch_dev,
1215                                                  ibdev.dev);
1216         PDBG("%s dev 0x%p\n", __func__, dev);
1217         return sprintf(buf, "%x.%x\n", iwch_dev->rdev.rnic_info.pdev->vendor,
1218                        iwch_dev->rdev.rnic_info.pdev->device);
1219 }
1220
1221 enum counters {
1222         IPINRECEIVES,
1223         IPINHDRERRORS,
1224         IPINADDRERRORS,
1225         IPINUNKNOWNPROTOS,
1226         IPINDISCARDS,
1227         IPINDELIVERS,
1228         IPOUTREQUESTS,
1229         IPOUTDISCARDS,
1230         IPOUTNOROUTES,
1231         IPREASMTIMEOUT,
1232         IPREASMREQDS,
1233         IPREASMOKS,
1234         IPREASMFAILS,
1235         TCPACTIVEOPENS,
1236         TCPPASSIVEOPENS,
1237         TCPATTEMPTFAILS,
1238         TCPESTABRESETS,
1239         TCPCURRESTAB,
1240         TCPINSEGS,
1241         TCPOUTSEGS,
1242         TCPRETRANSSEGS,
1243         TCPINERRS,
1244         TCPOUTRSTS,
1245         TCPRTOMIN,
1246         TCPRTOMAX,
1247         NR_COUNTERS
1248 };
1249
1250 static const char * const names[] = {
1251         [IPINRECEIVES] = "ipInReceives",
1252         [IPINHDRERRORS] = "ipInHdrErrors",
1253         [IPINADDRERRORS] = "ipInAddrErrors",
1254         [IPINUNKNOWNPROTOS] = "ipInUnknownProtos",
1255         [IPINDISCARDS] = "ipInDiscards",
1256         [IPINDELIVERS] = "ipInDelivers",
1257         [IPOUTREQUESTS] = "ipOutRequests",
1258         [IPOUTDISCARDS] = "ipOutDiscards",
1259         [IPOUTNOROUTES] = "ipOutNoRoutes",
1260         [IPREASMTIMEOUT] = "ipReasmTimeout",
1261         [IPREASMREQDS] = "ipReasmReqds",
1262         [IPREASMOKS] = "ipReasmOKs",
1263         [IPREASMFAILS] = "ipReasmFails",
1264         [TCPACTIVEOPENS] = "tcpActiveOpens",
1265         [TCPPASSIVEOPENS] = "tcpPassiveOpens",
1266         [TCPATTEMPTFAILS] = "tcpAttemptFails",
1267         [TCPESTABRESETS] = "tcpEstabResets",
1268         [TCPCURRESTAB] = "tcpCurrEstab",
1269         [TCPINSEGS] = "tcpInSegs",
1270         [TCPOUTSEGS] = "tcpOutSegs",
1271         [TCPRETRANSSEGS] = "tcpRetransSegs",
1272         [TCPINERRS] = "tcpInErrs",
1273         [TCPOUTRSTS] = "tcpOutRsts",
1274         [TCPRTOMIN] = "tcpRtoMin",
1275         [TCPRTOMAX] = "tcpRtoMax",
1276 };
1277
1278 static struct rdma_hw_stats *iwch_alloc_stats(struct ib_device *ibdev,
1279                                               u8 port_num)
1280 {
1281         BUILD_BUG_ON(ARRAY_SIZE(names) != NR_COUNTERS);
1282
1283         /* Our driver only supports device level stats */
1284         if (port_num != 0)
1285                 return NULL;
1286
1287         return rdma_alloc_hw_stats_struct(names, NR_COUNTERS,
1288                                           RDMA_HW_STATS_DEFAULT_LIFESPAN);
1289 }
1290
1291 static int iwch_get_mib(struct ib_device *ibdev, struct rdma_hw_stats *stats,
1292                         u8 port, int index)
1293 {
1294         struct iwch_dev *dev;
1295         struct tp_mib_stats m;
1296         int ret;
1297
1298         if (port != 0 || !stats)
1299                 return -ENOSYS;
1300
1301         PDBG("%s ibdev %p\n", __func__, ibdev);
1302         dev = to_iwch_dev(ibdev);
1303         ret = dev->rdev.t3cdev_p->ctl(dev->rdev.t3cdev_p, RDMA_GET_MIB, &m);
1304         if (ret)
1305                 return -ENOSYS;
1306
1307         stats->value[IPINRECEIVES] = ((u64)m.ipInReceive_hi << 32) +    m.ipInReceive_lo;
1308         stats->value[IPINHDRERRORS] = ((u64)m.ipInHdrErrors_hi << 32) + m.ipInHdrErrors_lo;
1309         stats->value[IPINADDRERRORS] = ((u64)m.ipInAddrErrors_hi << 32) + m.ipInAddrErrors_lo;
1310         stats->value[IPINUNKNOWNPROTOS] = ((u64)m.ipInUnknownProtos_hi << 32) + m.ipInUnknownProtos_lo;
1311         stats->value[IPINDISCARDS] = ((u64)m.ipInDiscards_hi << 32) + m.ipInDiscards_lo;
1312         stats->value[IPINDELIVERS] = ((u64)m.ipInDelivers_hi << 32) + m.ipInDelivers_lo;
1313         stats->value[IPOUTREQUESTS] = ((u64)m.ipOutRequests_hi << 32) + m.ipOutRequests_lo;
1314         stats->value[IPOUTDISCARDS] = ((u64)m.ipOutDiscards_hi << 32) + m.ipOutDiscards_lo;
1315         stats->value[IPOUTNOROUTES] = ((u64)m.ipOutNoRoutes_hi << 32) + m.ipOutNoRoutes_lo;
1316         stats->value[IPREASMTIMEOUT] =  m.ipReasmTimeout;
1317         stats->value[IPREASMREQDS] = m.ipReasmReqds;
1318         stats->value[IPREASMOKS] = m.ipReasmOKs;
1319         stats->value[IPREASMFAILS] = m.ipReasmFails;
1320         stats->value[TCPACTIVEOPENS] =  m.tcpActiveOpens;
1321         stats->value[TCPPASSIVEOPENS] = m.tcpPassiveOpens;
1322         stats->value[TCPATTEMPTFAILS] = m.tcpAttemptFails;
1323         stats->value[TCPESTABRESETS] = m.tcpEstabResets;
1324         stats->value[TCPCURRESTAB] = m.tcpOutRsts;
1325         stats->value[TCPINSEGS] = m.tcpCurrEstab;
1326         stats->value[TCPOUTSEGS] = ((u64)m.tcpInSegs_hi << 32) + m.tcpInSegs_lo;
1327         stats->value[TCPRETRANSSEGS] = ((u64)m.tcpOutSegs_hi << 32) + m.tcpOutSegs_lo;
1328         stats->value[TCPINERRS] = ((u64)m.tcpRetransSeg_hi << 32) + m.tcpRetransSeg_lo,
1329         stats->value[TCPOUTRSTS] = ((u64)m.tcpInErrs_hi << 32) + m.tcpInErrs_lo;
1330         stats->value[TCPRTOMIN] = m.tcpRtoMin;
1331         stats->value[TCPRTOMAX] = m.tcpRtoMax;
1332
1333         return stats->num_counters;
1334 }
1335
1336 static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL);
1337 static DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL);
1338 static DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL);
1339 static DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL);
1340
1341 static struct device_attribute *iwch_class_attributes[] = {
1342         &dev_attr_hw_rev,
1343         &dev_attr_fw_ver,
1344         &dev_attr_hca_type,
1345         &dev_attr_board_id,
1346 };
1347
1348 static int iwch_port_immutable(struct ib_device *ibdev, u8 port_num,
1349                                struct ib_port_immutable *immutable)
1350 {
1351         struct ib_port_attr attr;
1352         int err;
1353
1354         err = iwch_query_port(ibdev, port_num, &attr);
1355         if (err)
1356                 return err;
1357
1358         immutable->pkey_tbl_len = attr.pkey_tbl_len;
1359         immutable->gid_tbl_len = attr.gid_tbl_len;
1360         immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
1361
1362         return 0;
1363 }
1364
1365 int iwch_register_device(struct iwch_dev *dev)
1366 {
1367         int ret;
1368         int i;
1369
1370         PDBG("%s iwch_dev %p\n", __func__, dev);
1371         strlcpy(dev->ibdev.name, "cxgb3_%d", IB_DEVICE_NAME_MAX);
1372         memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid));
1373         memcpy(&dev->ibdev.node_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6);
1374         dev->ibdev.owner = THIS_MODULE;
1375         dev->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
1376                                 IB_DEVICE_MEM_WINDOW |
1377                                 IB_DEVICE_MEM_MGT_EXTENSIONS;
1378
1379         /* cxgb3 supports STag 0. */
1380         dev->ibdev.local_dma_lkey = 0;
1381
1382         dev->ibdev.uverbs_cmd_mask =
1383             (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
1384             (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
1385             (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
1386             (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
1387             (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
1388             (1ull << IB_USER_VERBS_CMD_REG_MR) |
1389             (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
1390             (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
1391             (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
1392             (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
1393             (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
1394             (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
1395             (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
1396             (1ull << IB_USER_VERBS_CMD_POLL_CQ) |
1397             (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
1398             (1ull << IB_USER_VERBS_CMD_POST_SEND) |
1399             (1ull << IB_USER_VERBS_CMD_POST_RECV);
1400         dev->ibdev.node_type = RDMA_NODE_RNIC;
1401         memcpy(dev->ibdev.node_desc, IWCH_NODE_DESC, sizeof(IWCH_NODE_DESC));
1402         dev->ibdev.phys_port_cnt = dev->rdev.port_info.nports;
1403         dev->ibdev.num_comp_vectors = 1;
1404         dev->ibdev.dma_device = &(dev->rdev.rnic_info.pdev->dev);
1405         dev->ibdev.query_device = iwch_query_device;
1406         dev->ibdev.query_port = iwch_query_port;
1407         dev->ibdev.query_pkey = iwch_query_pkey;
1408         dev->ibdev.query_gid = iwch_query_gid;
1409         dev->ibdev.alloc_ucontext = iwch_alloc_ucontext;
1410         dev->ibdev.dealloc_ucontext = iwch_dealloc_ucontext;
1411         dev->ibdev.mmap = iwch_mmap;
1412         dev->ibdev.alloc_pd = iwch_allocate_pd;
1413         dev->ibdev.dealloc_pd = iwch_deallocate_pd;
1414         dev->ibdev.create_ah = iwch_ah_create;
1415         dev->ibdev.destroy_ah = iwch_ah_destroy;
1416         dev->ibdev.create_qp = iwch_create_qp;
1417         dev->ibdev.modify_qp = iwch_ib_modify_qp;
1418         dev->ibdev.destroy_qp = iwch_destroy_qp;
1419         dev->ibdev.create_cq = iwch_create_cq;
1420         dev->ibdev.destroy_cq = iwch_destroy_cq;
1421         dev->ibdev.resize_cq = iwch_resize_cq;
1422         dev->ibdev.poll_cq = iwch_poll_cq;
1423         dev->ibdev.get_dma_mr = iwch_get_dma_mr;
1424         dev->ibdev.reg_user_mr = iwch_reg_user_mr;
1425         dev->ibdev.dereg_mr = iwch_dereg_mr;
1426         dev->ibdev.alloc_mw = iwch_alloc_mw;
1427         dev->ibdev.dealloc_mw = iwch_dealloc_mw;
1428         dev->ibdev.alloc_mr = iwch_alloc_mr;
1429         dev->ibdev.map_mr_sg = iwch_map_mr_sg;
1430         dev->ibdev.attach_mcast = iwch_multicast_attach;
1431         dev->ibdev.detach_mcast = iwch_multicast_detach;
1432         dev->ibdev.process_mad = iwch_process_mad;
1433         dev->ibdev.req_notify_cq = iwch_arm_cq;
1434         dev->ibdev.post_send = iwch_post_send;
1435         dev->ibdev.post_recv = iwch_post_receive;
1436         dev->ibdev.alloc_hw_stats = iwch_alloc_stats;
1437         dev->ibdev.get_hw_stats = iwch_get_mib;
1438         dev->ibdev.uverbs_abi_ver = IWCH_UVERBS_ABI_VERSION;
1439         dev->ibdev.get_port_immutable = iwch_port_immutable;
1440
1441         dev->ibdev.iwcm = kmalloc(sizeof(struct iw_cm_verbs), GFP_KERNEL);
1442         if (!dev->ibdev.iwcm)
1443                 return -ENOMEM;
1444
1445         dev->ibdev.iwcm->connect = iwch_connect;
1446         dev->ibdev.iwcm->accept = iwch_accept_cr;
1447         dev->ibdev.iwcm->reject = iwch_reject_cr;
1448         dev->ibdev.iwcm->create_listen = iwch_create_listen;
1449         dev->ibdev.iwcm->destroy_listen = iwch_destroy_listen;
1450         dev->ibdev.iwcm->add_ref = iwch_qp_add_ref;
1451         dev->ibdev.iwcm->rem_ref = iwch_qp_rem_ref;
1452         dev->ibdev.iwcm->get_qp = iwch_get_qp;
1453         memcpy(dev->ibdev.iwcm->ifname, dev->rdev.t3cdev_p->lldev->name,
1454                sizeof(dev->ibdev.iwcm->ifname));
1455
1456         ret = ib_register_device(&dev->ibdev, NULL);
1457         if (ret)
1458                 goto bail1;
1459
1460         for (i = 0; i < ARRAY_SIZE(iwch_class_attributes); ++i) {
1461                 ret = device_create_file(&dev->ibdev.dev,
1462                                          iwch_class_attributes[i]);
1463                 if (ret) {
1464                         goto bail2;
1465                 }
1466         }
1467         return 0;
1468 bail2:
1469         ib_unregister_device(&dev->ibdev);
1470 bail1:
1471         kfree(dev->ibdev.iwcm);
1472         return ret;
1473 }
1474
1475 void iwch_unregister_device(struct iwch_dev *dev)
1476 {
1477         int i;
1478
1479         PDBG("%s iwch_dev %p\n", __func__, dev);
1480         for (i = 0; i < ARRAY_SIZE(iwch_class_attributes); ++i)
1481                 device_remove_file(&dev->ibdev.dev,
1482                                    iwch_class_attributes[i]);
1483         ib_unregister_device(&dev->ibdev);
1484         kfree(dev->ibdev.iwcm);
1485         return;
1486 }