drivers/nvme/host/rdma.c

   1 /*
   2  * NVMe over Fabrics RDMA host code.
   3  * Copyright (c) 2015-2016 HGST, a Western Digital Company.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  */
  14 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  15 #include <linux/module.h>
  16 #include <linux/init.h>
  17 #include <linux/slab.h>
  18 #include <linux/err.h>
  19 #include <linux/string.h>
  20 #include <linux/atomic.h>
  21 #include <linux/blk-mq.h>
  22 #include <linux/types.h>
  23 #include <linux/list.h>
  24 #include <linux/mutex.h>
  25 #include <linux/scatterlist.h>
  26 #include <linux/nvme.h>
  27 #include <asm/unaligned.h>
  28
  29 #include <rdma/ib_verbs.h>
  30 #include <rdma/rdma_cm.h>
  31 #include <rdma/ib_cm.h>
  32 #include <linux/nvme-rdma.h>
  33
  34 #include "nvme.h"
  35 #include "fabrics.h"
  36
  37
  38 #define NVME_RDMA_CONNECT_TIMEOUT_MS    1000            /* 1 second */
  39
  40 #define NVME_RDMA_MAX_SEGMENT_SIZE      0xffffff        /* 24-bit SGL field */
  41
  42 #define NVME_RDMA_MAX_SEGMENTS          256
  43
  44 #define NVME_RDMA_MAX_INLINE_SEGMENTS   1
  45
  46 /*
  47  * We handle AEN commands ourselves and don't even let the
  48  * block layer know about them.
  49  */
  50 #define NVME_RDMA_NR_AEN_COMMANDS      1
  51 #define NVME_RDMA_AQ_BLKMQ_DEPTH       \
  52         (NVMF_AQ_DEPTH - NVME_RDMA_NR_AEN_COMMANDS)
  53
  54 struct nvme_rdma_device {
  55         struct ib_device       *dev;
  56         struct ib_pd           *pd;
  57         struct ib_mr           *mr;
  58         struct kref             ref;
  59         struct list_head        entry;
  60 };
  61
  62 struct nvme_rdma_qe {
  63         struct ib_cqe           cqe;
  64         void                    *data;
  65         u64                     dma;
  66 };
  67
  68 struct nvme_rdma_queue;
  69 struct nvme_rdma_request {
  70         struct ib_mr            *mr;
  71         struct nvme_rdma_qe     sqe;
  72         struct ib_sge           sge[1 + NVME_RDMA_MAX_INLINE_SEGMENTS];
  73         u32                     num_sge;
  74         int                     nents;
  75         bool                    inline_data;
  76         struct ib_reg_wr        reg_wr;
  77         struct ib_cqe           reg_cqe;
  78         struct nvme_rdma_queue  *queue;
  79         struct sg_table         sg_table;
  80         struct scatterlist      first_sgl[];
  81 };
  82
  83 enum nvme_rdma_queue_flags {
  84         NVME_RDMA_Q_CONNECTED = (1 << 0),
  85         NVME_RDMA_IB_QUEUE_ALLOCATED = (1 << 1),
  86         NVME_RDMA_Q_DELETING = (1 << 2),
  87 };
  88
  89 struct nvme_rdma_queue {
  90         struct nvme_rdma_qe     *rsp_ring;
  91         u8                      sig_count;
  92         int                     queue_size;
  93         size_t                  cmnd_capsule_len;
  94         struct nvme_rdma_ctrl   *ctrl;
  95         struct nvme_rdma_device *device;
  96         struct ib_cq            *ib_cq;
  97         struct ib_qp            *qp;
  98
  99         unsigned long           flags;
 100         struct rdma_cm_id       *cm_id;
 101         int                     cm_error;
 102         struct completion       cm_done;
 103 };
 104
 105 struct nvme_rdma_ctrl {
 106         /* read and written in the hot path */
 107         spinlock_t              lock;
 108
 109         /* read only in the hot path */
 110         struct nvme_rdma_queue  *queues;
 111         u32                     queue_count;
 112
 113         /* other member variables */
 114         struct blk_mq_tag_set   tag_set;
 115         struct work_struct      delete_work;
 116         struct work_struct      reset_work;
 117         struct work_struct      err_work;
 118
 119         struct nvme_rdma_qe     async_event_sqe;
 120
 121         int                     reconnect_delay;
 122         struct delayed_work     reconnect_work;
 123
 124         struct list_head        list;
 125
 126         struct blk_mq_tag_set   admin_tag_set;
 127         struct nvme_rdma_device *device;
 128
 129         u64                     cap;
 130         u32                     max_fr_pages;
 131
 132         union {
 133                 struct sockaddr addr;
 134                 struct sockaddr_in addr_in;
 135         };
 136
 137         struct nvme_ctrl        ctrl;
 138 };
 139
 140 static inline struct nvme_rdma_ctrl *to_rdma_ctrl(struct nvme_ctrl *ctrl)
 141 {
 142         return container_of(ctrl, struct nvme_rdma_ctrl, ctrl);
 143 }
 144
 145 static LIST_HEAD(device_list);
 146 static DEFINE_MUTEX(device_list_mutex);
 147
 148 static LIST_HEAD(nvme_rdma_ctrl_list);
 149 static DEFINE_MUTEX(nvme_rdma_ctrl_mutex);
 150
 151 static struct workqueue_struct *nvme_rdma_wq;
 152
 153 /*
 154  * Disabling this option makes small I/O goes faster, but is fundamentally
 155  * unsafe.  With it turned off we will have to register a global rkey that
 156  * allows read and write access to all physical memory.
 157  */
 158 static bool register_always = true;
 159 module_param(register_always, bool, 0444);
 160 MODULE_PARM_DESC(register_always,
 161          "Use memory registration even for contiguous memory regions");
 162
 163 static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,
 164                 struct rdma_cm_event *event);
 165 static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc);
 166
 167 /* XXX: really should move to a generic header sooner or later.. */
 168 static inline void put_unaligned_le24(u32 val, u8 *p)
 169 {
 170         *p++ = val;
 171         *p++ = val >> 8;
 172         *p++ = val >> 16;
 173 }
 174
 175 static inline int nvme_rdma_queue_idx(struct nvme_rdma_queue *queue)
 176 {
 177         return queue - queue->ctrl->queues;
 178 }
 179
 180 static inline size_t nvme_rdma_inline_data_size(struct nvme_rdma_queue *queue)
 181 {
 182         return queue->cmnd_capsule_len - sizeof(struct nvme_command);
 183 }
 184
 185 static void nvme_rdma_free_qe(struct ib_device *ibdev, struct nvme_rdma_qe *qe,
 186                 size_t capsule_size, enum dma_data_direction dir)
 187 {
 188         ib_dma_unmap_single(ibdev, qe->dma, capsule_size, dir);
 189         kfree(qe->data);
 190 }
 191
 192 static int nvme_rdma_alloc_qe(struct ib_device *ibdev, struct nvme_rdma_qe *qe,
 193                 size_t capsule_size, enum dma_data_direction dir)
 194 {
 195         qe->data = kzalloc(capsule_size, GFP_KERNEL);
 196         if (!qe->data)
 197                 return -ENOMEM;
 198
 199         qe->dma = ib_dma_map_single(ibdev, qe->data, capsule_size, dir);
 200         if (ib_dma_mapping_error(ibdev, qe->dma)) {
 201                 kfree(qe->data);
 202                 return -ENOMEM;
 203         }
 204
 205         return 0;
 206 }
 207
 208 static void nvme_rdma_free_ring(struct ib_device *ibdev,
 209                 struct nvme_rdma_qe *ring, size_t ib_queue_size,
 210                 size_t capsule_size, enum dma_data_direction dir)
 211 {
 212         int i;
 213
 214         for (i = 0; i < ib_queue_size; i++)
 215                 nvme_rdma_free_qe(ibdev, &ring[i], capsule_size, dir);
 216         kfree(ring);
 217 }
 218
 219 static struct nvme_rdma_qe *nvme_rdma_alloc_ring(struct ib_device *ibdev,
 220                 size_t ib_queue_size, size_t capsule_size,
 221                 enum dma_data_direction dir)
 222 {
 223         struct nvme_rdma_qe *ring;
 224         int i;
 225
 226         ring = kcalloc(ib_queue_size, sizeof(struct nvme_rdma_qe), GFP_KERNEL);
 227         if (!ring)
 228                 return NULL;
 229
 230         for (i = 0; i < ib_queue_size; i++) {
 231                 if (nvme_rdma_alloc_qe(ibdev, &ring[i], capsule_size, dir))
 232                         goto out_free_ring;
 233         }
 234
 235         return ring;
 236
 237 out_free_ring:
 238         nvme_rdma_free_ring(ibdev, ring, i, capsule_size, dir);
 239         return NULL;
 240 }
 241
 242 static void nvme_rdma_qp_event(struct ib_event *event, void *context)
 243 {
 244         pr_debug("QP event %d\n", event->event);
 245 }
 246
 247 static int nvme_rdma_wait_for_cm(struct nvme_rdma_queue *queue)
 248 {
 249         wait_for_completion_interruptible_timeout(&queue->cm_done,
 250                         msecs_to_jiffies(NVME_RDMA_CONNECT_TIMEOUT_MS) + 1);
 251         return queue->cm_error;
 252 }
 253
 254 static int nvme_rdma_create_qp(struct nvme_rdma_queue *queue, const int factor)
 255 {
 256         struct nvme_rdma_device *dev = queue->device;
 257         struct ib_qp_init_attr init_attr;
 258         int ret;
 259
 260         memset(&init_attr, 0, sizeof(init_attr));
 261         init_attr.event_handler = nvme_rdma_qp_event;
 262         /* +1 for drain */
 263         init_attr.cap.max_send_wr = factor * queue->queue_size + 1;
 264         /* +1 for drain */
 265         init_attr.cap.max_recv_wr = queue->queue_size + 1;
 266         init_attr.cap.max_recv_sge = 1;
 267         init_attr.cap.max_send_sge = 1 + NVME_RDMA_MAX_INLINE_SEGMENTS;
 268         init_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
 269         init_attr.qp_type = IB_QPT_RC;
 270         init_attr.send_cq = queue->ib_cq;
 271         init_attr.recv_cq = queue->ib_cq;
 272
 273         ret = rdma_create_qp(queue->cm_id, dev->pd, &init_attr);
 274
 275         queue->qp = queue->cm_id->qp;
 276         return ret;
 277 }
 278
 279 static int nvme_rdma_reinit_request(void *data, struct request *rq)
 280 {
 281         struct nvme_rdma_ctrl *ctrl = data;
 282         struct nvme_rdma_device *dev = ctrl->device;
 283         struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
 284         int ret = 0;
 285
 286         if (!req->mr->need_inval)
 287                 goto out;
 288
 289         ib_dereg_mr(req->mr);
 290
 291         req->mr = ib_alloc_mr(dev->pd, IB_MR_TYPE_MEM_REG,
 292                         ctrl->max_fr_pages);
 293         if (IS_ERR(req->mr)) {
 294                 ret = PTR_ERR(req->mr);
 295                 req->mr = NULL;
 296         }
 297
 298         req->mr->need_inval = false;
 299
 300 out:
 301         return ret;
 302 }
 303
 304 static void __nvme_rdma_exit_request(struct nvme_rdma_ctrl *ctrl,
 305                 struct request *rq, unsigned int queue_idx)
 306 {
 307         struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
 308         struct nvme_rdma_queue *queue = &ctrl->queues[queue_idx];
 309         struct nvme_rdma_device *dev = queue->device;
 310
 311         if (req->mr)
 312                 ib_dereg_mr(req->mr);
 313
 314         nvme_rdma_free_qe(dev->dev, &req->sqe, sizeof(struct nvme_command),
 315                         DMA_TO_DEVICE);
 316 }
 317
 318 static void nvme_rdma_exit_request(void *data, struct request *rq,
 319                                 unsigned int hctx_idx, unsigned int rq_idx)
 320 {
 321         return __nvme_rdma_exit_request(data, rq, hctx_idx + 1);
 322 }
 323
 324 static void nvme_rdma_exit_admin_request(void *data, struct request *rq,
 325                                 unsigned int hctx_idx, unsigned int rq_idx)
 326 {
 327         return __nvme_rdma_exit_request(data, rq, 0);
 328 }
 329
 330 static int __nvme_rdma_init_request(struct nvme_rdma_ctrl *ctrl,
 331                 struct request *rq, unsigned int queue_idx)
 332 {
 333         struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
 334         struct nvme_rdma_queue *queue = &ctrl->queues[queue_idx];
 335         struct nvme_rdma_device *dev = queue->device;
 336         struct ib_device *ibdev = dev->dev;
 337         int ret;
 338
 339         BUG_ON(queue_idx >= ctrl->queue_count);
 340
 341         ret = nvme_rdma_alloc_qe(ibdev, &req->sqe, sizeof(struct nvme_command),
 342                         DMA_TO_DEVICE);
 343         if (ret)
 344                 return ret;
 345
 346         req->mr = ib_alloc_mr(dev->pd, IB_MR_TYPE_MEM_REG,
 347                         ctrl->max_fr_pages);
 348         if (IS_ERR(req->mr)) {
 349                 ret = PTR_ERR(req->mr);
 350                 goto out_free_qe;
 351         }
 352
 353         req->queue = queue;
 354
 355         return 0;
 356
 357 out_free_qe:
 358         nvme_rdma_free_qe(dev->dev, &req->sqe, sizeof(struct nvme_command),
 359                         DMA_TO_DEVICE);
 360         return -ENOMEM;
 361 }
 362
 363 static int nvme_rdma_init_request(void *data, struct request *rq,
 364                                 unsigned int hctx_idx, unsigned int rq_idx,
 365                                 unsigned int numa_node)
 366 {
 367         return __nvme_rdma_init_request(data, rq, hctx_idx + 1);
 368 }
 369
 370 static int nvme_rdma_init_admin_request(void *data, struct request *rq,
 371                                 unsigned int hctx_idx, unsigned int rq_idx,
 372                                 unsigned int numa_node)
 373 {
 374         return __nvme_rdma_init_request(data, rq, 0);
 375 }
 376
 377 static int nvme_rdma_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 378                 unsigned int hctx_idx)
 379 {
 380         struct nvme_rdma_ctrl *ctrl = data;
 381         struct nvme_rdma_queue *queue = &ctrl->queues[hctx_idx + 1];
 382
 383         BUG_ON(hctx_idx >= ctrl->queue_count);
 384
 385         hctx->driver_data = queue;
 386         return 0;
 387 }
 388
 389 static int nvme_rdma_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 390                 unsigned int hctx_idx)
 391 {
 392         struct nvme_rdma_ctrl *ctrl = data;
 393         struct nvme_rdma_queue *queue = &ctrl->queues[0];
 394
 395         BUG_ON(hctx_idx != 0);
 396
 397         hctx->driver_data = queue;
 398         return 0;
 399 }
 400
 401 static void nvme_rdma_free_dev(struct kref *ref)
 402 {
 403         struct nvme_rdma_device *ndev =
 404                 container_of(ref, struct nvme_rdma_device, ref);
 405
 406         mutex_lock(&device_list_mutex);
 407         list_del(&ndev->entry);
 408         mutex_unlock(&device_list_mutex);
 409
 410         if (!register_always)
 411                 ib_dereg_mr(ndev->mr);
 412         ib_dealloc_pd(ndev->pd);
 413
 414         kfree(ndev);
 415 }
 416
 417 static void nvme_rdma_dev_put(struct nvme_rdma_device *dev)
 418 {
 419         kref_put(&dev->ref, nvme_rdma_free_dev);
 420 }
 421
 422 static int nvme_rdma_dev_get(struct nvme_rdma_device *dev)
 423 {
 424         return kref_get_unless_zero(&dev->ref);
 425 }
 426
 427 static struct nvme_rdma_device *
 428 nvme_rdma_find_get_device(struct rdma_cm_id *cm_id)
 429 {
 430         struct nvme_rdma_device *ndev;
 431
 432         mutex_lock(&device_list_mutex);
 433         list_for_each_entry(ndev, &device_list, entry) {
 434                 if (ndev->dev->node_guid == cm_id->device->node_guid &&
 435                     nvme_rdma_dev_get(ndev))
 436                         goto out_unlock;
 437         }
 438
 439         ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
 440         if (!ndev)
 441                 goto out_err;
 442
 443         ndev->dev = cm_id->device;
 444         kref_init(&ndev->ref);
 445
 446         ndev->pd = ib_alloc_pd(ndev->dev);
 447         if (IS_ERR(ndev->pd))
 448                 goto out_free_dev;
 449
 450         if (!register_always) {
 451                 ndev->mr = ib_get_dma_mr(ndev->pd,
 452                                             IB_ACCESS_LOCAL_WRITE |
 453                                             IB_ACCESS_REMOTE_READ |
 454                                             IB_ACCESS_REMOTE_WRITE);
 455                 if (IS_ERR(ndev->mr))
 456                         goto out_free_pd;
 457         }
 458
 459         if (!(ndev->dev->attrs.device_cap_flags &
 460               IB_DEVICE_MEM_MGT_EXTENSIONS)) {
 461                 dev_err(&ndev->dev->dev,
 462                         "Memory registrations not supported.\n");
 463                 goto out_free_mr;
 464         }
 465
 466         list_add(&ndev->entry, &device_list);
 467 out_unlock:
 468         mutex_unlock(&device_list_mutex);
 469         return ndev;
 470
 471 out_free_mr:
 472         if (!register_always)
 473                 ib_dereg_mr(ndev->mr);
 474 out_free_pd:
 475         ib_dealloc_pd(ndev->pd);
 476 out_free_dev:
 477         kfree(ndev);
 478 out_err:
 479         mutex_unlock(&device_list_mutex);
 480         return NULL;
 481 }
 482
 483 static void nvme_rdma_destroy_queue_ib(struct nvme_rdma_queue *queue)
 484 {
 485         struct nvme_rdma_device *dev;
 486         struct ib_device *ibdev;
 487
 488         if (!test_and_clear_bit(NVME_RDMA_IB_QUEUE_ALLOCATED, &queue->flags))
 489                 return;
 490
 491         dev = queue->device;
 492         ibdev = dev->dev;
 493         rdma_destroy_qp(queue->cm_id);
 494         ib_free_cq(queue->ib_cq);
 495
 496         nvme_rdma_free_ring(ibdev, queue->rsp_ring, queue->queue_size,
 497                         sizeof(struct nvme_completion), DMA_FROM_DEVICE);
 498
 499         nvme_rdma_dev_put(dev);
 500 }
 501
 502 static int nvme_rdma_create_queue_ib(struct nvme_rdma_queue *queue,
 503                 struct nvme_rdma_device *dev)
 504 {
 505         struct ib_device *ibdev = dev->dev;
 506         const int send_wr_factor = 3;                   /* MR, SEND, INV */
 507         const int cq_factor = send_wr_factor + 1;       /* + RECV */
 508         int comp_vector, idx = nvme_rdma_queue_idx(queue);
 509
 510         int ret;
 511
 512         queue->device = dev;
 513
 514         /*
 515          * The admin queue is barely used once the controller is live, so don't
 516          * bother to spread it out.
 517          */
 518         if (idx == 0)
 519                 comp_vector = 0;
 520         else
 521                 comp_vector = idx % ibdev->num_comp_vectors;
 522
 523
 524         /* +1 for ib_stop_cq */
 525         queue->ib_cq = ib_alloc_cq(dev->dev, queue,
 526                                 cq_factor * queue->queue_size + 1, comp_vector,
 527                                 IB_POLL_SOFTIRQ);
 528         if (IS_ERR(queue->ib_cq)) {
 529                 ret = PTR_ERR(queue->ib_cq);
 530                 goto out;
 531         }
 532
 533         ret = nvme_rdma_create_qp(queue, send_wr_factor);
 534         if (ret)
 535                 goto out_destroy_ib_cq;
 536
 537         queue->rsp_ring = nvme_rdma_alloc_ring(ibdev, queue->queue_size,
 538                         sizeof(struct nvme_completion), DMA_FROM_DEVICE);
 539         if (!queue->rsp_ring) {
 540                 ret = -ENOMEM;
 541                 goto out_destroy_qp;
 542         }
 543         set_bit(NVME_RDMA_IB_QUEUE_ALLOCATED, &queue->flags);
 544
 545         return 0;
 546
 547 out_destroy_qp:
 548         ib_destroy_qp(queue->qp);
 549 out_destroy_ib_cq:
 550         ib_free_cq(queue->ib_cq);
 551 out:
 552         return ret;
 553 }
 554
 555 static int nvme_rdma_init_queue(struct nvme_rdma_ctrl *ctrl,
 556                 int idx, size_t queue_size)
 557 {
 558         struct nvme_rdma_queue *queue;
 559         int ret;
 560
 561         queue = &ctrl->queues[idx];
 562         queue->ctrl = ctrl;
 563         queue->flags = 0;
 564         init_completion(&queue->cm_done);
 565
 566         if (idx > 0)
 567                 queue->cmnd_capsule_len = ctrl->ctrl.ioccsz * 16;
 568         else
 569                 queue->cmnd_capsule_len = sizeof(struct nvme_command);
 570
 571         queue->queue_size = queue_size;
 572
 573         queue->cm_id = rdma_create_id(&init_net, nvme_rdma_cm_handler, queue,
 574                         RDMA_PS_TCP, IB_QPT_RC);
 575         if (IS_ERR(queue->cm_id)) {
 576                 dev_info(ctrl->ctrl.device,
 577                         "failed to create CM ID: %ld\n", PTR_ERR(queue->cm_id));
 578                 return PTR_ERR(queue->cm_id);
 579         }
 580
 581         queue->cm_error = -ETIMEDOUT;
 582         ret = rdma_resolve_addr(queue->cm_id, NULL, &ctrl->addr,
 583                         NVME_RDMA_CONNECT_TIMEOUT_MS);
 584         if (ret) {
 585                 dev_info(ctrl->ctrl.device,
 586                         "rdma_resolve_addr failed (%d).\n", ret);
 587                 goto out_destroy_cm_id;
 588         }
 589
 590         ret = nvme_rdma_wait_for_cm(queue);
 591         if (ret) {
 592                 dev_info(ctrl->ctrl.device,
 593                         "rdma_resolve_addr wait failed (%d).\n", ret);
 594                 goto out_destroy_cm_id;
 595         }
 596
 597         set_bit(NVME_RDMA_Q_CONNECTED, &queue->flags);
 598
 599         return 0;
 600
 601 out_destroy_cm_id:
 602         nvme_rdma_destroy_queue_ib(queue);
 603         rdma_destroy_id(queue->cm_id);
 604         return ret;
 605 }
 606
 607 static void nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)
 608 {
 609         rdma_disconnect(queue->cm_id);
 610         ib_drain_qp(queue->qp);
 611 }
 612
 613 static void nvme_rdma_free_queue(struct nvme_rdma_queue *queue)
 614 {
 615         nvme_rdma_destroy_queue_ib(queue);
 616         rdma_destroy_id(queue->cm_id);
 617 }
 618
 619 static void nvme_rdma_stop_and_free_queue(struct nvme_rdma_queue *queue)
 620 {
 621         if (test_and_set_bit(NVME_RDMA_Q_DELETING, &queue->flags))
 622                 return;
 623         nvme_rdma_stop_queue(queue);
 624         nvme_rdma_free_queue(queue);
 625 }
 626
 627 static void nvme_rdma_free_io_queues(struct nvme_rdma_ctrl *ctrl)
 628 {
 629         int i;
 630
 631         for (i = 1; i < ctrl->queue_count; i++)
 632                 nvme_rdma_stop_and_free_queue(&ctrl->queues[i]);
 633 }
 634
 635 static int nvme_rdma_connect_io_queues(struct nvme_rdma_ctrl *ctrl)
 636 {
 637         int i, ret = 0;
 638
 639         for (i = 1; i < ctrl->queue_count; i++) {
 640                 ret = nvmf_connect_io_queue(&ctrl->ctrl, i);
 641                 if (ret)
 642                         break;
 643         }
 644
 645         return ret;
 646 }
 647
 648 static int nvme_rdma_init_io_queues(struct nvme_rdma_ctrl *ctrl)
 649 {
 650         int i, ret;
 651
 652         for (i = 1; i < ctrl->queue_count; i++) {
 653                 ret = nvme_rdma_init_queue(ctrl, i,
 654                                            ctrl->ctrl.opts->queue_size);
 655                 if (ret) {
 656                         dev_info(ctrl->ctrl.device,
 657                                 "failed to initialize i/o queue: %d\n", ret);
 658                         goto out_free_queues;
 659                 }
 660         }
 661
 662         return 0;
 663
 664 out_free_queues:
 665         for (i--; i >= 1; i--)
 666                 nvme_rdma_stop_and_free_queue(&ctrl->queues[i]);
 667
 668         return ret;
 669 }
 670
 671 static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl *ctrl)
 672 {
 673         nvme_rdma_free_qe(ctrl->queues[0].device->dev, &ctrl->async_event_sqe,
 674                         sizeof(struct nvme_command), DMA_TO_DEVICE);
 675         nvme_rdma_stop_and_free_queue(&ctrl->queues[0]);
 676         blk_cleanup_queue(ctrl->ctrl.admin_q);
 677         blk_mq_free_tag_set(&ctrl->admin_tag_set);
 678         nvme_rdma_dev_put(ctrl->device);
 679 }
 680
 681 static void nvme_rdma_free_ctrl(struct nvme_ctrl *nctrl)
 682 {
 683         struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
 684
 685         if (list_empty(&ctrl->list))
 686                 goto free_ctrl;
 687
 688         mutex_lock(&nvme_rdma_ctrl_mutex);
 689         list_del(&ctrl->list);
 690         mutex_unlock(&nvme_rdma_ctrl_mutex);
 691
 692         kfree(ctrl->queues);
 693         nvmf_free_options(nctrl->opts);
 694 free_ctrl:
 695         kfree(ctrl);
 696 }
 697
 698 static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work)
 699 {
 700         struct nvme_rdma_ctrl *ctrl = container_of(to_delayed_work(work),
 701                         struct nvme_rdma_ctrl, reconnect_work);
 702         bool changed;
 703         int ret;
 704
 705         if (ctrl->queue_count > 1) {
 706                 nvme_rdma_free_io_queues(ctrl);
 707
 708                 ret = blk_mq_reinit_tagset(&ctrl->tag_set);
 709                 if (ret)
 710                         goto requeue;
 711         }
 712
 713         nvme_rdma_stop_and_free_queue(&ctrl->queues[0]);
 714
 715         ret = blk_mq_reinit_tagset(&ctrl->admin_tag_set);
 716         if (ret)
 717                 goto requeue;
 718
 719         ret = nvme_rdma_init_queue(ctrl, 0, NVMF_AQ_DEPTH);
 720         if (ret)
 721                 goto requeue;
 722
 723         blk_mq_start_stopped_hw_queues(ctrl->ctrl.admin_q, true);
 724
 725         ret = nvmf_connect_admin_queue(&ctrl->ctrl);
 726         if (ret)
 727                 goto stop_admin_q;
 728
 729         ret = nvme_enable_ctrl(&ctrl->ctrl, ctrl->cap);
 730         if (ret)
 731                 goto stop_admin_q;
 732
 733         nvme_start_keep_alive(&ctrl->ctrl);
 734
 735         if (ctrl->queue_count > 1) {
 736                 ret = nvme_rdma_init_io_queues(ctrl);
 737                 if (ret)
 738                         goto stop_admin_q;
 739
 740                 ret = nvme_rdma_connect_io_queues(ctrl);
 741                 if (ret)
 742                         goto stop_admin_q;
 743         }
 744
 745         changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
 746         WARN_ON_ONCE(!changed);
 747
 748         if (ctrl->queue_count > 1) {
 749                 nvme_start_queues(&ctrl->ctrl);
 750                 nvme_queue_scan(&ctrl->ctrl);
 751                 nvme_queue_async_events(&ctrl->ctrl);
 752         }
 753
 754         dev_info(ctrl->ctrl.device, "Successfully reconnected\n");
 755
 756         return;
 757
 758 stop_admin_q:
 759         blk_mq_stop_hw_queues(ctrl->ctrl.admin_q);
 760 requeue:
 761         /* Make sure we are not resetting/deleting */
 762         if (ctrl->ctrl.state == NVME_CTRL_RECONNECTING) {
 763                 dev_info(ctrl->ctrl.device,
 764                         "Failed reconnect attempt, requeueing...\n");
 765                 queue_delayed_work(nvme_rdma_wq, &ctrl->reconnect_work,
 766                                         ctrl->reconnect_delay * HZ);
 767         }
 768 }
 769
 770 static void nvme_rdma_error_recovery_work(struct work_struct *work)
 771 {
 772         struct nvme_rdma_ctrl *ctrl = container_of(work,
 773                         struct nvme_rdma_ctrl, err_work);
 774         int i;
 775
 776         nvme_stop_keep_alive(&ctrl->ctrl);
 777
 778         for (i = 0; i < ctrl->queue_count; i++)
 779                 clear_bit(NVME_RDMA_Q_CONNECTED, &ctrl->queues[i].flags);
 780
 781         if (ctrl->queue_count > 1)
 782                 nvme_stop_queues(&ctrl->ctrl);
 783         blk_mq_stop_hw_queues(ctrl->ctrl.admin_q);
 784
 785         /* We must take care of fastfail/requeue all our inflight requests */
 786         if (ctrl->queue_count > 1)
 787                 blk_mq_tagset_busy_iter(&ctrl->tag_set,
 788                                         nvme_cancel_request, &ctrl->ctrl);
 789         blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
 790                                 nvme_cancel_request, &ctrl->ctrl);
 791
 792         dev_info(ctrl->ctrl.device, "reconnecting in %d seconds\n",
 793                 ctrl->reconnect_delay);
 794
 795         queue_delayed_work(nvme_rdma_wq, &ctrl->reconnect_work,
 796                                 ctrl->reconnect_delay * HZ);
 797 }
 798
 799 static void nvme_rdma_error_recovery(struct nvme_rdma_ctrl *ctrl)
 800 {
 801         if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RECONNECTING))
 802                 return;
 803
 804         queue_work(nvme_rdma_wq, &ctrl->err_work);
 805 }
 806
 807 static void nvme_rdma_wr_error(struct ib_cq *cq, struct ib_wc *wc,
 808                 const char *op)
 809 {
 810         struct nvme_rdma_queue *queue = cq->cq_context;
 811         struct nvme_rdma_ctrl *ctrl = queue->ctrl;
 812
 813         if (ctrl->ctrl.state == NVME_CTRL_LIVE)
 814                 dev_info(ctrl->ctrl.device,
 815                              "%s for CQE 0x%p failed with status %s (%d)\n",
 816                              op, wc->wr_cqe,
 817                              ib_wc_status_msg(wc->status), wc->status);
 818         nvme_rdma_error_recovery(ctrl);
 819 }
 820
 821 static void nvme_rdma_memreg_done(struct ib_cq *cq, struct ib_wc *wc)
 822 {
 823         if (unlikely(wc->status != IB_WC_SUCCESS))
 824                 nvme_rdma_wr_error(cq, wc, "MEMREG");
 825 }
 826
 827 static void nvme_rdma_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc)
 828 {
 829         if (unlikely(wc->status != IB_WC_SUCCESS))
 830                 nvme_rdma_wr_error(cq, wc, "LOCAL_INV");
 831 }
 832
 833 static int nvme_rdma_inv_rkey(struct nvme_rdma_queue *queue,
 834                 struct nvme_rdma_request *req)
 835 {
 836         struct ib_send_wr *bad_wr;
 837         struct ib_send_wr wr = {
 838                 .opcode             = IB_WR_LOCAL_INV,
 839                 .next               = NULL,
 840                 .num_sge            = 0,
 841                 .send_flags         = 0,
 842                 .ex.invalidate_rkey = req->mr->rkey,
 843         };
 844
 845         req->reg_cqe.done = nvme_rdma_inv_rkey_done;
 846         wr.wr_cqe = &req->reg_cqe;
 847
 848         return ib_post_send(queue->qp, &wr, &bad_wr);
 849 }
 850
 851 static void nvme_rdma_unmap_data(struct nvme_rdma_queue *queue,
 852                 struct request *rq)
 853 {
 854         struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
 855         struct nvme_rdma_ctrl *ctrl = queue->ctrl;
 856         struct nvme_rdma_device *dev = queue->device;
 857         struct ib_device *ibdev = dev->dev;
 858         int res;
 859
 860         if (!blk_rq_bytes(rq))
 861                 return;
 862
 863         if (req->mr->need_inval) {
 864                 res = nvme_rdma_inv_rkey(queue, req);
 865                 if (res < 0) {
 866                         dev_err(ctrl->ctrl.device,
 867                                 "Queueing INV WR for rkey %#x failed (%d)\n",
 868                                 req->mr->rkey, res);
 869                         nvme_rdma_error_recovery(queue->ctrl);
 870                 }
 871         }
 872
 873         ib_dma_unmap_sg(ibdev, req->sg_table.sgl,
 874                         req->nents, rq_data_dir(rq) ==
 875                                     WRITE ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
 876
 877         nvme_cleanup_cmd(rq);
 878         sg_free_table_chained(&req->sg_table, true);
 879 }
 880
 881 static int nvme_rdma_set_sg_null(struct nvme_command *c)
 882 {
 883         struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
 884
 885         sg->addr = 0;
 886         put_unaligned_le24(0, sg->length);
 887         put_unaligned_le32(0, sg->key);
 888         sg->type = NVME_KEY_SGL_FMT_DATA_DESC << 4;
 889         return 0;
 890 }
 891
 892 static int nvme_rdma_map_sg_inline(struct nvme_rdma_queue *queue,
 893                 struct nvme_rdma_request *req, struct nvme_command *c)
 894 {
 895         struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
 896
 897         req->sge[1].addr = sg_dma_address(req->sg_table.sgl);
 898         req->sge[1].length = sg_dma_len(req->sg_table.sgl);
 899         req->sge[1].lkey = queue->device->pd->local_dma_lkey;
 900
 901         sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff);
 902         sg->length = cpu_to_le32(sg_dma_len(req->sg_table.sgl));
 903         sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET;
 904
 905         req->inline_data = true;
 906         req->num_sge++;
 907         return 0;
 908 }
 909
 910 static int nvme_rdma_map_sg_single(struct nvme_rdma_queue *queue,
 911                 struct nvme_rdma_request *req, struct nvme_command *c)
 912 {
 913         struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
 914
 915         sg->addr = cpu_to_le64(sg_dma_address(req->sg_table.sgl));
 916         put_unaligned_le24(sg_dma_len(req->sg_table.sgl), sg->length);
 917         put_unaligned_le32(queue->device->mr->rkey, sg->key);
 918         sg->type = NVME_KEY_SGL_FMT_DATA_DESC << 4;
 919         return 0;
 920 }
 921
 922 static int nvme_rdma_map_sg_fr(struct nvme_rdma_queue *queue,
 923                 struct nvme_rdma_request *req, struct nvme_command *c,
 924                 int count)
 925 {
 926         struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
 927         int nr;
 928
 929         nr = ib_map_mr_sg(req->mr, req->sg_table.sgl, count, NULL, PAGE_SIZE);
 930         if (nr < count) {
 931                 if (nr < 0)
 932                         return nr;
 933                 return -EINVAL;
 934         }
 935
 936         ib_update_fast_reg_key(req->mr, ib_inc_rkey(req->mr->rkey));
 937
 938         req->reg_cqe.done = nvme_rdma_memreg_done;
 939         memset(&req->reg_wr, 0, sizeof(req->reg_wr));
 940         req->reg_wr.wr.opcode = IB_WR_REG_MR;
 941         req->reg_wr.wr.wr_cqe = &req->reg_cqe;
 942         req->reg_wr.wr.num_sge = 0;
 943         req->reg_wr.mr = req->mr;
 944         req->reg_wr.key = req->mr->rkey;
 945         req->reg_wr.access = IB_ACCESS_LOCAL_WRITE |
 946                              IB_ACCESS_REMOTE_READ |
 947                              IB_ACCESS_REMOTE_WRITE;
 948
 949         req->mr->need_inval = true;
 950
 951         sg->addr = cpu_to_le64(req->mr->iova);
 952         put_unaligned_le24(req->mr->length, sg->length);
 953         put_unaligned_le32(req->mr->rkey, sg->key);
 954         sg->type = (NVME_KEY_SGL_FMT_DATA_DESC << 4) |
 955                         NVME_SGL_FMT_INVALIDATE;
 956
 957         return 0;
 958 }
 959
 960 static int nvme_rdma_map_data(struct nvme_rdma_queue *queue,
 961                 struct request *rq, unsigned int map_len,
 962                 struct nvme_command *c)
 963 {
 964         struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
 965         struct nvme_rdma_device *dev = queue->device;
 966         struct ib_device *ibdev = dev->dev;
 967         int nents, count;
 968         int ret;
 969
 970         req->num_sge = 1;
 971         req->inline_data = false;
 972         req->mr->need_inval = false;
 973
 974         c->common.flags |= NVME_CMD_SGL_METABUF;
 975
 976         if (!blk_rq_bytes(rq))
 977                 return nvme_rdma_set_sg_null(c);
 978
 979         req->sg_table.sgl = req->first_sgl;
 980         ret = sg_alloc_table_chained(&req->sg_table, rq->nr_phys_segments,
 981                                 req->sg_table.sgl);
 982         if (ret)
 983                 return -ENOMEM;
 984
 985         nents = blk_rq_map_sg(rq->q, rq, req->sg_table.sgl);
 986         BUG_ON(nents > rq->nr_phys_segments);
 987         req->nents = nents;
 988
 989         count = ib_dma_map_sg(ibdev, req->sg_table.sgl, nents,
 990                     rq_data_dir(rq) == WRITE ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
 991         if (unlikely(count <= 0)) {
 992                 sg_free_table_chained(&req->sg_table, true);
 993                 return -EIO;
 994         }
 995
 996         if (count == 1) {
 997                 if (rq_data_dir(rq) == WRITE &&
 998                     map_len <= nvme_rdma_inline_data_size(queue) &&
 999                     nvme_rdma_queue_idx(queue))
1000                         return nvme_rdma_map_sg_inline(queue, req, c);
1001
1002                 if (!register_always)
1003                         return nvme_rdma_map_sg_single(queue, req, c);
1004         }
1005
1006         return nvme_rdma_map_sg_fr(queue, req, c, count);
1007 }
1008
1009 static void nvme_rdma_send_done(struct ib_cq *cq, struct ib_wc *wc)
1010 {
1011         if (unlikely(wc->status != IB_WC_SUCCESS))
1012                 nvme_rdma_wr_error(cq, wc, "SEND");
1013 }
1014
1015 static int nvme_rdma_post_send(struct nvme_rdma_queue *queue,
1016                 struct nvme_rdma_qe *qe, struct ib_sge *sge, u32 num_sge,
1017                 struct ib_send_wr *first, bool flush)
1018 {
1019         struct ib_send_wr wr, *bad_wr;
1020         int ret;
1021
1022         sge->addr   = qe->dma;
1023         sge->length = sizeof(struct nvme_command),
1024         sge->lkey   = queue->device->pd->local_dma_lkey;
1025
1026         qe->cqe.done = nvme_rdma_send_done;
1027
1028         wr.next       = NULL;
1029         wr.wr_cqe     = &qe->cqe;
1030         wr.sg_list    = sge;
1031         wr.num_sge    = num_sge;
1032         wr.opcode     = IB_WR_SEND;
1033         wr.send_flags = 0;
1034
1035         /*
1036          * Unsignalled send completions are another giant desaster in the
1037          * IB Verbs spec:  If we don't regularly post signalled sends
1038          * the send queue will fill up and only a QP reset will rescue us.
1039          * Would have been way to obvious to handle this in hardware or
1040          * at least the RDMA stack..
1041          *
1042          * This messy and racy code sniplet is copy and pasted from the iSER
1043          * initiator, and the magic '32' comes from there as well.
1044          *
1045          * Always signal the flushes. The magic request used for the flush
1046          * sequencer is not allocated in our driver's tagset and it's
1047          * triggered to be freed by blk_cleanup_queue(). So we need to
1048          * always mark it as signaled to ensure that the "wr_cqe", which is
1049          * embeded in request's payload, is not freed when __ib_process_cq()
1050          * calls wr_cqe->done().
1051          */
1052         if ((++queue->sig_count % 32) == 0 || flush)
1053                 wr.send_flags |= IB_SEND_SIGNALED;
1054
1055         if (first)
1056                 first->next = &wr;
1057         else
1058                 first = &wr;
1059
1060         ret = ib_post_send(queue->qp, first, &bad_wr);
1061         if (ret) {
1062                 dev_err(queue->ctrl->ctrl.device,
1063                              "%s failed with error code %d\n", __func__, ret);
1064         }
1065         return ret;
1066 }
1067
1068 static int nvme_rdma_post_recv(struct nvme_rdma_queue *queue,
1069                 struct nvme_rdma_qe *qe)
1070 {
1071         struct ib_recv_wr wr, *bad_wr;
1072         struct ib_sge list;
1073         int ret;
1074
1075         list.addr   = qe->dma;
1076         list.length = sizeof(struct nvme_completion);
1077         list.lkey   = queue->device->pd->local_dma_lkey;
1078
1079         qe->cqe.done = nvme_rdma_recv_done;
1080
1081         wr.next     = NULL;
1082         wr.wr_cqe   = &qe->cqe;
1083         wr.sg_list  = &list;
1084         wr.num_sge  = 1;
1085
1086         ret = ib_post_recv(queue->qp, &wr, &bad_wr);
1087         if (ret) {
1088                 dev_err(queue->ctrl->ctrl.device,
1089                         "%s failed with error code %d\n", __func__, ret);
1090         }
1091         return ret;
1092 }
1093
1094 static struct blk_mq_tags *nvme_rdma_tagset(struct nvme_rdma_queue *queue)
1095 {
1096         u32 queue_idx = nvme_rdma_queue_idx(queue);
1097
1098         if (queue_idx == 0)
1099                 return queue->ctrl->admin_tag_set.tags[queue_idx];
1100         return queue->ctrl->tag_set.tags[queue_idx - 1];
1101 }
1102
1103 static void nvme_rdma_submit_async_event(struct nvme_ctrl *arg, int aer_idx)
1104 {
1105         struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(arg);
1106         struct nvme_rdma_queue *queue = &ctrl->queues[0];
1107         struct ib_device *dev = queue->device->dev;
1108         struct nvme_rdma_qe *sqe = &ctrl->async_event_sqe;
1109         struct nvme_command *cmd = sqe->data;
1110         struct ib_sge sge;
1111         int ret;
1112
1113         if (WARN_ON_ONCE(aer_idx != 0))
1114                 return;
1115
1116         ib_dma_sync_single_for_cpu(dev, sqe->dma, sizeof(*cmd), DMA_TO_DEVICE);
1117
1118         memset(cmd, 0, sizeof(*cmd));
1119         cmd->common.opcode = nvme_admin_async_event;
1120         cmd->common.command_id = NVME_RDMA_AQ_BLKMQ_DEPTH;
1121         cmd->common.flags |= NVME_CMD_SGL_METABUF;
1122         nvme_rdma_set_sg_null(cmd);
1123
1124         ib_dma_sync_single_for_device(dev, sqe->dma, sizeof(*cmd),
1125                         DMA_TO_DEVICE);
1126
1127         ret = nvme_rdma_post_send(queue, sqe, &sge, 1, NULL, false);
1128         WARN_ON_ONCE(ret);
1129 }
1130
1131 static int nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue,
1132                 struct nvme_completion *cqe, struct ib_wc *wc, int tag)
1133 {
1134         u16 status = le16_to_cpu(cqe->status);
1135         struct request *rq;
1136         struct nvme_rdma_request *req;
1137         int ret = 0;
1138
1139         status >>= 1;
1140
1141         rq = blk_mq_tag_to_rq(nvme_rdma_tagset(queue), cqe->command_id);
1142         if (!rq) {
1143                 dev_err(queue->ctrl->ctrl.device,
1144                         "tag 0x%x on QP %#x not found\n",
1145                         cqe->command_id, queue->qp->qp_num);
1146                 nvme_rdma_error_recovery(queue->ctrl);
1147                 return ret;
1148         }
1149         req = blk_mq_rq_to_pdu(rq);
1150
1151         if (rq->cmd_type == REQ_TYPE_DRV_PRIV && rq->special)
1152                 memcpy(rq->special, cqe, sizeof(*cqe));
1153
1154         if (rq->tag == tag)
1155                 ret = 1;
1156
1157         if ((wc->wc_flags & IB_WC_WITH_INVALIDATE) &&
1158             wc->ex.invalidate_rkey == req->mr->rkey)
1159                 req->mr->need_inval = false;
1160
1161         blk_mq_complete_request(rq, status);
1162
1163         return ret;
1164 }
1165
1166 static int __nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc, int tag)
1167 {
1168         struct nvme_rdma_qe *qe =
1169                 container_of(wc->wr_cqe, struct nvme_rdma_qe, cqe);
1170         struct nvme_rdma_queue *queue = cq->cq_context;
1171         struct ib_device *ibdev = queue->device->dev;
1172         struct nvme_completion *cqe = qe->data;
1173         const size_t len = sizeof(struct nvme_completion);
1174         int ret = 0;
1175
1176         if (unlikely(wc->status != IB_WC_SUCCESS)) {
1177                 nvme_rdma_wr_error(cq, wc, "RECV");
1178                 return 0;
1179         }
1180
1181         ib_dma_sync_single_for_cpu(ibdev, qe->dma, len, DMA_FROM_DEVICE);
1182         /*
1183          * AEN requests are special as they don't time out and can
1184          * survive any kind of queue freeze and often don't respond to
1185          * aborts.  We don't even bother to allocate a struct request
1186          * for them but rather special case them here.
1187          */
1188         if (unlikely(nvme_rdma_queue_idx(queue) == 0 &&
1189                         cqe->command_id >= NVME_RDMA_AQ_BLKMQ_DEPTH))
1190                 nvme_complete_async_event(&queue->ctrl->ctrl, cqe);
1191         else
1192                 ret = nvme_rdma_process_nvme_rsp(queue, cqe, wc, tag);
1193         ib_dma_sync_single_for_device(ibdev, qe->dma, len, DMA_FROM_DEVICE);
1194
1195         nvme_rdma_post_recv(queue, qe);
1196         return ret;
1197 }
1198
1199 static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc)
1200 {
1201         __nvme_rdma_recv_done(cq, wc, -1);
1202 }
1203
1204 static int nvme_rdma_conn_established(struct nvme_rdma_queue *queue)
1205 {
1206         int ret, i;
1207
1208         for (i = 0; i < queue->queue_size; i++) {
1209                 ret = nvme_rdma_post_recv(queue, &queue->rsp_ring[i]);
1210                 if (ret)
1211                         goto out_destroy_queue_ib;
1212         }
1213
1214         return 0;
1215
1216 out_destroy_queue_ib:
1217         nvme_rdma_destroy_queue_ib(queue);
1218         return ret;
1219 }
1220
1221 static int nvme_rdma_conn_rejected(struct nvme_rdma_queue *queue,
1222                 struct rdma_cm_event *ev)
1223 {
1224         if (ev->param.conn.private_data_len) {
1225                 struct nvme_rdma_cm_rej *rej =
1226                         (struct nvme_rdma_cm_rej *)ev->param.conn.private_data;
1227
1228                 dev_err(queue->ctrl->ctrl.device,
1229                         "Connect rejected, status %d.", le16_to_cpu(rej->sts));
1230                 /* XXX: Think of something clever to do here... */
1231         } else {
1232                 dev_err(queue->ctrl->ctrl.device,
1233                         "Connect rejected, no private data.\n");
1234         }
1235
1236         return -ECONNRESET;
1237 }
1238
1239 static int nvme_rdma_addr_resolved(struct nvme_rdma_queue *queue)
1240 {
1241         struct nvme_rdma_device *dev;
1242         int ret;
1243
1244         dev = nvme_rdma_find_get_device(queue->cm_id);
1245         if (!dev) {
1246                 dev_err(queue->cm_id->device->dma_device,
1247                         "no client data found!\n");
1248                 return -ECONNREFUSED;
1249         }
1250
1251         ret = nvme_rdma_create_queue_ib(queue, dev);
1252         if (ret) {
1253                 nvme_rdma_dev_put(dev);
1254                 goto out;
1255         }
1256
1257         ret = rdma_resolve_route(queue->cm_id, NVME_RDMA_CONNECT_TIMEOUT_MS);
1258         if (ret) {
1259                 dev_err(queue->ctrl->ctrl.device,
1260                         "rdma_resolve_route failed (%d).\n",
1261                         queue->cm_error);
1262                 goto out_destroy_queue;
1263         }
1264
1265         return 0;
1266
1267 out_destroy_queue:
1268         nvme_rdma_destroy_queue_ib(queue);
1269 out:
1270         return ret;
1271 }
1272
1273 static int nvme_rdma_route_resolved(struct nvme_rdma_queue *queue)
1274 {
1275         struct nvme_rdma_ctrl *ctrl = queue->ctrl;
1276         struct rdma_conn_param param = { };
1277         struct nvme_rdma_cm_req priv = { };
1278         int ret;
1279
1280         param.qp_num = queue->qp->qp_num;
1281         param.flow_control = 1;
1282
1283         param.responder_resources = queue->device->dev->attrs.max_qp_rd_atom;
1284         /* maximum retry count */
1285         param.retry_count = 7;
1286         param.rnr_retry_count = 7;
1287         param.private_data = &priv;
1288         param.private_data_len = sizeof(priv);
1289
1290         priv.recfmt = cpu_to_le16(NVME_RDMA_CM_FMT_1_0);
1291         priv.qid = cpu_to_le16(nvme_rdma_queue_idx(queue));
1292         /*
1293          * set the admin queue depth to the minimum size
1294          * specified by the Fabrics standard.
1295          */
1296         if (priv.qid == 0) {
1297                 priv.hrqsize = cpu_to_le16(NVMF_AQ_DEPTH);
1298                 priv.hsqsize = cpu_to_le16(NVMF_AQ_DEPTH - 1);
1299         } else {
1300                 /*
1301                  * current interpretation of the fabrics spec
1302                  * is at minimum you make hrqsize sqsize+1, or a
1303                  * 1's based representation of sqsize.
1304                  */
1305                 priv.hrqsize = cpu_to_le16(queue->queue_size);
1306                 priv.hsqsize = cpu_to_le16(queue->ctrl->ctrl.sqsize);
1307         }
1308
1309         ret = rdma_connect(queue->cm_id, &param);
1310         if (ret) {
1311                 dev_err(ctrl->ctrl.device,
1312                         "rdma_connect failed (%d).\n", ret);
1313                 goto out_destroy_queue_ib;
1314         }
1315
1316         return 0;
1317
1318 out_destroy_queue_ib:
1319         nvme_rdma_destroy_queue_ib(queue);
1320         return ret;
1321 }
1322
1323 static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,
1324                 struct rdma_cm_event *ev)
1325 {
1326         struct nvme_rdma_queue *queue = cm_id->context;
1327         int cm_error = 0;
1328
1329         dev_dbg(queue->ctrl->ctrl.device, "%s (%d): status %d id %p\n",
1330                 rdma_event_msg(ev->event), ev->event,
1331                 ev->status, cm_id);
1332
1333         switch (ev->event) {
1334         case RDMA_CM_EVENT_ADDR_RESOLVED:
1335                 cm_error = nvme_rdma_addr_resolved(queue);
1336                 break;
1337         case RDMA_CM_EVENT_ROUTE_RESOLVED:
1338                 cm_error = nvme_rdma_route_resolved(queue);
1339                 break;
1340         case RDMA_CM_EVENT_ESTABLISHED:
1341                 queue->cm_error = nvme_rdma_conn_established(queue);
1342                 /* complete cm_done regardless of success/failure */
1343                 complete(&queue->cm_done);
1344                 return 0;
1345         case RDMA_CM_EVENT_REJECTED:
1346                 cm_error = nvme_rdma_conn_rejected(queue, ev);
1347                 break;
1348         case RDMA_CM_EVENT_ADDR_ERROR:
1349         case RDMA_CM_EVENT_ROUTE_ERROR:
1350         case RDMA_CM_EVENT_CONNECT_ERROR:
1351         case RDMA_CM_EVENT_UNREACHABLE:
1352                 dev_dbg(queue->ctrl->ctrl.device,
1353                         "CM error event %d\n", ev->event);
1354                 cm_error = -ECONNRESET;
1355                 break;
1356         case RDMA_CM_EVENT_DISCONNECTED:
1357         case RDMA_CM_EVENT_ADDR_CHANGE:
1358         case RDMA_CM_EVENT_TIMEWAIT_EXIT:
1359                 dev_dbg(queue->ctrl->ctrl.device,
1360                         "disconnect received - connection closed\n");
1361                 nvme_rdma_error_recovery(queue->ctrl);
1362                 break;
1363         case RDMA_CM_EVENT_DEVICE_REMOVAL:
1364                 /* device removal is handled via the ib_client API */
1365                 break;
1366         default:
1367                 dev_err(queue->ctrl->ctrl.device,
1368                         "Unexpected RDMA CM event (%d)\n", ev->event);
1369                 nvme_rdma_error_recovery(queue->ctrl);
1370                 break;
1371         }
1372
1373         if (cm_error) {
1374                 queue->cm_error = cm_error;
1375                 complete(&queue->cm_done);
1376         }
1377
1378         return 0;
1379 }
1380
1381 static enum blk_eh_timer_return
1382 nvme_rdma_timeout(struct request *rq, bool reserved)
1383 {
1384         struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1385
1386         /* queue error recovery */
1387         nvme_rdma_error_recovery(req->queue->ctrl);
1388
1389         /* fail with DNR on cmd timeout */
1390         rq->errors = NVME_SC_ABORT_REQ | NVME_SC_DNR;
1391
1392         return BLK_EH_HANDLED;
1393 }
1394
1395 static int nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx,
1396                 const struct blk_mq_queue_data *bd)
1397 {
1398         struct nvme_ns *ns = hctx->queue->queuedata;
1399         struct nvme_rdma_queue *queue = hctx->driver_data;
1400         struct request *rq = bd->rq;
1401         struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1402         struct nvme_rdma_qe *sqe = &req->sqe;
1403         struct nvme_command *c = sqe->data;
1404         bool flush = false;
1405         struct ib_device *dev;
1406         unsigned int map_len;
1407         int ret;
1408
1409         WARN_ON_ONCE(rq->tag < 0);
1410
1411         dev = queue->device->dev;
1412         ib_dma_sync_single_for_cpu(dev, sqe->dma,
1413                         sizeof(struct nvme_command), DMA_TO_DEVICE);
1414
1415         ret = nvme_setup_cmd(ns, rq, c);
1416         if (ret)
1417                 return ret;
1418
1419         c->common.command_id = rq->tag;
1420         blk_mq_start_request(rq);
1421
1422         map_len = nvme_map_len(rq);
1423         ret = nvme_rdma_map_data(queue, rq, map_len, c);
1424         if (ret < 0) {
1425                 dev_err(queue->ctrl->ctrl.device,
1426                              "Failed to map data (%d)\n", ret);
1427                 nvme_cleanup_cmd(rq);
1428                 goto err;
1429         }
1430
1431         ib_dma_sync_single_for_device(dev, sqe->dma,
1432                         sizeof(struct nvme_command), DMA_TO_DEVICE);
1433
1434         if (rq->cmd_type == REQ_TYPE_FS && req_op(rq) == REQ_OP_FLUSH)
1435                 flush = true;
1436         ret = nvme_rdma_post_send(queue, sqe, req->sge, req->num_sge,
1437                         req->mr->need_inval ? &req->reg_wr.wr : NULL, flush);
1438         if (ret) {
1439                 nvme_rdma_unmap_data(queue, rq);
1440                 goto err;
1441         }
1442
1443         return BLK_MQ_RQ_QUEUE_OK;
1444 err:
1445         return (ret == -ENOMEM || ret == -EAGAIN) ?
1446                 BLK_MQ_RQ_QUEUE_BUSY : BLK_MQ_RQ_QUEUE_ERROR;
1447 }
1448
1449 static int nvme_rdma_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
1450 {
1451         struct nvme_rdma_queue *queue = hctx->driver_data;
1452         struct ib_cq *cq = queue->ib_cq;
1453         struct ib_wc wc;
1454         int found = 0;
1455
1456         ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
1457         while (ib_poll_cq(cq, 1, &wc) > 0) {
1458                 struct ib_cqe *cqe = wc.wr_cqe;
1459
1460                 if (cqe) {
1461                         if (cqe->done == nvme_rdma_recv_done)
1462                                 found |= __nvme_rdma_recv_done(cq, &wc, tag);
1463                         else
1464                                 cqe->done(cq, &wc);
1465                 }
1466         }
1467
1468         return found;
1469 }
1470
1471 static void nvme_rdma_complete_rq(struct request *rq)
1472 {
1473         struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1474         struct nvme_rdma_queue *queue = req->queue;
1475         int error = 0;
1476
1477         nvme_rdma_unmap_data(queue, rq);
1478
1479         if (unlikely(rq->errors)) {
1480                 if (nvme_req_needs_retry(rq, rq->errors)) {
1481                         nvme_requeue_req(rq);
1482                         return;
1483                 }
1484
1485                 if (rq->cmd_type == REQ_TYPE_DRV_PRIV)
1486                         error = rq->errors;
1487                 else
1488                         error = nvme_error_status(rq->errors);
1489         }
1490
1491         blk_mq_end_request(rq, error);
1492 }
1493
1494 static struct blk_mq_ops nvme_rdma_mq_ops = {
1495         .queue_rq       = nvme_rdma_queue_rq,
1496         .complete       = nvme_rdma_complete_rq,
1497         .map_queue      = blk_mq_map_queue,
1498         .init_request   = nvme_rdma_init_request,
1499         .exit_request   = nvme_rdma_exit_request,
1500         .reinit_request = nvme_rdma_reinit_request,
1501         .init_hctx      = nvme_rdma_init_hctx,
1502         .poll           = nvme_rdma_poll,
1503         .timeout        = nvme_rdma_timeout,
1504 };
1505
1506 static struct blk_mq_ops nvme_rdma_admin_mq_ops = {
1507         .queue_rq       = nvme_rdma_queue_rq,
1508         .complete       = nvme_rdma_complete_rq,
1509         .map_queue      = blk_mq_map_queue,
1510         .init_request   = nvme_rdma_init_admin_request,
1511         .exit_request   = nvme_rdma_exit_admin_request,
1512         .reinit_request = nvme_rdma_reinit_request,
1513         .init_hctx      = nvme_rdma_init_admin_hctx,
1514         .timeout        = nvme_rdma_timeout,
1515 };
1516
1517 static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl)
1518 {
1519         int error;
1520
1521         error = nvme_rdma_init_queue(ctrl, 0, NVMF_AQ_DEPTH);
1522         if (error)
1523                 return error;
1524
1525         ctrl->device = ctrl->queues[0].device;
1526
1527         /*
1528          * We need a reference on the device as long as the tag_set is alive,
1529          * as the MRs in the request structures need a valid ib_device.
1530          */
1531         error = -EINVAL;
1532         if (!nvme_rdma_dev_get(ctrl->device))
1533                 goto out_free_queue;
1534
1535         ctrl->max_fr_pages = min_t(u32, NVME_RDMA_MAX_SEGMENTS,
1536                 ctrl->device->dev->attrs.max_fast_reg_page_list_len);
1537
1538         memset(&ctrl->admin_tag_set, 0, sizeof(ctrl->admin_tag_set));
1539         ctrl->admin_tag_set.ops = &nvme_rdma_admin_mq_ops;
1540         ctrl->admin_tag_set.queue_depth = NVME_RDMA_AQ_BLKMQ_DEPTH;
1541         ctrl->admin_tag_set.reserved_tags = 2; /* connect + keep-alive */
1542         ctrl->admin_tag_set.numa_node = NUMA_NO_NODE;
1543         ctrl->admin_tag_set.cmd_size = sizeof(struct nvme_rdma_request) +
1544                 SG_CHUNK_SIZE * sizeof(struct scatterlist);
1545         ctrl->admin_tag_set.driver_data = ctrl;
1546         ctrl->admin_tag_set.nr_hw_queues = 1;
1547         ctrl->admin_tag_set.timeout = ADMIN_TIMEOUT;
1548
1549         error = blk_mq_alloc_tag_set(&ctrl->admin_tag_set);
1550         if (error)
1551                 goto out_put_dev;
1552
1553         ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set);
1554         if (IS_ERR(ctrl->ctrl.admin_q)) {
1555                 error = PTR_ERR(ctrl->ctrl.admin_q);
1556                 goto out_free_tagset;
1557         }
1558
1559         error = nvmf_connect_admin_queue(&ctrl->ctrl);
1560         if (error)
1561                 goto out_cleanup_queue;
1562
1563         error = nvmf_reg_read64(&ctrl->ctrl, NVME_REG_CAP, &ctrl->cap);
1564         if (error) {
1565                 dev_err(ctrl->ctrl.device,
1566                         "prop_get NVME_REG_CAP failed\n");
1567                 goto out_cleanup_queue;
1568         }
1569
1570         ctrl->ctrl.sqsize =
1571                 min_t(int, NVME_CAP_MQES(ctrl->cap) + 1, ctrl->ctrl.sqsize);
1572
1573         error = nvme_enable_ctrl(&ctrl->ctrl, ctrl->cap);
1574         if (error)
1575                 goto out_cleanup_queue;
1576
1577         ctrl->ctrl.max_hw_sectors =
1578                 (ctrl->max_fr_pages - 1) << (PAGE_SHIFT - 9);
1579
1580         error = nvme_init_identify(&ctrl->ctrl);
1581         if (error)
1582                 goto out_cleanup_queue;
1583
1584         error = nvme_rdma_alloc_qe(ctrl->queues[0].device->dev,
1585                         &ctrl->async_event_sqe, sizeof(struct nvme_command),
1586                         DMA_TO_DEVICE);
1587         if (error)
1588                 goto out_cleanup_queue;
1589
1590         nvme_start_keep_alive(&ctrl->ctrl);
1591
1592         return 0;
1593
1594 out_cleanup_queue:
1595         blk_cleanup_queue(ctrl->ctrl.admin_q);
1596 out_free_tagset:
1597         /* disconnect and drain the queue before freeing the tagset */
1598         nvme_rdma_stop_queue(&ctrl->queues[0]);
1599         blk_mq_free_tag_set(&ctrl->admin_tag_set);
1600 out_put_dev:
1601         nvme_rdma_dev_put(ctrl->device);
1602 out_free_queue:
1603         nvme_rdma_free_queue(&ctrl->queues[0]);
1604         return error;
1605 }
1606
1607 static void nvme_rdma_shutdown_ctrl(struct nvme_rdma_ctrl *ctrl)
1608 {
1609         nvme_stop_keep_alive(&ctrl->ctrl);
1610         cancel_work_sync(&ctrl->err_work);
1611         cancel_delayed_work_sync(&ctrl->reconnect_work);
1612
1613         if (ctrl->queue_count > 1) {
1614                 nvme_stop_queues(&ctrl->ctrl);
1615                 blk_mq_tagset_busy_iter(&ctrl->tag_set,
1616                                         nvme_cancel_request, &ctrl->ctrl);
1617                 nvme_rdma_free_io_queues(ctrl);
1618         }
1619
1620         if (test_bit(NVME_RDMA_Q_CONNECTED, &ctrl->queues[0].flags))
1621                 nvme_shutdown_ctrl(&ctrl->ctrl);
1622
1623         blk_mq_stop_hw_queues(ctrl->ctrl.admin_q);
1624         blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
1625                                 nvme_cancel_request, &ctrl->ctrl);
1626         nvme_rdma_destroy_admin_queue(ctrl);
1627 }
1628
1629 static void __nvme_rdma_remove_ctrl(struct nvme_rdma_ctrl *ctrl, bool shutdown)
1630 {
1631         nvme_uninit_ctrl(&ctrl->ctrl);
1632         if (shutdown)
1633                 nvme_rdma_shutdown_ctrl(ctrl);
1634
1635         if (ctrl->ctrl.tagset) {
1636                 blk_cleanup_queue(ctrl->ctrl.connect_q);
1637                 blk_mq_free_tag_set(&ctrl->tag_set);
1638                 nvme_rdma_dev_put(ctrl->device);
1639         }
1640
1641         nvme_put_ctrl(&ctrl->ctrl);
1642 }
1643
1644 static void nvme_rdma_del_ctrl_work(struct work_struct *work)
1645 {
1646         struct nvme_rdma_ctrl *ctrl = container_of(work,
1647                                 struct nvme_rdma_ctrl, delete_work);
1648
1649         __nvme_rdma_remove_ctrl(ctrl, true);
1650 }
1651
1652 static int __nvme_rdma_del_ctrl(struct nvme_rdma_ctrl *ctrl)
1653 {
1654         if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_DELETING))
1655                 return -EBUSY;
1656
1657         if (!queue_work(nvme_rdma_wq, &ctrl->delete_work))
1658                 return -EBUSY;
1659
1660         return 0;
1661 }
1662
1663 static int nvme_rdma_del_ctrl(struct nvme_ctrl *nctrl)
1664 {
1665         struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
1666         int ret = 0;
1667
1668         /*
1669          * Keep a reference until all work is flushed since
1670          * __nvme_rdma_del_ctrl can free the ctrl mem
1671          */
1672         if (!kref_get_unless_zero(&ctrl->ctrl.kref))
1673                 return -EBUSY;
1674         ret = __nvme_rdma_del_ctrl(ctrl);
1675         if (!ret)
1676                 flush_work(&ctrl->delete_work);
1677         nvme_put_ctrl(&ctrl->ctrl);
1678         return ret;
1679 }
1680
1681 static void nvme_rdma_remove_ctrl_work(struct work_struct *work)
1682 {
1683         struct nvme_rdma_ctrl *ctrl = container_of(work,
1684                                 struct nvme_rdma_ctrl, delete_work);
1685
1686         __nvme_rdma_remove_ctrl(ctrl, false);
1687 }
1688
1689 static void nvme_rdma_reset_ctrl_work(struct work_struct *work)
1690 {
1691         struct nvme_rdma_ctrl *ctrl = container_of(work,
1692                                         struct nvme_rdma_ctrl, reset_work);
1693         int ret;
1694         bool changed;
1695
1696         nvme_rdma_shutdown_ctrl(ctrl);
1697
1698         ret = nvme_rdma_configure_admin_queue(ctrl);
1699         if (ret) {
1700                 /* ctrl is already shutdown, just remove the ctrl */
1701                 INIT_WORK(&ctrl->delete_work, nvme_rdma_remove_ctrl_work);
1702                 goto del_dead_ctrl;
1703         }
1704
1705         if (ctrl->queue_count > 1) {
1706                 ret = blk_mq_reinit_tagset(&ctrl->tag_set);
1707                 if (ret)
1708                         goto del_dead_ctrl;
1709
1710                 ret = nvme_rdma_init_io_queues(ctrl);
1711                 if (ret)
1712                         goto del_dead_ctrl;
1713
1714                 ret = nvme_rdma_connect_io_queues(ctrl);
1715                 if (ret)
1716                         goto del_dead_ctrl;
1717         }
1718
1719         changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
1720         WARN_ON_ONCE(!changed);
1721
1722         if (ctrl->queue_count > 1) {
1723                 nvme_start_queues(&ctrl->ctrl);
1724                 nvme_queue_scan(&ctrl->ctrl);
1725                 nvme_queue_async_events(&ctrl->ctrl);
1726         }
1727
1728         return;
1729
1730 del_dead_ctrl:
1731         /* Deleting this dead controller... */
1732         dev_warn(ctrl->ctrl.device, "Removing after reset failure\n");
1733         WARN_ON(!queue_work(nvme_rdma_wq, &ctrl->delete_work));
1734 }
1735
1736 static int nvme_rdma_reset_ctrl(struct nvme_ctrl *nctrl)
1737 {
1738         struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
1739
1740         if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RESETTING))
1741                 return -EBUSY;
1742
1743         if (!queue_work(nvme_rdma_wq, &ctrl->reset_work))
1744                 return -EBUSY;
1745
1746         flush_work(&ctrl->reset_work);
1747
1748         return 0;
1749 }
1750
1751 static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = {
1752         .name                   = "rdma",
1753         .module                 = THIS_MODULE,
1754         .is_fabrics             = true,
1755         .reg_read32             = nvmf_reg_read32,
1756         .reg_read64             = nvmf_reg_read64,
1757         .reg_write32            = nvmf_reg_write32,
1758         .reset_ctrl             = nvme_rdma_reset_ctrl,
1759         .free_ctrl              = nvme_rdma_free_ctrl,
1760         .submit_async_event     = nvme_rdma_submit_async_event,
1761         .delete_ctrl            = nvme_rdma_del_ctrl,
1762         .get_subsysnqn          = nvmf_get_subsysnqn,
1763         .get_address            = nvmf_get_address,
1764 };
1765
1766 static int nvme_rdma_create_io_queues(struct nvme_rdma_ctrl *ctrl)
1767 {
1768         struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
1769         int ret;
1770
1771         ret = nvme_set_queue_count(&ctrl->ctrl, &opts->nr_io_queues);
1772         if (ret)
1773                 return ret;
1774
1775         ctrl->queue_count = opts->nr_io_queues + 1;
1776         if (ctrl->queue_count < 2)
1777                 return 0;
1778
1779         dev_info(ctrl->ctrl.device,
1780                 "creating %d I/O queues.\n", opts->nr_io_queues);
1781
1782         ret = nvme_rdma_init_io_queues(ctrl);
1783         if (ret)
1784                 return ret;
1785
1786         /*
1787          * We need a reference on the device as long as the tag_set is alive,
1788          * as the MRs in the request structures need a valid ib_device.
1789          */
1790         ret = -EINVAL;
1791         if (!nvme_rdma_dev_get(ctrl->device))
1792                 goto out_free_io_queues;
1793
1794         memset(&ctrl->tag_set, 0, sizeof(ctrl->tag_set));
1795         ctrl->tag_set.ops = &nvme_rdma_mq_ops;
1796         ctrl->tag_set.queue_depth = ctrl->ctrl.opts->queue_size;
1797         ctrl->tag_set.reserved_tags = 1; /* fabric connect */
1798         ctrl->tag_set.numa_node = NUMA_NO_NODE;
1799         ctrl->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
1800         ctrl->tag_set.cmd_size = sizeof(struct nvme_rdma_request) +
1801                 SG_CHUNK_SIZE * sizeof(struct scatterlist);
1802         ctrl->tag_set.driver_data = ctrl;
1803         ctrl->tag_set.nr_hw_queues = ctrl->queue_count - 1;
1804         ctrl->tag_set.timeout = NVME_IO_TIMEOUT;
1805
1806         ret = blk_mq_alloc_tag_set(&ctrl->tag_set);
1807         if (ret)
1808                 goto out_put_dev;
1809         ctrl->ctrl.tagset = &ctrl->tag_set;
1810
1811         ctrl->ctrl.connect_q = blk_mq_init_queue(&ctrl->tag_set);
1812         if (IS_ERR(ctrl->ctrl.connect_q)) {
1813                 ret = PTR_ERR(ctrl->ctrl.connect_q);
1814                 goto out_free_tag_set;
1815         }
1816
1817         ret = nvme_rdma_connect_io_queues(ctrl);
1818         if (ret)
1819                 goto out_cleanup_connect_q;
1820
1821         return 0;
1822
1823 out_cleanup_connect_q:
1824         blk_cleanup_queue(ctrl->ctrl.connect_q);
1825 out_free_tag_set:
1826         blk_mq_free_tag_set(&ctrl->tag_set);
1827 out_put_dev:
1828         nvme_rdma_dev_put(ctrl->device);
1829 out_free_io_queues:
1830         nvme_rdma_free_io_queues(ctrl);
1831         return ret;
1832 }
1833
1834 static int nvme_rdma_parse_ipaddr(struct sockaddr_in *in_addr, char *p)
1835 {
1836         u8 *addr = (u8 *)&in_addr->sin_addr.s_addr;
1837         size_t buflen = strlen(p);
1838
1839         /* XXX: handle IPv6 addresses */
1840
1841         if (buflen > INET_ADDRSTRLEN)
1842                 return -EINVAL;
1843         if (in4_pton(p, buflen, addr, '\0', NULL) == 0)
1844                 return -EINVAL;
1845         in_addr->sin_family = AF_INET;
1846         return 0;
1847 }
1848
1849 static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev,
1850                 struct nvmf_ctrl_options *opts)
1851 {
1852         struct nvme_rdma_ctrl *ctrl;
1853         int ret;
1854         bool changed;
1855
1856         ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
1857         if (!ctrl)
1858                 return ERR_PTR(-ENOMEM);
1859         ctrl->ctrl.opts = opts;
1860         INIT_LIST_HEAD(&ctrl->list);
1861
1862         ret = nvme_rdma_parse_ipaddr(&ctrl->addr_in, opts->traddr);
1863         if (ret) {
1864                 pr_err("malformed IP address passed: %s\n", opts->traddr);
1865                 goto out_free_ctrl;
1866         }
1867
1868         if (opts->mask & NVMF_OPT_TRSVCID) {
1869                 u16 port;
1870
1871                 ret = kstrtou16(opts->trsvcid, 0, &port);
1872                 if (ret)
1873                         goto out_free_ctrl;
1874
1875                 ctrl->addr_in.sin_port = cpu_to_be16(port);
1876         } else {
1877                 ctrl->addr_in.sin_port = cpu_to_be16(NVME_RDMA_IP_PORT);
1878         }
1879
1880         ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_rdma_ctrl_ops,
1881                                 0 /* no quirks, we're perfect! */);
1882         if (ret)
1883                 goto out_free_ctrl;
1884
1885         ctrl->reconnect_delay = opts->reconnect_delay;
1886         INIT_DELAYED_WORK(&ctrl->reconnect_work,
1887                         nvme_rdma_reconnect_ctrl_work);
1888         INIT_WORK(&ctrl->err_work, nvme_rdma_error_recovery_work);
1889         INIT_WORK(&ctrl->delete_work, nvme_rdma_del_ctrl_work);
1890         INIT_WORK(&ctrl->reset_work, nvme_rdma_reset_ctrl_work);
1891         spin_lock_init(&ctrl->lock);
1892
1893         ctrl->queue_count = opts->nr_io_queues + 1; /* +1 for admin queue */
1894         ctrl->ctrl.sqsize = opts->queue_size - 1;
1895         ctrl->ctrl.kato = opts->kato;
1896
1897         ret = -ENOMEM;
1898         ctrl->queues = kcalloc(ctrl->queue_count, sizeof(*ctrl->queues),
1899                                 GFP_KERNEL);
1900         if (!ctrl->queues)
1901                 goto out_uninit_ctrl;
1902
1903         ret = nvme_rdma_configure_admin_queue(ctrl);
1904         if (ret)
1905                 goto out_kfree_queues;
1906
1907         /* sanity check icdoff */
1908         if (ctrl->ctrl.icdoff) {
1909                 dev_err(ctrl->ctrl.device, "icdoff is not supported!\n");
1910                 goto out_remove_admin_queue;
1911         }
1912
1913         /* sanity check keyed sgls */
1914         if (!(ctrl->ctrl.sgls & (1 << 20))) {
1915                 dev_err(ctrl->ctrl.device, "Mandatory keyed sgls are not support\n");
1916                 goto out_remove_admin_queue;
1917         }
1918
1919         if (opts->queue_size > ctrl->ctrl.maxcmd) {
1920                 /* warn if maxcmd is lower than queue_size */
1921                 dev_warn(ctrl->ctrl.device,
1922                         "queue_size %zu > ctrl maxcmd %u, clamping down\n",
1923                         opts->queue_size, ctrl->ctrl.maxcmd);
1924                 opts->queue_size = ctrl->ctrl.maxcmd;
1925         }
1926
1927         if (opts->nr_io_queues) {
1928                 ret = nvme_rdma_create_io_queues(ctrl);
1929                 if (ret)
1930                         goto out_remove_admin_queue;
1931         }
1932
1933         changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
1934         WARN_ON_ONCE(!changed);
1935
1936         dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp\n",
1937                 ctrl->ctrl.opts->subsysnqn, &ctrl->addr);
1938
1939         kref_get(&ctrl->ctrl.kref);
1940
1941         mutex_lock(&nvme_rdma_ctrl_mutex);
1942         list_add_tail(&ctrl->list, &nvme_rdma_ctrl_list);
1943         mutex_unlock(&nvme_rdma_ctrl_mutex);
1944
1945         if (opts->nr_io_queues) {
1946                 nvme_queue_scan(&ctrl->ctrl);
1947                 nvme_queue_async_events(&ctrl->ctrl);
1948         }
1949
1950         return &ctrl->ctrl;
1951
1952 out_remove_admin_queue:
1953         nvme_stop_keep_alive(&ctrl->ctrl);
1954         nvme_rdma_destroy_admin_queue(ctrl);
1955 out_kfree_queues:
1956         kfree(ctrl->queues);
1957 out_uninit_ctrl:
1958         nvme_uninit_ctrl(&ctrl->ctrl);
1959         nvme_put_ctrl(&ctrl->ctrl);
1960         if (ret > 0)
1961                 ret = -EIO;
1962         return ERR_PTR(ret);
1963 out_free_ctrl:
1964         kfree(ctrl);
1965         return ERR_PTR(ret);
1966 }
1967
1968 static struct nvmf_transport_ops nvme_rdma_transport = {
1969         .name           = "rdma",
1970         .required_opts  = NVMF_OPT_TRADDR,
1971         .allowed_opts   = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY,
1972         .create_ctrl    = nvme_rdma_create_ctrl,
1973 };
1974
1975 static void nvme_rdma_add_one(struct ib_device *ib_device)
1976 {
1977 }
1978
1979 static void nvme_rdma_remove_one(struct ib_device *ib_device, void *client_data)
1980 {
1981         struct nvme_rdma_ctrl *ctrl;
1982
1983         /* Delete all controllers using this device */
1984         mutex_lock(&nvme_rdma_ctrl_mutex);
1985         list_for_each_entry(ctrl, &nvme_rdma_ctrl_list, list) {
1986                 if (ctrl->device->dev != ib_device)
1987                         continue;
1988                 dev_info(ctrl->ctrl.device,
1989                         "Removing ctrl: NQN \"%s\", addr %pISp\n",
1990                         ctrl->ctrl.opts->subsysnqn, &ctrl->addr);
1991                 __nvme_rdma_del_ctrl(ctrl);
1992         }
1993         mutex_unlock(&nvme_rdma_ctrl_mutex);
1994
1995         flush_workqueue(nvme_rdma_wq);
1996 }
1997
1998 static struct ib_client nvme_rdma_ib_client = {
1999         .name   = "nvme_rdma",
2000         .add = nvme_rdma_add_one,
2001         .remove = nvme_rdma_remove_one
2002 };
2003
2004 static int __init nvme_rdma_init_module(void)
2005 {
2006         int ret;
2007
2008         nvme_rdma_wq = create_workqueue("nvme_rdma_wq");
2009         if (!nvme_rdma_wq)
2010                 return -ENOMEM;
2011
2012         ret = ib_register_client(&nvme_rdma_ib_client);
2013         if (ret) {
2014                 destroy_workqueue(nvme_rdma_wq);
2015                 return ret;
2016         }
2017
2018         nvmf_register_transport(&nvme_rdma_transport);
2019         return 0;
2020 }
2021
2022 static void __exit nvme_rdma_cleanup_module(void)
2023 {
2024         nvmf_unregister_transport(&nvme_rdma_transport);
2025         ib_unregister_client(&nvme_rdma_ib_client);
2026         destroy_workqueue(nvme_rdma_wq);
2027 }
2028
2029 module_init(nvme_rdma_init_module);
2030 module_exit(nvme_rdma_cleanup_module);
2031
2032 MODULE_LICENSE("GPL v2");