drivers/infiniband/ulp/srp/ib_srp.c

   1 /*
   2  * Copyright (c) 2005 Cisco Systems.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  34
  35 #include <linux/module.h>
  36 #include <linux/init.h>
  37 #include <linux/slab.h>
  38 #include <linux/err.h>
  39 #include <linux/string.h>
  40 #include <linux/parser.h>
  41 #include <linux/random.h>
  42 #include <linux/jiffies.h>
  43
  44 #include <linux/atomic.h>
  45
  46 #include <scsi/scsi.h>
  47 #include <scsi/scsi_device.h>
  48 #include <scsi/scsi_dbg.h>
  49 #include <scsi/scsi_tcq.h>
  50 #include <scsi/srp.h>
  51 #include <scsi/scsi_transport_srp.h>
  52
  53 #include "ib_srp.h"
  54
  55 #define DRV_NAME        "ib_srp"
  56 #define PFX             DRV_NAME ": "
  57 #define DRV_VERSION     "1.0"
  58 #define DRV_RELDATE     "July 1, 2013"
  59
  60 MODULE_AUTHOR("Roland Dreier");
  61 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator "
  62                    "v" DRV_VERSION " (" DRV_RELDATE ")");
  63 MODULE_LICENSE("Dual BSD/GPL");
  64
  65 static unsigned int srp_sg_tablesize;
  66 static unsigned int cmd_sg_entries;
  67 static unsigned int indirect_sg_entries;
  68 static bool allow_ext_sg;
  69 static bool prefer_fr;
  70 static bool register_always;
  71 static int topspin_workarounds = 1;
  72
  73 module_param(srp_sg_tablesize, uint, 0444);
  74 MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries");
  75
  76 module_param(cmd_sg_entries, uint, 0444);
  77 MODULE_PARM_DESC(cmd_sg_entries,
  78                  "Default number of gather/scatter entries in the SRP command (default is 12, max 255)");
  79
  80 module_param(indirect_sg_entries, uint, 0444);
  81 MODULE_PARM_DESC(indirect_sg_entries,
  82                  "Default max number of gather/scatter entries (default is 12, max is " __stringify(SCSI_MAX_SG_CHAIN_SEGMENTS) ")");
  83
  84 module_param(allow_ext_sg, bool, 0444);
  85 MODULE_PARM_DESC(allow_ext_sg,
  86                   "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)");
  87
  88 module_param(topspin_workarounds, int, 0444);
  89 MODULE_PARM_DESC(topspin_workarounds,
  90                  "Enable workarounds for Topspin/Cisco SRP target bugs if != 0");
  91
  92 module_param(prefer_fr, bool, 0444);
  93 MODULE_PARM_DESC(prefer_fr,
  94 "Whether to use fast registration if both FMR and fast registration are supported");
  95
  96 module_param(register_always, bool, 0444);
  97 MODULE_PARM_DESC(register_always,
  98                  "Use memory registration even for contiguous memory regions");
  99
 100 static struct kernel_param_ops srp_tmo_ops;
 101
 102 static int srp_reconnect_delay = 10;
 103 module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay,
 104                 S_IRUGO | S_IWUSR);
 105 MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts");
 106
 107 static int srp_fast_io_fail_tmo = 15;
 108 module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo,
 109                 S_IRUGO | S_IWUSR);
 110 MODULE_PARM_DESC(fast_io_fail_tmo,
 111                  "Number of seconds between the observation of a transport"
 112                  " layer error and failing all I/O. \"off\" means that this"
 113                  " functionality is disabled.");
 114
 115 static int srp_dev_loss_tmo = 600;
 116 module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo,
 117                 S_IRUGO | S_IWUSR);
 118 MODULE_PARM_DESC(dev_loss_tmo,
 119                  "Maximum number of seconds that the SRP transport should"
 120                  " insulate transport layer errors. After this time has been"
 121                  " exceeded the SCSI host is removed. Should be"
 122                  " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT)
 123                  " if fast_io_fail_tmo has not been set. \"off\" means that"
 124                  " this functionality is disabled.");
 125
 126 static void srp_add_one(struct ib_device *device);
 127 static void srp_remove_one(struct ib_device *device);
 128 static void srp_recv_completion(struct ib_cq *cq, void *target_ptr);
 129 static void srp_send_completion(struct ib_cq *cq, void *target_ptr);
 130 static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event);
 131
 132 static struct scsi_transport_template *ib_srp_transport_template;
 133 static struct workqueue_struct *srp_remove_wq;
 134
 135 static struct ib_client srp_client = {
 136         .name   = "srp",
 137         .add    = srp_add_one,
 138         .remove = srp_remove_one
 139 };
 140
 141 static struct ib_sa_client srp_sa_client;
 142
 143 static int srp_tmo_get(char *buffer, const struct kernel_param *kp)
 144 {
 145         int tmo = *(int *)kp->arg;
 146
 147         if (tmo >= 0)
 148                 return sprintf(buffer, "%d", tmo);
 149         else
 150                 return sprintf(buffer, "off");
 151 }
 152
 153 static int srp_tmo_set(const char *val, const struct kernel_param *kp)
 154 {
 155         int tmo, res;
 156
 157         if (strncmp(val, "off", 3) != 0) {
 158                 res = kstrtoint(val, 0, &tmo);
 159                 if (res)
 160                         goto out;
 161         } else {
 162                 tmo = -1;
 163         }
 164         if (kp->arg == &srp_reconnect_delay)
 165                 res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo,
 166                                     srp_dev_loss_tmo);
 167         else if (kp->arg == &srp_fast_io_fail_tmo)
 168                 res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo);
 169         else
 170                 res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo,
 171                                     tmo);
 172         if (res)
 173                 goto out;
 174         *(int *)kp->arg = tmo;
 175
 176 out:
 177         return res;
 178 }
 179
 180 static struct kernel_param_ops srp_tmo_ops = {
 181         .get = srp_tmo_get,
 182         .set = srp_tmo_set,
 183 };
 184
 185 static inline struct srp_target_port *host_to_target(struct Scsi_Host *host)
 186 {
 187         return (struct srp_target_port *) host->hostdata;
 188 }
 189
 190 static const char *srp_target_info(struct Scsi_Host *host)
 191 {
 192         return host_to_target(host)->target_name;
 193 }
 194
 195 static int srp_target_is_topspin(struct srp_target_port *target)
 196 {
 197         static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad };
 198         static const u8 cisco_oui[3]   = { 0x00, 0x1b, 0x0d };
 199
 200         return topspin_workarounds &&
 201                 (!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) ||
 202                  !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui));
 203 }
 204
 205 static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size,
 206                                    gfp_t gfp_mask,
 207                                    enum dma_data_direction direction)
 208 {
 209         struct srp_iu *iu;
 210
 211         iu = kmalloc(sizeof *iu, gfp_mask);
 212         if (!iu)
 213                 goto out;
 214
 215         iu->buf = kzalloc(size, gfp_mask);
 216         if (!iu->buf)
 217                 goto out_free_iu;
 218
 219         iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size,
 220                                     direction);
 221         if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma))
 222                 goto out_free_buf;
 223
 224         iu->size      = size;
 225         iu->direction = direction;
 226
 227         return iu;
 228
 229 out_free_buf:
 230         kfree(iu->buf);
 231 out_free_iu:
 232         kfree(iu);
 233 out:
 234         return NULL;
 235 }
 236
 237 static void srp_free_iu(struct srp_host *host, struct srp_iu *iu)
 238 {
 239         if (!iu)
 240                 return;
 241
 242         ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size,
 243                             iu->direction);
 244         kfree(iu->buf);
 245         kfree(iu);
 246 }
 247
 248 static void srp_qp_event(struct ib_event *event, void *context)
 249 {
 250         pr_debug("QP event %d\n", event->event);
 251 }
 252
 253 static int srp_init_qp(struct srp_target_port *target,
 254                        struct ib_qp *qp)
 255 {
 256         struct ib_qp_attr *attr;
 257         int ret;
 258
 259         attr = kmalloc(sizeof *attr, GFP_KERNEL);
 260         if (!attr)
 261                 return -ENOMEM;
 262
 263         ret = ib_find_pkey(target->srp_host->srp_dev->dev,
 264                            target->srp_host->port,
 265                            be16_to_cpu(target->path.pkey),
 266                            &attr->pkey_index);
 267         if (ret)
 268                 goto out;
 269
 270         attr->qp_state        = IB_QPS_INIT;
 271         attr->qp_access_flags = (IB_ACCESS_REMOTE_READ |
 272                                     IB_ACCESS_REMOTE_WRITE);
 273         attr->port_num        = target->srp_host->port;
 274
 275         ret = ib_modify_qp(qp, attr,
 276                            IB_QP_STATE          |
 277                            IB_QP_PKEY_INDEX     |
 278                            IB_QP_ACCESS_FLAGS   |
 279                            IB_QP_PORT);
 280
 281 out:
 282         kfree(attr);
 283         return ret;
 284 }
 285
 286 static int srp_new_cm_id(struct srp_target_port *target)
 287 {
 288         struct ib_cm_id *new_cm_id;
 289
 290         new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev,
 291                                     srp_cm_handler, target);
 292         if (IS_ERR(new_cm_id))
 293                 return PTR_ERR(new_cm_id);
 294
 295         if (target->cm_id)
 296                 ib_destroy_cm_id(target->cm_id);
 297         target->cm_id = new_cm_id;
 298
 299         return 0;
 300 }
 301
 302 static struct ib_fmr_pool *srp_alloc_fmr_pool(struct srp_target_port *target)
 303 {
 304         struct srp_device *dev = target->srp_host->srp_dev;
 305         struct ib_fmr_pool_param fmr_param;
 306
 307         memset(&fmr_param, 0, sizeof(fmr_param));
 308         fmr_param.pool_size         = target->scsi_host->can_queue;
 309         fmr_param.dirty_watermark   = fmr_param.pool_size / 4;
 310         fmr_param.cache             = 1;
 311         fmr_param.max_pages_per_fmr = dev->max_pages_per_mr;
 312         fmr_param.page_shift        = ilog2(dev->mr_page_size);
 313         fmr_param.access            = (IB_ACCESS_LOCAL_WRITE |
 314                                        IB_ACCESS_REMOTE_WRITE |
 315                                        IB_ACCESS_REMOTE_READ);
 316
 317         return ib_create_fmr_pool(dev->pd, &fmr_param);
 318 }
 319
 320 /**
 321  * srp_destroy_fr_pool() - free the resources owned by a pool
 322  * @pool: Fast registration pool to be destroyed.
 323  */
 324 static void srp_destroy_fr_pool(struct srp_fr_pool *pool)
 325 {
 326         int i;
 327         struct srp_fr_desc *d;
 328
 329         if (!pool)
 330                 return;
 331
 332         for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
 333                 if (d->frpl)
 334                         ib_free_fast_reg_page_list(d->frpl);
 335                 if (d->mr)
 336                         ib_dereg_mr(d->mr);
 337         }
 338         kfree(pool);
 339 }
 340
 341 /**
 342  * srp_create_fr_pool() - allocate and initialize a pool for fast registration
 343  * @device:            IB device to allocate fast registration descriptors for.
 344  * @pd:                Protection domain associated with the FR descriptors.
 345  * @pool_size:         Number of descriptors to allocate.
 346  * @max_page_list_len: Maximum fast registration work request page list length.
 347  */
 348 static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device,
 349                                               struct ib_pd *pd, int pool_size,
 350                                               int max_page_list_len)
 351 {
 352         struct srp_fr_pool *pool;
 353         struct srp_fr_desc *d;
 354         struct ib_mr *mr;
 355         struct ib_fast_reg_page_list *frpl;
 356         int i, ret = -EINVAL;
 357
 358         if (pool_size <= 0)
 359                 goto err;
 360         ret = -ENOMEM;
 361         pool = kzalloc(sizeof(struct srp_fr_pool) +
 362                        pool_size * sizeof(struct srp_fr_desc), GFP_KERNEL);
 363         if (!pool)
 364                 goto err;
 365         pool->size = pool_size;
 366         pool->max_page_list_len = max_page_list_len;
 367         spin_lock_init(&pool->lock);
 368         INIT_LIST_HEAD(&pool->free_list);
 369
 370         for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
 371                 mr = ib_alloc_fast_reg_mr(pd, max_page_list_len);
 372                 if (IS_ERR(mr)) {
 373                         ret = PTR_ERR(mr);
 374                         goto destroy_pool;
 375                 }
 376                 d->mr = mr;
 377                 frpl = ib_alloc_fast_reg_page_list(device, max_page_list_len);
 378                 if (IS_ERR(frpl)) {
 379                         ret = PTR_ERR(frpl);
 380                         goto destroy_pool;
 381                 }
 382                 d->frpl = frpl;
 383                 list_add_tail(&d->entry, &pool->free_list);
 384         }
 385
 386 out:
 387         return pool;
 388
 389 destroy_pool:
 390         srp_destroy_fr_pool(pool);
 391
 392 err:
 393         pool = ERR_PTR(ret);
 394         goto out;
 395 }
 396
 397 /**
 398  * srp_fr_pool_get() - obtain a descriptor suitable for fast registration
 399  * @pool: Pool to obtain descriptor from.
 400  */
 401 static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool)
 402 {
 403         struct srp_fr_desc *d = NULL;
 404         unsigned long flags;
 405
 406         spin_lock_irqsave(&pool->lock, flags);
 407         if (!list_empty(&pool->free_list)) {
 408                 d = list_first_entry(&pool->free_list, typeof(*d), entry);
 409                 list_del(&d->entry);
 410         }
 411         spin_unlock_irqrestore(&pool->lock, flags);
 412
 413         return d;
 414 }
 415
 416 /**
 417  * srp_fr_pool_put() - put an FR descriptor back in the free list
 418  * @pool: Pool the descriptor was allocated from.
 419  * @desc: Pointer to an array of fast registration descriptor pointers.
 420  * @n:    Number of descriptors to put back.
 421  *
 422  * Note: The caller must already have queued an invalidation request for
 423  * desc->mr->rkey before calling this function.
 424  */
 425 static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc,
 426                             int n)
 427 {
 428         unsigned long flags;
 429         int i;
 430
 431         spin_lock_irqsave(&pool->lock, flags);
 432         for (i = 0; i < n; i++)
 433                 list_add(&desc[i]->entry, &pool->free_list);
 434         spin_unlock_irqrestore(&pool->lock, flags);
 435 }
 436
 437 static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target)
 438 {
 439         struct srp_device *dev = target->srp_host->srp_dev;
 440
 441         return srp_create_fr_pool(dev->dev, dev->pd,
 442                                   target->scsi_host->can_queue,
 443                                   dev->max_pages_per_mr);
 444 }
 445
 446 static int srp_create_target_ib(struct srp_target_port *target)
 447 {
 448         struct srp_device *dev = target->srp_host->srp_dev;
 449         struct ib_qp_init_attr *init_attr;
 450         struct ib_cq *recv_cq, *send_cq;
 451         struct ib_qp *qp;
 452         struct ib_fmr_pool *fmr_pool = NULL;
 453         struct srp_fr_pool *fr_pool = NULL;
 454         const int m = 1 + dev->use_fast_reg;
 455         int ret;
 456
 457         init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL);
 458         if (!init_attr)
 459                 return -ENOMEM;
 460
 461         recv_cq = ib_create_cq(dev->dev, srp_recv_completion, NULL, target,
 462                                target->queue_size, target->comp_vector);
 463         if (IS_ERR(recv_cq)) {
 464                 ret = PTR_ERR(recv_cq);
 465                 goto err;
 466         }
 467
 468         send_cq = ib_create_cq(dev->dev, srp_send_completion, NULL, target,
 469                                m * target->queue_size, target->comp_vector);
 470         if (IS_ERR(send_cq)) {
 471                 ret = PTR_ERR(send_cq);
 472                 goto err_recv_cq;
 473         }
 474
 475         ib_req_notify_cq(recv_cq, IB_CQ_NEXT_COMP);
 476
 477         init_attr->event_handler       = srp_qp_event;
 478         init_attr->cap.max_send_wr     = m * target->queue_size;
 479         init_attr->cap.max_recv_wr     = target->queue_size;
 480         init_attr->cap.max_recv_sge    = 1;
 481         init_attr->cap.max_send_sge    = 1;
 482         init_attr->sq_sig_type         = IB_SIGNAL_REQ_WR;
 483         init_attr->qp_type             = IB_QPT_RC;
 484         init_attr->send_cq             = send_cq;
 485         init_attr->recv_cq             = recv_cq;
 486
 487         qp = ib_create_qp(dev->pd, init_attr);
 488         if (IS_ERR(qp)) {
 489                 ret = PTR_ERR(qp);
 490                 goto err_send_cq;
 491         }
 492
 493         ret = srp_init_qp(target, qp);
 494         if (ret)
 495                 goto err_qp;
 496
 497         if (dev->use_fast_reg && dev->has_fr) {
 498                 fr_pool = srp_alloc_fr_pool(target);
 499                 if (IS_ERR(fr_pool)) {
 500                         ret = PTR_ERR(fr_pool);
 501                         shost_printk(KERN_WARNING, target->scsi_host, PFX
 502                                      "FR pool allocation failed (%d)\n", ret);
 503                         goto err_qp;
 504                 }
 505                 if (target->fr_pool)
 506                         srp_destroy_fr_pool(target->fr_pool);
 507                 target->fr_pool = fr_pool;
 508         } else if (!dev->use_fast_reg && dev->has_fmr) {
 509                 fmr_pool = srp_alloc_fmr_pool(target);
 510                 if (IS_ERR(fmr_pool)) {
 511                         ret = PTR_ERR(fmr_pool);
 512                         shost_printk(KERN_WARNING, target->scsi_host, PFX
 513                                      "FMR pool allocation failed (%d)\n", ret);
 514                         goto err_qp;
 515                 }
 516                 if (target->fmr_pool)
 517                         ib_destroy_fmr_pool(target->fmr_pool);
 518                 target->fmr_pool = fmr_pool;
 519         }
 520
 521         if (target->qp)
 522                 ib_destroy_qp(target->qp);
 523         if (target->recv_cq)
 524                 ib_destroy_cq(target->recv_cq);
 525         if (target->send_cq)
 526                 ib_destroy_cq(target->send_cq);
 527
 528         target->qp = qp;
 529         target->recv_cq = recv_cq;
 530         target->send_cq = send_cq;
 531
 532         kfree(init_attr);
 533         return 0;
 534
 535 err_qp:
 536         ib_destroy_qp(qp);
 537
 538 err_send_cq:
 539         ib_destroy_cq(send_cq);
 540
 541 err_recv_cq:
 542         ib_destroy_cq(recv_cq);
 543
 544 err:
 545         kfree(init_attr);
 546         return ret;
 547 }
 548
 549 /*
 550  * Note: this function may be called without srp_alloc_iu_bufs() having been
 551  * invoked. Hence the target->[rt]x_ring checks.
 552  */
 553 static void srp_free_target_ib(struct srp_target_port *target)
 554 {
 555         struct srp_device *dev = target->srp_host->srp_dev;
 556         int i;
 557
 558         if (dev->use_fast_reg) {
 559                 if (target->fr_pool)
 560                         srp_destroy_fr_pool(target->fr_pool);
 561         } else {
 562                 if (target->fmr_pool)
 563                         ib_destroy_fmr_pool(target->fmr_pool);
 564         }
 565         ib_destroy_qp(target->qp);
 566         ib_destroy_cq(target->send_cq);
 567         ib_destroy_cq(target->recv_cq);
 568
 569         target->qp = NULL;
 570         target->send_cq = target->recv_cq = NULL;
 571
 572         if (target->rx_ring) {
 573                 for (i = 0; i < target->queue_size; ++i)
 574                         srp_free_iu(target->srp_host, target->rx_ring[i]);
 575                 kfree(target->rx_ring);
 576                 target->rx_ring = NULL;
 577         }
 578         if (target->tx_ring) {
 579                 for (i = 0; i < target->queue_size; ++i)
 580                         srp_free_iu(target->srp_host, target->tx_ring[i]);
 581                 kfree(target->tx_ring);
 582                 target->tx_ring = NULL;
 583         }
 584 }
 585
 586 static void srp_path_rec_completion(int status,
 587                                     struct ib_sa_path_rec *pathrec,
 588                                     void *target_ptr)
 589 {
 590         struct srp_target_port *target = target_ptr;
 591
 592         target->status = status;
 593         if (status)
 594                 shost_printk(KERN_ERR, target->scsi_host,
 595                              PFX "Got failed path rec status %d\n", status);
 596         else
 597                 target->path = *pathrec;
 598         complete(&target->done);
 599 }
 600
 601 static int srp_lookup_path(struct srp_target_port *target)
 602 {
 603         int ret;
 604
 605         target->path.numb_path = 1;
 606
 607         init_completion(&target->done);
 608
 609         target->path_query_id = ib_sa_path_rec_get(&srp_sa_client,
 610                                                    target->srp_host->srp_dev->dev,
 611                                                    target->srp_host->port,
 612                                                    &target->path,
 613                                                    IB_SA_PATH_REC_SERVICE_ID    |
 614                                                    IB_SA_PATH_REC_DGID          |
 615                                                    IB_SA_PATH_REC_SGID          |
 616                                                    IB_SA_PATH_REC_NUMB_PATH     |
 617                                                    IB_SA_PATH_REC_PKEY,
 618                                                    SRP_PATH_REC_TIMEOUT_MS,
 619                                                    GFP_KERNEL,
 620                                                    srp_path_rec_completion,
 621                                                    target, &target->path_query);
 622         if (target->path_query_id < 0)
 623                 return target->path_query_id;
 624
 625         ret = wait_for_completion_interruptible(&target->done);
 626         if (ret < 0)
 627                 return ret;
 628
 629         if (target->status < 0)
 630                 shost_printk(KERN_WARNING, target->scsi_host,
 631                              PFX "Path record query failed\n");
 632
 633         return target->status;
 634 }
 635
 636 static int srp_send_req(struct srp_target_port *target)
 637 {
 638         struct {
 639                 struct ib_cm_req_param param;
 640                 struct srp_login_req   priv;
 641         } *req = NULL;
 642         int status;
 643
 644         req = kzalloc(sizeof *req, GFP_KERNEL);
 645         if (!req)
 646                 return -ENOMEM;
 647
 648         req->param.primary_path               = &target->path;
 649         req->param.alternate_path             = NULL;
 650         req->param.service_id                 = target->service_id;
 651         req->param.qp_num                     = target->qp->qp_num;
 652         req->param.qp_type                    = target->qp->qp_type;
 653         req->param.private_data               = &req->priv;
 654         req->param.private_data_len           = sizeof req->priv;
 655         req->param.flow_control               = 1;
 656
 657         get_random_bytes(&req->param.starting_psn, 4);
 658         req->param.starting_psn              &= 0xffffff;
 659
 660         /*
 661          * Pick some arbitrary defaults here; we could make these
 662          * module parameters if anyone cared about setting them.
 663          */
 664         req->param.responder_resources        = 4;
 665         req->param.remote_cm_response_timeout = 20;
 666         req->param.local_cm_response_timeout  = 20;
 667         req->param.retry_count                = target->tl_retry_count;
 668         req->param.rnr_retry_count            = 7;
 669         req->param.max_cm_retries             = 15;
 670
 671         req->priv.opcode        = SRP_LOGIN_REQ;
 672         req->priv.tag           = 0;
 673         req->priv.req_it_iu_len = cpu_to_be32(target->max_iu_len);
 674         req->priv.req_buf_fmt   = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
 675                                               SRP_BUF_FORMAT_INDIRECT);
 676         /*
 677          * In the published SRP specification (draft rev. 16a), the
 678          * port identifier format is 8 bytes of ID extension followed
 679          * by 8 bytes of GUID.  Older drafts put the two halves in the
 680          * opposite order, so that the GUID comes first.
 681          *
 682          * Targets conforming to these obsolete drafts can be
 683          * recognized by the I/O Class they report.
 684          */
 685         if (target->io_class == SRP_REV10_IB_IO_CLASS) {
 686                 memcpy(req->priv.initiator_port_id,
 687                        &target->path.sgid.global.interface_id, 8);
 688                 memcpy(req->priv.initiator_port_id + 8,
 689                        &target->initiator_ext, 8);
 690                 memcpy(req->priv.target_port_id,     &target->ioc_guid, 8);
 691                 memcpy(req->priv.target_port_id + 8, &target->id_ext, 8);
 692         } else {
 693                 memcpy(req->priv.initiator_port_id,
 694                        &target->initiator_ext, 8);
 695                 memcpy(req->priv.initiator_port_id + 8,
 696                        &target->path.sgid.global.interface_id, 8);
 697                 memcpy(req->priv.target_port_id,     &target->id_ext, 8);
 698                 memcpy(req->priv.target_port_id + 8, &target->ioc_guid, 8);
 699         }
 700
 701         /*
 702          * Topspin/Cisco SRP targets will reject our login unless we
 703          * zero out the first 8 bytes of our initiator port ID and set
 704          * the second 8 bytes to the local node GUID.
 705          */
 706         if (srp_target_is_topspin(target)) {
 707                 shost_printk(KERN_DEBUG, target->scsi_host,
 708                              PFX "Topspin/Cisco initiator port ID workaround "
 709                              "activated for target GUID %016llx\n",
 710                              (unsigned long long) be64_to_cpu(target->ioc_guid));
 711                 memset(req->priv.initiator_port_id, 0, 8);
 712                 memcpy(req->priv.initiator_port_id + 8,
 713                        &target->srp_host->srp_dev->dev->node_guid, 8);
 714         }
 715
 716         status = ib_send_cm_req(target->cm_id, &req->param);
 717
 718         kfree(req);
 719
 720         return status;
 721 }
 722
 723 static bool srp_queue_remove_work(struct srp_target_port *target)
 724 {
 725         bool changed = false;
 726
 727         spin_lock_irq(&target->lock);
 728         if (target->state != SRP_TARGET_REMOVED) {
 729                 target->state = SRP_TARGET_REMOVED;
 730                 changed = true;
 731         }
 732         spin_unlock_irq(&target->lock);
 733
 734         if (changed)
 735                 queue_work(srp_remove_wq, &target->remove_work);
 736
 737         return changed;
 738 }
 739
 740 static bool srp_change_conn_state(struct srp_target_port *target,
 741                                   bool connected)
 742 {
 743         bool changed = false;
 744
 745         spin_lock_irq(&target->lock);
 746         if (target->connected != connected) {
 747                 target->connected = connected;
 748                 changed = true;
 749         }
 750         spin_unlock_irq(&target->lock);
 751
 752         return changed;
 753 }
 754
 755 static void srp_disconnect_target(struct srp_target_port *target)
 756 {
 757         if (srp_change_conn_state(target, false)) {
 758                 /* XXX should send SRP_I_LOGOUT request */
 759
 760                 if (ib_send_cm_dreq(target->cm_id, NULL, 0)) {
 761                         shost_printk(KERN_DEBUG, target->scsi_host,
 762                                      PFX "Sending CM DREQ failed\n");
 763                 }
 764         }
 765 }
 766
 767 static void srp_free_req_data(struct srp_target_port *target)
 768 {
 769         struct srp_device *dev = target->srp_host->srp_dev;
 770         struct ib_device *ibdev = dev->dev;
 771         struct srp_request *req;
 772         int i;
 773
 774         if (!target->req_ring)
 775                 return;
 776
 777         for (i = 0; i < target->req_ring_size; ++i) {
 778                 req = &target->req_ring[i];
 779                 if (dev->use_fast_reg)
 780                         kfree(req->fr_list);
 781                 else
 782                         kfree(req->fmr_list);
 783                 kfree(req->map_page);
 784                 if (req->indirect_dma_addr) {
 785                         ib_dma_unmap_single(ibdev, req->indirect_dma_addr,
 786                                             target->indirect_size,
 787                                             DMA_TO_DEVICE);
 788                 }
 789                 kfree(req->indirect_desc);
 790         }
 791
 792         kfree(target->req_ring);
 793         target->req_ring = NULL;
 794 }
 795
 796 static int srp_alloc_req_data(struct srp_target_port *target)
 797 {
 798         struct srp_device *srp_dev = target->srp_host->srp_dev;
 799         struct ib_device *ibdev = srp_dev->dev;
 800         struct srp_request *req;
 801         void *mr_list;
 802         dma_addr_t dma_addr;
 803         int i, ret = -ENOMEM;
 804
 805         INIT_LIST_HEAD(&target->free_reqs);
 806
 807         target->req_ring = kzalloc(target->req_ring_size *
 808                                    sizeof(*target->req_ring), GFP_KERNEL);
 809         if (!target->req_ring)
 810                 goto out;
 811
 812         for (i = 0; i < target->req_ring_size; ++i) {
 813                 req = &target->req_ring[i];
 814                 mr_list = kmalloc(target->cmd_sg_cnt * sizeof(void *),
 815                                   GFP_KERNEL);
 816                 if (!mr_list)
 817                         goto out;
 818                 if (srp_dev->use_fast_reg)
 819                         req->fr_list = mr_list;
 820                 else
 821                         req->fmr_list = mr_list;
 822                 req->map_page = kmalloc(srp_dev->max_pages_per_mr *
 823                                         sizeof(void *), GFP_KERNEL);
 824                 if (!req->map_page)
 825                         goto out;
 826                 req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL);
 827                 if (!req->indirect_desc)
 828                         goto out;
 829
 830                 dma_addr = ib_dma_map_single(ibdev, req->indirect_desc,
 831                                              target->indirect_size,
 832                                              DMA_TO_DEVICE);
 833                 if (ib_dma_mapping_error(ibdev, dma_addr))
 834                         goto out;
 835
 836                 req->indirect_dma_addr = dma_addr;
 837                 req->index = i;
 838                 list_add_tail(&req->list, &target->free_reqs);
 839         }
 840         ret = 0;
 841
 842 out:
 843         return ret;
 844 }
 845
 846 /**
 847  * srp_del_scsi_host_attr() - Remove attributes defined in the host template.
 848  * @shost: SCSI host whose attributes to remove from sysfs.
 849  *
 850  * Note: Any attributes defined in the host template and that did not exist
 851  * before invocation of this function will be ignored.
 852  */
 853 static void srp_del_scsi_host_attr(struct Scsi_Host *shost)
 854 {
 855         struct device_attribute **attr;
 856
 857         for (attr = shost->hostt->shost_attrs; attr && *attr; ++attr)
 858                 device_remove_file(&shost->shost_dev, *attr);
 859 }
 860
 861 static void srp_remove_target(struct srp_target_port *target)
 862 {
 863         WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
 864
 865         srp_del_scsi_host_attr(target->scsi_host);
 866         srp_rport_get(target->rport);
 867         srp_remove_host(target->scsi_host);
 868         scsi_remove_host(target->scsi_host);
 869         srp_stop_rport_timers(target->rport);
 870         srp_disconnect_target(target);
 871         ib_destroy_cm_id(target->cm_id);
 872         srp_free_target_ib(target);
 873         cancel_work_sync(&target->tl_err_work);
 874         srp_rport_put(target->rport);
 875         srp_free_req_data(target);
 876
 877         spin_lock(&target->srp_host->target_lock);
 878         list_del(&target->list);
 879         spin_unlock(&target->srp_host->target_lock);
 880
 881         scsi_host_put(target->scsi_host);
 882 }
 883
 884 static void srp_remove_work(struct work_struct *work)
 885 {
 886         struct srp_target_port *target =
 887                 container_of(work, struct srp_target_port, remove_work);
 888
 889         WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
 890
 891         srp_remove_target(target);
 892 }
 893
 894 static void srp_rport_delete(struct srp_rport *rport)
 895 {
 896         struct srp_target_port *target = rport->lld_data;
 897
 898         srp_queue_remove_work(target);
 899 }
 900
 901 static int srp_connect_target(struct srp_target_port *target)
 902 {
 903         int retries = 3;
 904         int ret;
 905
 906         WARN_ON_ONCE(target->connected);
 907
 908         target->qp_in_error = false;
 909
 910         ret = srp_lookup_path(target);
 911         if (ret)
 912                 return ret;
 913
 914         while (1) {
 915                 init_completion(&target->done);
 916                 ret = srp_send_req(target);
 917                 if (ret)
 918                         return ret;
 919                 ret = wait_for_completion_interruptible(&target->done);
 920                 if (ret < 0)
 921                         return ret;
 922
 923                 /*
 924                  * The CM event handling code will set status to
 925                  * SRP_PORT_REDIRECT if we get a port redirect REJ
 926                  * back, or SRP_DLID_REDIRECT if we get a lid/qp
 927                  * redirect REJ back.
 928                  */
 929                 switch (target->status) {
 930                 case 0:
 931                         srp_change_conn_state(target, true);
 932                         return 0;
 933
 934                 case SRP_PORT_REDIRECT:
 935                         ret = srp_lookup_path(target);
 936                         if (ret)
 937                                 return ret;
 938                         break;
 939
 940                 case SRP_DLID_REDIRECT:
 941                         break;
 942
 943                 case SRP_STALE_CONN:
 944                         /* Our current CM id was stale, and is now in timewait.
 945                          * Try to reconnect with a new one.
 946                          */
 947                         if (!retries-- || srp_new_cm_id(target)) {
 948                                 shost_printk(KERN_ERR, target->scsi_host, PFX
 949                                              "giving up on stale connection\n");
 950                                 target->status = -ECONNRESET;
 951                                 return target->status;
 952                         }
 953
 954                         shost_printk(KERN_ERR, target->scsi_host, PFX
 955                                      "retrying stale connection\n");
 956                         break;
 957
 958                 default:
 959                         return target->status;
 960                 }
 961         }
 962 }
 963
 964 static int srp_inv_rkey(struct srp_target_port *target, u32 rkey)
 965 {
 966         struct ib_send_wr *bad_wr;
 967         struct ib_send_wr wr = {
 968                 .opcode             = IB_WR_LOCAL_INV,
 969                 .wr_id              = LOCAL_INV_WR_ID_MASK,
 970                 .next               = NULL,
 971                 .num_sge            = 0,
 972                 .send_flags         = 0,
 973                 .ex.invalidate_rkey = rkey,
 974         };
 975
 976         return ib_post_send(target->qp, &wr, &bad_wr);
 977 }
 978
 979 static void srp_unmap_data(struct scsi_cmnd *scmnd,
 980                            struct srp_target_port *target,
 981                            struct srp_request *req)
 982 {
 983         struct srp_device *dev = target->srp_host->srp_dev;
 984         struct ib_device *ibdev = dev->dev;
 985         int i, res;
 986
 987         if (!scsi_sglist(scmnd) ||
 988             (scmnd->sc_data_direction != DMA_TO_DEVICE &&
 989              scmnd->sc_data_direction != DMA_FROM_DEVICE))
 990                 return;
 991
 992         if (dev->use_fast_reg) {
 993                 struct srp_fr_desc **pfr;
 994
 995                 for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) {
 996                         res = srp_inv_rkey(target, (*pfr)->mr->rkey);
 997                         if (res < 0) {
 998                                 shost_printk(KERN_ERR, target->scsi_host, PFX
 999                                   "Queueing INV WR for rkey %#x failed (%d)\n",
1000                                   (*pfr)->mr->rkey, res);
1001                                 queue_work(system_long_wq,
1002                                            &target->tl_err_work);
1003                         }
1004                 }
1005                 if (req->nmdesc)
1006                         srp_fr_pool_put(target->fr_pool, req->fr_list,
1007                                         req->nmdesc);
1008         } else {
1009                 struct ib_pool_fmr **pfmr;
1010
1011                 for (i = req->nmdesc, pfmr = req->fmr_list; i > 0; i--, pfmr++)
1012                         ib_fmr_pool_unmap(*pfmr);
1013         }
1014
1015         ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd),
1016                         scmnd->sc_data_direction);
1017 }
1018
1019 /**
1020  * srp_claim_req - Take ownership of the scmnd associated with a request.
1021  * @target: SRP target port.
1022  * @req: SRP request.
1023  * @sdev: If not NULL, only take ownership for this SCSI device.
1024  * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take
1025  *         ownership of @req->scmnd if it equals @scmnd.
1026  *
1027  * Return value:
1028  * Either NULL or a pointer to the SCSI command the caller became owner of.
1029  */
1030 static struct scsi_cmnd *srp_claim_req(struct srp_target_port *target,
1031                                        struct srp_request *req,
1032                                        struct scsi_device *sdev,
1033                                        struct scsi_cmnd *scmnd)
1034 {
1035         unsigned long flags;
1036
1037         spin_lock_irqsave(&target->lock, flags);
1038         if (req->scmnd &&
1039             (!sdev || req->scmnd->device == sdev) &&
1040             (!scmnd || req->scmnd == scmnd)) {
1041                 scmnd = req->scmnd;
1042                 req->scmnd = NULL;
1043         } else {
1044                 scmnd = NULL;
1045         }
1046         spin_unlock_irqrestore(&target->lock, flags);
1047
1048         return scmnd;
1049 }
1050
1051 /**
1052  * srp_free_req() - Unmap data and add request to the free request list.
1053  * @target: SRP target port.
1054  * @req:    Request to be freed.
1055  * @scmnd:  SCSI command associated with @req.
1056  * @req_lim_delta: Amount to be added to @target->req_lim.
1057  */
1058 static void srp_free_req(struct srp_target_port *target,
1059                          struct srp_request *req, struct scsi_cmnd *scmnd,
1060                          s32 req_lim_delta)
1061 {
1062         unsigned long flags;
1063
1064         srp_unmap_data(scmnd, target, req);
1065
1066         spin_lock_irqsave(&target->lock, flags);
1067         target->req_lim += req_lim_delta;
1068         list_add_tail(&req->list, &target->free_reqs);
1069         spin_unlock_irqrestore(&target->lock, flags);
1070 }
1071
1072 static void srp_finish_req(struct srp_target_port *target,
1073                            struct srp_request *req, struct scsi_device *sdev,
1074                            int result)
1075 {
1076         struct scsi_cmnd *scmnd = srp_claim_req(target, req, sdev, NULL);
1077
1078         if (scmnd) {
1079                 srp_free_req(target, req, scmnd, 0);
1080                 scmnd->result = result;
1081                 scmnd->scsi_done(scmnd);
1082         }
1083 }
1084
1085 static void srp_terminate_io(struct srp_rport *rport)
1086 {
1087         struct srp_target_port *target = rport->lld_data;
1088         struct Scsi_Host *shost = target->scsi_host;
1089         struct scsi_device *sdev;
1090         int i;
1091
1092         /*
1093          * Invoking srp_terminate_io() while srp_queuecommand() is running
1094          * is not safe. Hence the warning statement below.
1095          */
1096         shost_for_each_device(sdev, shost)
1097                 WARN_ON_ONCE(sdev->request_queue->request_fn_active);
1098
1099         for (i = 0; i < target->req_ring_size; ++i) {
1100                 struct srp_request *req = &target->req_ring[i];
1101                 srp_finish_req(target, req, NULL, DID_TRANSPORT_FAILFAST << 16);
1102         }
1103 }
1104
1105 /*
1106  * It is up to the caller to ensure that srp_rport_reconnect() calls are
1107  * serialized and that no concurrent srp_queuecommand(), srp_abort(),
1108  * srp_reset_device() or srp_reset_host() calls will occur while this function
1109  * is in progress. One way to realize that is not to call this function
1110  * directly but to call srp_reconnect_rport() instead since that last function
1111  * serializes calls of this function via rport->mutex and also blocks
1112  * srp_queuecommand() calls before invoking this function.
1113  */
1114 static int srp_rport_reconnect(struct srp_rport *rport)
1115 {
1116         struct srp_target_port *target = rport->lld_data;
1117         int i, ret;
1118
1119         srp_disconnect_target(target);
1120         /*
1121          * Now get a new local CM ID so that we avoid confusing the target in
1122          * case things are really fouled up. Doing so also ensures that all CM
1123          * callbacks will have finished before a new QP is allocated.
1124          */
1125         ret = srp_new_cm_id(target);
1126
1127         for (i = 0; i < target->req_ring_size; ++i) {
1128                 struct srp_request *req = &target->req_ring[i];
1129                 srp_finish_req(target, req, NULL, DID_RESET << 16);
1130         }
1131
1132         /*
1133          * Whether or not creating a new CM ID succeeded, create a new
1134          * QP. This guarantees that all callback functions for the old QP have
1135          * finished before any send requests are posted on the new QP.
1136          */
1137         ret += srp_create_target_ib(target);
1138
1139         INIT_LIST_HEAD(&target->free_tx);
1140         for (i = 0; i < target->queue_size; ++i)
1141                 list_add(&target->tx_ring[i]->list, &target->free_tx);
1142
1143         if (ret == 0)
1144                 ret = srp_connect_target(target);
1145
1146         if (ret == 0)
1147                 shost_printk(KERN_INFO, target->scsi_host,
1148                              PFX "reconnect succeeded\n");
1149
1150         return ret;
1151 }
1152
1153 static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr,
1154                          unsigned int dma_len, u32 rkey)
1155 {
1156         struct srp_direct_buf *desc = state->desc;
1157
1158         desc->va = cpu_to_be64(dma_addr);
1159         desc->key = cpu_to_be32(rkey);
1160         desc->len = cpu_to_be32(dma_len);
1161
1162         state->total_len += dma_len;
1163         state->desc++;
1164         state->ndesc++;
1165 }
1166
1167 static int srp_map_finish_fmr(struct srp_map_state *state,
1168                               struct srp_target_port *target)
1169 {
1170         struct ib_pool_fmr *fmr;
1171         u64 io_addr = 0;
1172
1173         fmr = ib_fmr_pool_map_phys(target->fmr_pool, state->pages,
1174                                    state->npages, io_addr);
1175         if (IS_ERR(fmr))
1176                 return PTR_ERR(fmr);
1177
1178         *state->next_fmr++ = fmr;
1179         state->nmdesc++;
1180
1181         srp_map_desc(state, 0, state->dma_len, fmr->fmr->rkey);
1182
1183         return 0;
1184 }
1185
1186 static int srp_map_finish_fr(struct srp_map_state *state,
1187                              struct srp_target_port *target)
1188 {
1189         struct srp_device *dev = target->srp_host->srp_dev;
1190         struct ib_send_wr *bad_wr;
1191         struct ib_send_wr wr;
1192         struct srp_fr_desc *desc;
1193         u32 rkey;
1194
1195         desc = srp_fr_pool_get(target->fr_pool);
1196         if (!desc)
1197                 return -ENOMEM;
1198
1199         rkey = ib_inc_rkey(desc->mr->rkey);
1200         ib_update_fast_reg_key(desc->mr, rkey);
1201
1202         memcpy(desc->frpl->page_list, state->pages,
1203                sizeof(state->pages[0]) * state->npages);
1204
1205         memset(&wr, 0, sizeof(wr));
1206         wr.opcode = IB_WR_FAST_REG_MR;
1207         wr.wr_id = FAST_REG_WR_ID_MASK;
1208         wr.wr.fast_reg.iova_start = state->base_dma_addr;
1209         wr.wr.fast_reg.page_list = desc->frpl;
1210         wr.wr.fast_reg.page_list_len = state->npages;
1211         wr.wr.fast_reg.page_shift = ilog2(dev->mr_page_size);
1212         wr.wr.fast_reg.length = state->dma_len;
1213         wr.wr.fast_reg.access_flags = (IB_ACCESS_LOCAL_WRITE |
1214                                        IB_ACCESS_REMOTE_READ |
1215                                        IB_ACCESS_REMOTE_WRITE);
1216         wr.wr.fast_reg.rkey = desc->mr->lkey;
1217
1218         *state->next_fr++ = desc;
1219         state->nmdesc++;
1220
1221         srp_map_desc(state, state->base_dma_addr, state->dma_len,
1222                      desc->mr->rkey);
1223
1224         return ib_post_send(target->qp, &wr, &bad_wr);
1225 }
1226
1227 static int srp_finish_mapping(struct srp_map_state *state,
1228                               struct srp_target_port *target)
1229 {
1230         int ret = 0;
1231
1232         if (state->npages == 0)
1233                 return 0;
1234
1235         if (state->npages == 1 && !register_always)
1236                 srp_map_desc(state, state->base_dma_addr, state->dma_len,
1237                              target->rkey);
1238         else
1239                 ret = target->srp_host->srp_dev->use_fast_reg ?
1240                         srp_map_finish_fr(state, target) :
1241                         srp_map_finish_fmr(state, target);
1242
1243         if (ret == 0) {
1244                 state->npages = 0;
1245                 state->dma_len = 0;
1246         }
1247
1248         return ret;
1249 }
1250
1251 static void srp_map_update_start(struct srp_map_state *state,
1252                                  struct scatterlist *sg, int sg_index,
1253                                  dma_addr_t dma_addr)
1254 {
1255         state->unmapped_sg = sg;
1256         state->unmapped_index = sg_index;
1257         state->unmapped_addr = dma_addr;
1258 }
1259
1260 static int srp_map_sg_entry(struct srp_map_state *state,
1261                             struct srp_target_port *target,
1262                             struct scatterlist *sg, int sg_index,
1263                             bool use_mr)
1264 {
1265         struct srp_device *dev = target->srp_host->srp_dev;
1266         struct ib_device *ibdev = dev->dev;
1267         dma_addr_t dma_addr = ib_sg_dma_address(ibdev, sg);
1268         unsigned int dma_len = ib_sg_dma_len(ibdev, sg);
1269         unsigned int len;
1270         int ret;
1271
1272         if (!dma_len)
1273                 return 0;
1274
1275         if (!use_mr) {
1276                 /*
1277                  * Once we're in direct map mode for a request, we don't
1278                  * go back to FMR or FR mode, so no need to update anything
1279                  * other than the descriptor.
1280                  */
1281                 srp_map_desc(state, dma_addr, dma_len, target->rkey);
1282                 return 0;
1283         }
1284
1285         /*
1286          * Since not all RDMA HW drivers support non-zero page offsets for
1287          * FMR, if we start at an offset into a page, don't merge into the
1288          * current FMR mapping. Finish it out, and use the kernel's MR for
1289          * this sg entry.
1290          */
1291         if ((!dev->use_fast_reg && dma_addr & ~dev->mr_page_mask) ||
1292             dma_len > dev->mr_max_size) {
1293                 ret = srp_finish_mapping(state, target);
1294                 if (ret)
1295                         return ret;
1296
1297                 srp_map_desc(state, dma_addr, dma_len, target->rkey);
1298                 srp_map_update_start(state, NULL, 0, 0);
1299                 return 0;
1300         }
1301
1302         /*
1303          * If this is the first sg that will be mapped via FMR or via FR, save
1304          * our position. We need to know the first unmapped entry, its index,
1305          * and the first unmapped address within that entry to be able to
1306          * restart mapping after an error.
1307          */
1308         if (!state->unmapped_sg)
1309                 srp_map_update_start(state, sg, sg_index, dma_addr);
1310
1311         while (dma_len) {
1312                 unsigned offset = dma_addr & ~dev->mr_page_mask;
1313                 if (state->npages == dev->max_pages_per_mr || offset != 0) {
1314                         ret = srp_finish_mapping(state, target);
1315                         if (ret)
1316                                 return ret;
1317
1318                         srp_map_update_start(state, sg, sg_index, dma_addr);
1319                 }
1320
1321                 len = min_t(unsigned int, dma_len, dev->mr_page_size - offset);
1322
1323                 if (!state->npages)
1324                         state->base_dma_addr = dma_addr;
1325                 state->pages[state->npages++] = dma_addr & dev->mr_page_mask;
1326                 state->dma_len += len;
1327                 dma_addr += len;
1328                 dma_len -= len;
1329         }
1330
1331         /*
1332          * If the last entry of the MR wasn't a full page, then we need to
1333          * close it out and start a new one -- we can only merge at page
1334          * boundries.
1335          */
1336         ret = 0;
1337         if (len != dev->mr_page_size) {
1338                 ret = srp_finish_mapping(state, target);
1339                 if (!ret)
1340                         srp_map_update_start(state, NULL, 0, 0);
1341         }
1342         return ret;
1343 }
1344
1345 static int srp_map_sg(struct srp_map_state *state,
1346                       struct srp_target_port *target, struct srp_request *req,
1347                       struct scatterlist *scat, int count)
1348 {
1349         struct srp_device *dev = target->srp_host->srp_dev;
1350         struct ib_device *ibdev = dev->dev;
1351         struct scatterlist *sg;
1352         int i;
1353         bool use_mr;
1354
1355         state->desc     = req->indirect_desc;
1356         state->pages    = req->map_page;
1357         if (dev->use_fast_reg) {
1358                 state->next_fr = req->fr_list;
1359                 use_mr = !!target->fr_pool;
1360         } else {
1361                 state->next_fmr = req->fmr_list;
1362                 use_mr = !!target->fmr_pool;
1363         }
1364
1365         for_each_sg(scat, sg, count, i) {
1366                 if (srp_map_sg_entry(state, target, sg, i, use_mr)) {
1367                         /*
1368                          * Memory registration failed, so backtrack to the
1369                          * first unmapped entry and continue on without using
1370                          * memory registration.
1371                          */
1372                         dma_addr_t dma_addr;
1373                         unsigned int dma_len;
1374
1375 backtrack:
1376                         sg = state->unmapped_sg;
1377                         i = state->unmapped_index;
1378
1379                         dma_addr = ib_sg_dma_address(ibdev, sg);
1380                         dma_len = ib_sg_dma_len(ibdev, sg);
1381                         dma_len -= (state->unmapped_addr - dma_addr);
1382                         dma_addr = state->unmapped_addr;
1383                         use_mr = false;
1384                         srp_map_desc(state, dma_addr, dma_len, target->rkey);
1385                 }
1386         }
1387
1388         if (use_mr && srp_finish_mapping(state, target))
1389                 goto backtrack;
1390
1391         req->nmdesc = state->nmdesc;
1392
1393         return 0;
1394 }
1395
1396 static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_target_port *target,
1397                         struct srp_request *req)
1398 {
1399         struct scatterlist *scat;
1400         struct srp_cmd *cmd = req->cmd->buf;
1401         int len, nents, count;
1402         struct srp_device *dev;
1403         struct ib_device *ibdev;
1404         struct srp_map_state state;
1405         struct srp_indirect_buf *indirect_hdr;
1406         u32 table_len;
1407         u8 fmt;
1408
1409         if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE)
1410                 return sizeof (struct srp_cmd);
1411
1412         if (scmnd->sc_data_direction != DMA_FROM_DEVICE &&
1413             scmnd->sc_data_direction != DMA_TO_DEVICE) {
1414                 shost_printk(KERN_WARNING, target->scsi_host,
1415                              PFX "Unhandled data direction %d\n",
1416                              scmnd->sc_data_direction);
1417                 return -EINVAL;
1418         }
1419
1420         nents = scsi_sg_count(scmnd);
1421         scat  = scsi_sglist(scmnd);
1422
1423         dev = target->srp_host->srp_dev;
1424         ibdev = dev->dev;
1425
1426         count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction);
1427         if (unlikely(count == 0))
1428                 return -EIO;
1429
1430         fmt = SRP_DATA_DESC_DIRECT;
1431         len = sizeof (struct srp_cmd) + sizeof (struct srp_direct_buf);
1432
1433         if (count == 1 && !register_always) {
1434                 /*
1435                  * The midlayer only generated a single gather/scatter
1436                  * entry, or DMA mapping coalesced everything to a
1437                  * single entry.  So a direct descriptor along with
1438                  * the DMA MR suffices.
1439                  */
1440                 struct srp_direct_buf *buf = (void *) cmd->add_data;
1441
1442                 buf->va  = cpu_to_be64(ib_sg_dma_address(ibdev, scat));
1443                 buf->key = cpu_to_be32(target->rkey);
1444                 buf->len = cpu_to_be32(ib_sg_dma_len(ibdev, scat));
1445
1446                 req->nmdesc = 0;
1447                 goto map_complete;
1448         }
1449
1450         /*
1451          * We have more than one scatter/gather entry, so build our indirect
1452          * descriptor table, trying to merge as many entries as we can.
1453          */
1454         indirect_hdr = (void *) cmd->add_data;
1455
1456         ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr,
1457                                    target->indirect_size, DMA_TO_DEVICE);
1458
1459         memset(&state, 0, sizeof(state));
1460         srp_map_sg(&state, target, req, scat, count);
1461
1462         /* We've mapped the request, now pull as much of the indirect
1463          * descriptor table as we can into the command buffer. If this
1464          * target is not using an external indirect table, we are
1465          * guaranteed to fit into the command, as the SCSI layer won't
1466          * give us more S/G entries than we allow.
1467          */
1468         if (state.ndesc == 1) {
1469                 /*
1470                  * Memory registration collapsed the sg-list into one entry,
1471                  * so use a direct descriptor.
1472                  */
1473                 struct srp_direct_buf *buf = (void *) cmd->add_data;
1474
1475                 *buf = req->indirect_desc[0];
1476                 goto map_complete;
1477         }
1478
1479         if (unlikely(target->cmd_sg_cnt < state.ndesc &&
1480                                                 !target->allow_ext_sg)) {
1481                 shost_printk(KERN_ERR, target->scsi_host,
1482                              "Could not fit S/G list into SRP_CMD\n");
1483                 return -EIO;
1484         }
1485
1486         count = min(state.ndesc, target->cmd_sg_cnt);
1487         table_len = state.ndesc * sizeof (struct srp_direct_buf);
1488
1489         fmt = SRP_DATA_DESC_INDIRECT;
1490         len = sizeof(struct srp_cmd) + sizeof (struct srp_indirect_buf);
1491         len += count * sizeof (struct srp_direct_buf);
1492
1493         memcpy(indirect_hdr->desc_list, req->indirect_desc,
1494                count * sizeof (struct srp_direct_buf));
1495
1496         indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr);
1497         indirect_hdr->table_desc.key = cpu_to_be32(target->rkey);
1498         indirect_hdr->table_desc.len = cpu_to_be32(table_len);
1499         indirect_hdr->len = cpu_to_be32(state.total_len);
1500
1501         if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1502                 cmd->data_out_desc_cnt = count;
1503         else
1504                 cmd->data_in_desc_cnt = count;
1505
1506         ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len,
1507                                       DMA_TO_DEVICE);
1508
1509 map_complete:
1510         if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1511                 cmd->buf_fmt = fmt << 4;
1512         else
1513                 cmd->buf_fmt = fmt;
1514
1515         return len;
1516 }
1517
1518 /*
1519  * Return an IU and possible credit to the free pool
1520  */
1521 static void srp_put_tx_iu(struct srp_target_port *target, struct srp_iu *iu,
1522                           enum srp_iu_type iu_type)
1523 {
1524         unsigned long flags;
1525
1526         spin_lock_irqsave(&target->lock, flags);
1527         list_add(&iu->list, &target->free_tx);
1528         if (iu_type != SRP_IU_RSP)
1529                 ++target->req_lim;
1530         spin_unlock_irqrestore(&target->lock, flags);
1531 }
1532
1533 /*
1534  * Must be called with target->lock held to protect req_lim and free_tx.
1535  * If IU is not sent, it must be returned using srp_put_tx_iu().
1536  *
1537  * Note:
1538  * An upper limit for the number of allocated information units for each
1539  * request type is:
1540  * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues
1541  *   more than Scsi_Host.can_queue requests.
1542  * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE.
1543  * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than
1544  *   one unanswered SRP request to an initiator.
1545  */
1546 static struct srp_iu *__srp_get_tx_iu(struct srp_target_port *target,
1547                                       enum srp_iu_type iu_type)
1548 {
1549         s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE;
1550         struct srp_iu *iu;
1551
1552         srp_send_completion(target->send_cq, target);
1553
1554         if (list_empty(&target->free_tx))
1555                 return NULL;
1556
1557         /* Initiator responses to target requests do not consume credits */
1558         if (iu_type != SRP_IU_RSP) {
1559                 if (target->req_lim <= rsv) {
1560                         ++target->zero_req_lim;
1561                         return NULL;
1562                 }
1563
1564                 --target->req_lim;
1565         }
1566
1567         iu = list_first_entry(&target->free_tx, struct srp_iu, list);
1568         list_del(&iu->list);
1569         return iu;
1570 }
1571
1572 static int srp_post_send(struct srp_target_port *target,
1573                          struct srp_iu *iu, int len)
1574 {
1575         struct ib_sge list;
1576         struct ib_send_wr wr, *bad_wr;
1577
1578         list.addr   = iu->dma;
1579         list.length = len;
1580         list.lkey   = target->lkey;
1581
1582         wr.next       = NULL;
1583         wr.wr_id      = (uintptr_t) iu;
1584         wr.sg_list    = &list;
1585         wr.num_sge    = 1;
1586         wr.opcode     = IB_WR_SEND;
1587         wr.send_flags = IB_SEND_SIGNALED;
1588
1589         return ib_post_send(target->qp, &wr, &bad_wr);
1590 }
1591
1592 static int srp_post_recv(struct srp_target_port *target, struct srp_iu *iu)
1593 {
1594         struct ib_recv_wr wr, *bad_wr;
1595         struct ib_sge list;
1596
1597         list.addr   = iu->dma;
1598         list.length = iu->size;
1599         list.lkey   = target->lkey;
1600
1601         wr.next     = NULL;
1602         wr.wr_id    = (uintptr_t) iu;
1603         wr.sg_list  = &list;
1604         wr.num_sge  = 1;
1605
1606         return ib_post_recv(target->qp, &wr, &bad_wr);
1607 }
1608
1609 static void srp_process_rsp(struct srp_target_port *target, struct srp_rsp *rsp)
1610 {
1611         struct srp_request *req;
1612         struct scsi_cmnd *scmnd;
1613         unsigned long flags;
1614
1615         if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) {
1616                 spin_lock_irqsave(&target->lock, flags);
1617                 target->req_lim += be32_to_cpu(rsp->req_lim_delta);
1618                 spin_unlock_irqrestore(&target->lock, flags);
1619
1620                 target->tsk_mgmt_status = -1;
1621                 if (be32_to_cpu(rsp->resp_data_len) >= 4)
1622                         target->tsk_mgmt_status = rsp->data[3];
1623                 complete(&target->tsk_mgmt_done);
1624         } else {
1625                 req = &target->req_ring[rsp->tag];
1626                 scmnd = srp_claim_req(target, req, NULL, NULL);
1627                 if (!scmnd) {
1628                         shost_printk(KERN_ERR, target->scsi_host,
1629                                      "Null scmnd for RSP w/tag %016llx\n",
1630                                      (unsigned long long) rsp->tag);
1631
1632                         spin_lock_irqsave(&target->lock, flags);
1633                         target->req_lim += be32_to_cpu(rsp->req_lim_delta);
1634                         spin_unlock_irqrestore(&target->lock, flags);
1635
1636                         return;
1637                 }
1638                 scmnd->result = rsp->status;
1639
1640                 if (rsp->flags & SRP_RSP_FLAG_SNSVALID) {
1641                         memcpy(scmnd->sense_buffer, rsp->data +
1642                                be32_to_cpu(rsp->resp_data_len),
1643                                min_t(int, be32_to_cpu(rsp->sense_data_len),
1644                                      SCSI_SENSE_BUFFERSIZE));
1645                 }
1646
1647                 if (unlikely(rsp->flags & SRP_RSP_FLAG_DIUNDER))
1648                         scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt));
1649                 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DIOVER))
1650                         scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_in_res_cnt));
1651                 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOUNDER))
1652                         scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt));
1653                 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOOVER))
1654                         scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_out_res_cnt));
1655
1656                 srp_free_req(target, req, scmnd,
1657                              be32_to_cpu(rsp->req_lim_delta));
1658
1659                 scmnd->host_scribble = NULL;
1660                 scmnd->scsi_done(scmnd);
1661         }
1662 }
1663
1664 static int srp_response_common(struct srp_target_port *target, s32 req_delta,
1665                                void *rsp, int len)
1666 {
1667         struct ib_device *dev = target->srp_host->srp_dev->dev;
1668         unsigned long flags;
1669         struct srp_iu *iu;
1670         int err;
1671
1672         spin_lock_irqsave(&target->lock, flags);
1673         target->req_lim += req_delta;
1674         iu = __srp_get_tx_iu(target, SRP_IU_RSP);
1675         spin_unlock_irqrestore(&target->lock, flags);
1676
1677         if (!iu) {
1678                 shost_printk(KERN_ERR, target->scsi_host, PFX
1679                              "no IU available to send response\n");
1680                 return 1;
1681         }
1682
1683         ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE);
1684         memcpy(iu->buf, rsp, len);
1685         ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE);
1686
1687         err = srp_post_send(target, iu, len);
1688         if (err) {
1689                 shost_printk(KERN_ERR, target->scsi_host, PFX
1690                              "unable to post response: %d\n", err);
1691                 srp_put_tx_iu(target, iu, SRP_IU_RSP);
1692         }
1693
1694         return err;
1695 }
1696
1697 static void srp_process_cred_req(struct srp_target_port *target,
1698                                  struct srp_cred_req *req)
1699 {
1700         struct srp_cred_rsp rsp = {
1701                 .opcode = SRP_CRED_RSP,
1702                 .tag = req->tag,
1703         };
1704         s32 delta = be32_to_cpu(req->req_lim_delta);
1705
1706         if (srp_response_common(target, delta, &rsp, sizeof rsp))
1707                 shost_printk(KERN_ERR, target->scsi_host, PFX
1708                              "problems processing SRP_CRED_REQ\n");
1709 }
1710
1711 static void srp_process_aer_req(struct srp_target_port *target,
1712                                 struct srp_aer_req *req)
1713 {
1714         struct srp_aer_rsp rsp = {
1715                 .opcode = SRP_AER_RSP,
1716                 .tag = req->tag,
1717         };
1718         s32 delta = be32_to_cpu(req->req_lim_delta);
1719
1720         shost_printk(KERN_ERR, target->scsi_host, PFX
1721                      "ignoring AER for LUN %llu\n", be64_to_cpu(req->lun));
1722
1723         if (srp_response_common(target, delta, &rsp, sizeof rsp))
1724                 shost_printk(KERN_ERR, target->scsi_host, PFX
1725                              "problems processing SRP_AER_REQ\n");
1726 }
1727
1728 static void srp_handle_recv(struct srp_target_port *target, struct ib_wc *wc)
1729 {
1730         struct ib_device *dev = target->srp_host->srp_dev->dev;
1731         struct srp_iu *iu = (struct srp_iu *) (uintptr_t) wc->wr_id;
1732         int res;
1733         u8 opcode;
1734
1735         ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_ti_iu_len,
1736                                    DMA_FROM_DEVICE);
1737
1738         opcode = *(u8 *) iu->buf;
1739
1740         if (0) {
1741                 shost_printk(KERN_ERR, target->scsi_host,
1742                              PFX "recv completion, opcode 0x%02x\n", opcode);
1743                 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1,
1744                                iu->buf, wc->byte_len, true);
1745         }
1746
1747         switch (opcode) {
1748         case SRP_RSP:
1749                 srp_process_rsp(target, iu->buf);
1750                 break;
1751
1752         case SRP_CRED_REQ:
1753                 srp_process_cred_req(target, iu->buf);
1754                 break;
1755
1756         case SRP_AER_REQ:
1757                 srp_process_aer_req(target, iu->buf);
1758                 break;
1759
1760         case SRP_T_LOGOUT:
1761                 /* XXX Handle target logout */
1762                 shost_printk(KERN_WARNING, target->scsi_host,
1763                              PFX "Got target logout request\n");
1764                 break;
1765
1766         default:
1767                 shost_printk(KERN_WARNING, target->scsi_host,
1768                              PFX "Unhandled SRP opcode 0x%02x\n", opcode);
1769                 break;
1770         }
1771
1772         ib_dma_sync_single_for_device(dev, iu->dma, target->max_ti_iu_len,
1773                                       DMA_FROM_DEVICE);
1774
1775         res = srp_post_recv(target, iu);
1776         if (res != 0)
1777                 shost_printk(KERN_ERR, target->scsi_host,
1778                              PFX "Recv failed with error code %d\n", res);
1779 }
1780
1781 /**
1782  * srp_tl_err_work() - handle a transport layer error
1783  * @work: Work structure embedded in an SRP target port.
1784  *
1785  * Note: This function may get invoked before the rport has been created,
1786  * hence the target->rport test.
1787  */
1788 static void srp_tl_err_work(struct work_struct *work)
1789 {
1790         struct srp_target_port *target;
1791
1792         target = container_of(work, struct srp_target_port, tl_err_work);
1793         if (target->rport)
1794                 srp_start_tl_fail_timers(target->rport);
1795 }
1796
1797 static void srp_handle_qp_err(u64 wr_id, enum ib_wc_status wc_status,
1798                               bool send_err, struct srp_target_port *target)
1799 {
1800         if (target->connected && !target->qp_in_error) {
1801                 if (wr_id & LOCAL_INV_WR_ID_MASK) {
1802                         shost_printk(KERN_ERR, target->scsi_host, PFX
1803                                      "LOCAL_INV failed with status %d\n",
1804                                      wc_status);
1805                 } else if (wr_id & FAST_REG_WR_ID_MASK) {
1806                         shost_printk(KERN_ERR, target->scsi_host, PFX
1807                                      "FAST_REG_MR failed status %d\n",
1808                                      wc_status);
1809                 } else {
1810                         shost_printk(KERN_ERR, target->scsi_host,
1811                                      PFX "failed %s status %d for iu %p\n",
1812                                      send_err ? "send" : "receive",
1813                                      wc_status, (void *)(uintptr_t)wr_id);
1814                 }
1815                 queue_work(system_long_wq, &target->tl_err_work);
1816         }
1817         target->qp_in_error = true;
1818 }
1819
1820 static void srp_recv_completion(struct ib_cq *cq, void *target_ptr)
1821 {
1822         struct srp_target_port *target = target_ptr;
1823         struct ib_wc wc;
1824
1825         ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
1826         while (ib_poll_cq(cq, 1, &wc) > 0) {
1827                 if (likely(wc.status == IB_WC_SUCCESS)) {
1828                         srp_handle_recv(target, &wc);
1829                 } else {
1830                         srp_handle_qp_err(wc.wr_id, wc.status, false, target);
1831                 }
1832         }
1833 }
1834
1835 static void srp_send_completion(struct ib_cq *cq, void *target_ptr)
1836 {
1837         struct srp_target_port *target = target_ptr;
1838         struct ib_wc wc;
1839         struct srp_iu *iu;
1840
1841         while (ib_poll_cq(cq, 1, &wc) > 0) {
1842                 if (likely(wc.status == IB_WC_SUCCESS)) {
1843                         iu = (struct srp_iu *) (uintptr_t) wc.wr_id;
1844                         list_add(&iu->list, &target->free_tx);
1845                 } else {
1846                         srp_handle_qp_err(wc.wr_id, wc.status, true, target);
1847                 }
1848         }
1849 }
1850
1851 static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
1852 {
1853         struct srp_target_port *target = host_to_target(shost);
1854         struct srp_rport *rport = target->rport;
1855         struct srp_request *req;
1856         struct srp_iu *iu;
1857         struct srp_cmd *cmd;
1858         struct ib_device *dev;
1859         unsigned long flags;
1860         int len, ret;
1861         const bool in_scsi_eh = !in_interrupt() && current == shost->ehandler;
1862
1863         /*
1864          * The SCSI EH thread is the only context from which srp_queuecommand()
1865          * can get invoked for blocked devices (SDEV_BLOCK /
1866          * SDEV_CREATED_BLOCK). Avoid racing with srp_reconnect_rport() by
1867          * locking the rport mutex if invoked from inside the SCSI EH.
1868          */
1869         if (in_scsi_eh)
1870                 mutex_lock(&rport->mutex);
1871
1872         scmnd->result = srp_chkready(target->rport);
1873         if (unlikely(scmnd->result))
1874                 goto err;
1875
1876         spin_lock_irqsave(&target->lock, flags);
1877         iu = __srp_get_tx_iu(target, SRP_IU_CMD);
1878         if (!iu)
1879                 goto err_unlock;
1880
1881         req = list_first_entry(&target->free_reqs, struct srp_request, list);
1882         list_del(&req->list);
1883         spin_unlock_irqrestore(&target->lock, flags);
1884
1885         dev = target->srp_host->srp_dev->dev;
1886         ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len,
1887                                    DMA_TO_DEVICE);
1888
1889         scmnd->host_scribble = (void *) req;
1890
1891         cmd = iu->buf;
1892         memset(cmd, 0, sizeof *cmd);
1893
1894         cmd->opcode = SRP_CMD;
1895         cmd->lun    = cpu_to_be64((u64) scmnd->device->lun << 48);
1896         cmd->tag    = req->index;
1897         memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len);
1898
1899         req->scmnd    = scmnd;
1900         req->cmd      = iu;
1901
1902         len = srp_map_data(scmnd, target, req);
1903         if (len < 0) {
1904                 shost_printk(KERN_ERR, target->scsi_host,
1905                              PFX "Failed to map data (%d)\n", len);
1906                 /*
1907                  * If we ran out of memory descriptors (-ENOMEM) because an
1908                  * application is queuing many requests with more than
1909                  * max_pages_per_mr sg-list elements, tell the SCSI mid-layer
1910                  * to reduce queue depth temporarily.
1911                  */
1912                 scmnd->result = len == -ENOMEM ?
1913                         DID_OK << 16 | QUEUE_FULL << 1 : DID_ERROR << 16;
1914                 goto err_iu;
1915         }
1916
1917         ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len,
1918                                       DMA_TO_DEVICE);
1919
1920         if (srp_post_send(target, iu, len)) {
1921                 shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n");
1922                 goto err_unmap;
1923         }
1924
1925         ret = 0;
1926
1927 unlock_rport:
1928         if (in_scsi_eh)
1929                 mutex_unlock(&rport->mutex);
1930
1931         return ret;
1932
1933 err_unmap:
1934         srp_unmap_data(scmnd, target, req);
1935
1936 err_iu:
1937         srp_put_tx_iu(target, iu, SRP_IU_CMD);
1938
1939         /*
1940          * Avoid that the loops that iterate over the request ring can
1941          * encounter a dangling SCSI command pointer.
1942          */
1943         req->scmnd = NULL;
1944
1945         spin_lock_irqsave(&target->lock, flags);
1946         list_add(&req->list, &target->free_reqs);
1947
1948 err_unlock:
1949         spin_unlock_irqrestore(&target->lock, flags);
1950
1951 err:
1952         if (scmnd->result) {
1953                 scmnd->scsi_done(scmnd);
1954                 ret = 0;
1955         } else {
1956                 ret = SCSI_MLQUEUE_HOST_BUSY;
1957         }
1958
1959         goto unlock_rport;
1960 }
1961
1962 /*
1963  * Note: the resources allocated in this function are freed in
1964  * srp_free_target_ib().
1965  */
1966 static int srp_alloc_iu_bufs(struct srp_target_port *target)
1967 {
1968         int i;
1969
1970         target->rx_ring = kzalloc(target->queue_size * sizeof(*target->rx_ring),
1971                                   GFP_KERNEL);
1972         if (!target->rx_ring)
1973                 goto err_no_ring;
1974         target->tx_ring = kzalloc(target->queue_size * sizeof(*target->tx_ring),
1975                                   GFP_KERNEL);
1976         if (!target->tx_ring)
1977                 goto err_no_ring;
1978
1979         for (i = 0; i < target->queue_size; ++i) {
1980                 target->rx_ring[i] = srp_alloc_iu(target->srp_host,
1981                                                   target->max_ti_iu_len,
1982                                                   GFP_KERNEL, DMA_FROM_DEVICE);
1983                 if (!target->rx_ring[i])
1984                         goto err;
1985         }
1986
1987         for (i = 0; i < target->queue_size; ++i) {
1988                 target->tx_ring[i] = srp_alloc_iu(target->srp_host,
1989                                                   target->max_iu_len,
1990                                                   GFP_KERNEL, DMA_TO_DEVICE);
1991                 if (!target->tx_ring[i])
1992                         goto err;
1993
1994                 list_add(&target->tx_ring[i]->list, &target->free_tx);
1995         }
1996
1997         return 0;
1998
1999 err:
2000         for (i = 0; i < target->queue_size; ++i) {
2001                 srp_free_iu(target->srp_host, target->rx_ring[i]);
2002                 srp_free_iu(target->srp_host, target->tx_ring[i]);
2003         }
2004
2005
2006 err_no_ring:
2007         kfree(target->tx_ring);
2008         target->tx_ring = NULL;
2009         kfree(target->rx_ring);
2010         target->rx_ring = NULL;
2011
2012         return -ENOMEM;
2013 }
2014
2015 static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask)
2016 {
2017         uint64_t T_tr_ns, max_compl_time_ms;
2018         uint32_t rq_tmo_jiffies;
2019
2020         /*
2021          * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair,
2022          * table 91), both the QP timeout and the retry count have to be set
2023          * for RC QP's during the RTR to RTS transition.
2024          */
2025         WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) !=
2026                      (IB_QP_TIMEOUT | IB_QP_RETRY_CNT));
2027
2028         /*
2029          * Set target->rq_tmo_jiffies to one second more than the largest time
2030          * it can take before an error completion is generated. See also
2031          * C9-140..142 in the IBTA spec for more information about how to
2032          * convert the QP Local ACK Timeout value to nanoseconds.
2033          */
2034         T_tr_ns = 4096 * (1ULL << qp_attr->timeout);
2035         max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns;
2036         do_div(max_compl_time_ms, NSEC_PER_MSEC);
2037         rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000);
2038
2039         return rq_tmo_jiffies;
2040 }
2041
2042 static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
2043                                struct srp_login_rsp *lrsp,
2044                                struct srp_target_port *target)
2045 {
2046         struct ib_qp_attr *qp_attr = NULL;
2047         int attr_mask = 0;
2048         int ret;
2049         int i;
2050
2051         if (lrsp->opcode == SRP_LOGIN_RSP) {
2052                 target->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len);
2053                 target->req_lim       = be32_to_cpu(lrsp->req_lim_delta);
2054
2055                 /*
2056                  * Reserve credits for task management so we don't
2057                  * bounce requests back to the SCSI mid-layer.
2058                  */
2059                 target->scsi_host->can_queue
2060                         = min(target->req_lim - SRP_TSK_MGMT_SQ_SIZE,
2061                               target->scsi_host->can_queue);
2062                 target->scsi_host->cmd_per_lun
2063                         = min_t(int, target->scsi_host->can_queue,
2064                                 target->scsi_host->cmd_per_lun);
2065         } else {
2066                 shost_printk(KERN_WARNING, target->scsi_host,
2067                              PFX "Unhandled RSP opcode %#x\n", lrsp->opcode);
2068                 ret = -ECONNRESET;
2069                 goto error;
2070         }
2071
2072         if (!target->rx_ring) {
2073                 ret = srp_alloc_iu_bufs(target);
2074                 if (ret)
2075                         goto error;
2076         }
2077
2078         ret = -ENOMEM;
2079         qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL);
2080         if (!qp_attr)
2081                 goto error;
2082
2083         qp_attr->qp_state = IB_QPS_RTR;
2084         ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2085         if (ret)
2086                 goto error_free;
2087
2088         ret = ib_modify_qp(target->qp, qp_attr, attr_mask);
2089         if (ret)
2090                 goto error_free;
2091
2092         for (i = 0; i < target->queue_size; i++) {
2093                 struct srp_iu *iu = target->rx_ring[i];
2094                 ret = srp_post_recv(target, iu);
2095                 if (ret)
2096                         goto error_free;
2097         }
2098
2099         qp_attr->qp_state = IB_QPS_RTS;
2100         ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2101         if (ret)
2102                 goto error_free;
2103
2104         target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask);
2105
2106         ret = ib_modify_qp(target->qp, qp_attr, attr_mask);
2107         if (ret)
2108                 goto error_free;
2109
2110         ret = ib_send_cm_rtu(cm_id, NULL, 0);
2111
2112 error_free:
2113         kfree(qp_attr);
2114
2115 error:
2116         target->status = ret;
2117 }
2118
2119 static void srp_cm_rej_handler(struct ib_cm_id *cm_id,
2120                                struct ib_cm_event *event,
2121                                struct srp_target_port *target)
2122 {
2123         struct Scsi_Host *shost = target->scsi_host;
2124         struct ib_class_port_info *cpi;
2125         int opcode;
2126
2127         switch (event->param.rej_rcvd.reason) {
2128         case IB_CM_REJ_PORT_CM_REDIRECT:
2129                 cpi = event->param.rej_rcvd.ari;
2130                 target->path.dlid = cpi->redirect_lid;
2131                 target->path.pkey = cpi->redirect_pkey;
2132                 cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff;
2133                 memcpy(target->path.dgid.raw, cpi->redirect_gid, 16);
2134
2135                 target->status = target->path.dlid ?
2136                         SRP_DLID_REDIRECT : SRP_PORT_REDIRECT;
2137                 break;
2138
2139         case IB_CM_REJ_PORT_REDIRECT:
2140                 if (srp_target_is_topspin(target)) {
2141                         /*
2142                          * Topspin/Cisco SRP gateways incorrectly send
2143                          * reject reason code 25 when they mean 24
2144                          * (port redirect).
2145                          */
2146                         memcpy(target->path.dgid.raw,
2147                                event->param.rej_rcvd.ari, 16);
2148
2149                         shost_printk(KERN_DEBUG, shost,
2150                                      PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n",
2151                                      (unsigned long long) be64_to_cpu(target->path.dgid.global.subnet_prefix),
2152                                      (unsigned long long) be64_to_cpu(target->path.dgid.global.interface_id));
2153
2154                         target->status = SRP_PORT_REDIRECT;
2155                 } else {
2156                         shost_printk(KERN_WARNING, shost,
2157                                      "  REJ reason: IB_CM_REJ_PORT_REDIRECT\n");
2158                         target->status = -ECONNRESET;
2159                 }
2160                 break;
2161
2162         case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
2163                 shost_printk(KERN_WARNING, shost,
2164                             "  REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
2165                 target->status = -ECONNRESET;
2166                 break;
2167
2168         case IB_CM_REJ_CONSUMER_DEFINED:
2169                 opcode = *(u8 *) event->private_data;
2170                 if (opcode == SRP_LOGIN_REJ) {
2171                         struct srp_login_rej *rej = event->private_data;
2172                         u32 reason = be32_to_cpu(rej->reason);
2173
2174                         if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE)
2175                                 shost_printk(KERN_WARNING, shost,
2176                                              PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n");
2177                         else
2178                                 shost_printk(KERN_WARNING, shost, PFX
2179                                              "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n",
2180                                              target->path.sgid.raw,
2181                                              target->orig_dgid, reason);
2182                 } else
2183                         shost_printk(KERN_WARNING, shost,
2184                                      "  REJ reason: IB_CM_REJ_CONSUMER_DEFINED,"
2185                                      " opcode 0x%02x\n", opcode);
2186                 target->status = -ECONNRESET;
2187                 break;
2188
2189         case IB_CM_REJ_STALE_CONN:
2190                 shost_printk(KERN_WARNING, shost, "  REJ reason: stale connection\n");
2191                 target->status = SRP_STALE_CONN;
2192                 break;
2193
2194         default:
2195                 shost_printk(KERN_WARNING, shost, "  REJ reason 0x%x\n",
2196                              event->param.rej_rcvd.reason);
2197                 target->status = -ECONNRESET;
2198         }
2199 }
2200
2201 static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2202 {
2203         struct srp_target_port *target = cm_id->context;
2204         int comp = 0;
2205
2206         switch (event->event) {
2207         case IB_CM_REQ_ERROR:
2208                 shost_printk(KERN_DEBUG, target->scsi_host,
2209                              PFX "Sending CM REQ failed\n");
2210                 comp = 1;
2211                 target->status = -ECONNRESET;
2212                 break;
2213
2214         case IB_CM_REP_RECEIVED:
2215                 comp = 1;
2216                 srp_cm_rep_handler(cm_id, event->private_data, target);
2217                 break;
2218
2219         case IB_CM_REJ_RECEIVED:
2220                 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
2221                 comp = 1;
2222
2223                 srp_cm_rej_handler(cm_id, event, target);
2224                 break;
2225
2226         case IB_CM_DREQ_RECEIVED:
2227                 shost_printk(KERN_WARNING, target->scsi_host,
2228                              PFX "DREQ received - connection closed\n");
2229                 srp_change_conn_state(target, false);
2230                 if (ib_send_cm_drep(cm_id, NULL, 0))
2231                         shost_printk(KERN_ERR, target->scsi_host,
2232                                      PFX "Sending CM DREP failed\n");
2233                 queue_work(system_long_wq, &target->tl_err_work);
2234                 break;
2235
2236         case IB_CM_TIMEWAIT_EXIT:
2237                 shost_printk(KERN_ERR, target->scsi_host,
2238                              PFX "connection closed\n");
2239                 comp = 1;
2240
2241                 target->status = 0;
2242                 break;
2243
2244         case IB_CM_MRA_RECEIVED:
2245         case IB_CM_DREQ_ERROR:
2246         case IB_CM_DREP_RECEIVED:
2247                 break;
2248
2249         default:
2250                 shost_printk(KERN_WARNING, target->scsi_host,
2251                              PFX "Unhandled CM event %d\n", event->event);
2252                 break;
2253         }
2254
2255         if (comp)
2256                 complete(&target->done);
2257
2258         return 0;
2259 }
2260
2261 /**
2262  * srp_change_queue_type - changing device queue tag type
2263  * @sdev: scsi device struct
2264  * @tag_type: requested tag type
2265  *
2266  * Returns queue tag type.
2267  */
2268 static int
2269 srp_change_queue_type(struct scsi_device *sdev, int tag_type)
2270 {
2271         if (sdev->tagged_supported) {
2272                 scsi_set_tag_type(sdev, tag_type);
2273                 if (tag_type)
2274                         scsi_activate_tcq(sdev, sdev->queue_depth);
2275                 else
2276                         scsi_deactivate_tcq(sdev, sdev->queue_depth);
2277         } else
2278                 tag_type = 0;
2279
2280         return tag_type;
2281 }
2282
2283 /**
2284  * srp_change_queue_depth - setting device queue depth
2285  * @sdev: scsi device struct
2286  * @qdepth: requested queue depth
2287  * @reason: SCSI_QDEPTH_DEFAULT/SCSI_QDEPTH_QFULL/SCSI_QDEPTH_RAMP_UP
2288  * (see include/scsi/scsi_host.h for definition)
2289  *
2290  * Returns queue depth.
2291  */
2292 static int
2293 srp_change_queue_depth(struct scsi_device *sdev, int qdepth, int reason)
2294 {
2295         struct Scsi_Host *shost = sdev->host;
2296         int max_depth;
2297         if (reason == SCSI_QDEPTH_DEFAULT || reason == SCSI_QDEPTH_RAMP_UP) {
2298                 max_depth = shost->can_queue;
2299                 if (!sdev->tagged_supported)
2300                         max_depth = 1;
2301                 if (qdepth > max_depth)
2302                         qdepth = max_depth;
2303                 scsi_adjust_queue_depth(sdev, scsi_get_tag_type(sdev), qdepth);
2304         } else if (reason == SCSI_QDEPTH_QFULL)
2305                 scsi_track_queue_full(sdev, qdepth);
2306         else
2307                 return -EOPNOTSUPP;
2308
2309         return sdev->queue_depth;
2310 }
2311
2312 static int srp_send_tsk_mgmt(struct srp_target_port *target,
2313                              u64 req_tag, unsigned int lun, u8 func)
2314 {
2315         struct srp_rport *rport = target->rport;
2316         struct ib_device *dev = target->srp_host->srp_dev->dev;
2317         struct srp_iu *iu;
2318         struct srp_tsk_mgmt *tsk_mgmt;
2319
2320         if (!target->connected || target->qp_in_error)
2321                 return -1;
2322
2323         init_completion(&target->tsk_mgmt_done);
2324
2325         /*
2326          * Lock the rport mutex to avoid that srp_create_target_ib() is
2327          * invoked while a task management function is being sent.
2328          */
2329         mutex_lock(&rport->mutex);
2330         spin_lock_irq(&target->lock);
2331         iu = __srp_get_tx_iu(target, SRP_IU_TSK_MGMT);
2332         spin_unlock_irq(&target->lock);
2333
2334         if (!iu) {
2335                 mutex_unlock(&rport->mutex);
2336
2337                 return -1;
2338         }
2339
2340         ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt,
2341                                    DMA_TO_DEVICE);
2342         tsk_mgmt = iu->buf;
2343         memset(tsk_mgmt, 0, sizeof *tsk_mgmt);
2344
2345         tsk_mgmt->opcode        = SRP_TSK_MGMT;
2346         tsk_mgmt->lun           = cpu_to_be64((u64) lun << 48);
2347         tsk_mgmt->tag           = req_tag | SRP_TAG_TSK_MGMT;
2348         tsk_mgmt->tsk_mgmt_func = func;
2349         tsk_mgmt->task_tag      = req_tag;
2350
2351         ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt,
2352                                       DMA_TO_DEVICE);
2353         if (srp_post_send(target, iu, sizeof *tsk_mgmt)) {
2354                 srp_put_tx_iu(target, iu, SRP_IU_TSK_MGMT);
2355                 mutex_unlock(&rport->mutex);
2356
2357                 return -1;
2358         }
2359         mutex_unlock(&rport->mutex);
2360
2361         if (!wait_for_completion_timeout(&target->tsk_mgmt_done,
2362                                          msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS)))
2363                 return -1;
2364
2365         return 0;
2366 }
2367
2368 static int srp_abort(struct scsi_cmnd *scmnd)
2369 {
2370         struct srp_target_port *target = host_to_target(scmnd->device->host);
2371         struct srp_request *req = (struct srp_request *) scmnd->host_scribble;
2372         int ret;
2373
2374         shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n");
2375
2376         if (!req || !srp_claim_req(target, req, NULL, scmnd))
2377                 return SUCCESS;
2378         if (srp_send_tsk_mgmt(target, req->index, scmnd->device->lun,
2379                               SRP_TSK_ABORT_TASK) == 0)
2380                 ret = SUCCESS;
2381         else if (target->rport->state == SRP_RPORT_LOST)
2382                 ret = FAST_IO_FAIL;
2383         else
2384                 ret = FAILED;
2385         srp_free_req(target, req, scmnd, 0);
2386         scmnd->result = DID_ABORT << 16;
2387         scmnd->scsi_done(scmnd);
2388
2389         return ret;
2390 }
2391
2392 static int srp_reset_device(struct scsi_cmnd *scmnd)
2393 {
2394         struct srp_target_port *target = host_to_target(scmnd->device->host);
2395         int i;
2396
2397         shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n");
2398
2399         if (srp_send_tsk_mgmt(target, SRP_TAG_NO_REQ, scmnd->device->lun,
2400                               SRP_TSK_LUN_RESET))
2401                 return FAILED;
2402         if (target->tsk_mgmt_status)
2403                 return FAILED;
2404
2405         for (i = 0; i < target->req_ring_size; ++i) {
2406                 struct srp_request *req = &target->req_ring[i];
2407                 srp_finish_req(target, req, scmnd->device, DID_RESET << 16);
2408         }
2409
2410         return SUCCESS;
2411 }
2412
2413 static int srp_reset_host(struct scsi_cmnd *scmnd)
2414 {
2415         struct srp_target_port *target = host_to_target(scmnd->device->host);
2416
2417         shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n");
2418
2419         return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED;
2420 }
2421
2422 static int srp_slave_configure(struct scsi_device *sdev)
2423 {
2424         struct Scsi_Host *shost = sdev->host;
2425         struct srp_target_port *target = host_to_target(shost);
2426         struct request_queue *q = sdev->request_queue;
2427         unsigned long timeout;
2428
2429         if (sdev->type == TYPE_DISK) {
2430                 timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies);
2431                 blk_queue_rq_timeout(q, timeout);
2432         }
2433
2434         return 0;
2435 }
2436
2437 static ssize_t show_id_ext(struct device *dev, struct device_attribute *attr,
2438                            char *buf)
2439 {
2440         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2441
2442         return sprintf(buf, "0x%016llx\n",
2443                        (unsigned long long) be64_to_cpu(target->id_ext));
2444 }
2445
2446 static ssize_t show_ioc_guid(struct device *dev, struct device_attribute *attr,
2447                              char *buf)
2448 {
2449         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2450
2451         return sprintf(buf, "0x%016llx\n",
2452                        (unsigned long long) be64_to_cpu(target->ioc_guid));
2453 }
2454
2455 static ssize_t show_service_id(struct device *dev,
2456                                struct device_attribute *attr, char *buf)
2457 {
2458         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2459
2460         return sprintf(buf, "0x%016llx\n",
2461                        (unsigned long long) be64_to_cpu(target->service_id));
2462 }
2463
2464 static ssize_t show_pkey(struct device *dev, struct device_attribute *attr,
2465                          char *buf)
2466 {
2467         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2468
2469         return sprintf(buf, "0x%04x\n", be16_to_cpu(target->path.pkey));
2470 }
2471
2472 static ssize_t show_sgid(struct device *dev, struct device_attribute *attr,
2473                          char *buf)
2474 {
2475         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2476
2477         return sprintf(buf, "%pI6\n", target->path.sgid.raw);
2478 }
2479
2480 static ssize_t show_dgid(struct device *dev, struct device_attribute *attr,
2481                          char *buf)
2482 {
2483         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2484
2485         return sprintf(buf, "%pI6\n", target->path.dgid.raw);
2486 }
2487
2488 static ssize_t show_orig_dgid(struct device *dev,
2489                               struct device_attribute *attr, char *buf)
2490 {
2491         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2492
2493         return sprintf(buf, "%pI6\n", target->orig_dgid);
2494 }
2495
2496 static ssize_t show_req_lim(struct device *dev,
2497                             struct device_attribute *attr, char *buf)
2498 {
2499         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2500
2501         return sprintf(buf, "%d\n", target->req_lim);
2502 }
2503
2504 static ssize_t show_zero_req_lim(struct device *dev,
2505                                  struct device_attribute *attr, char *buf)
2506 {
2507         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2508
2509         return sprintf(buf, "%d\n", target->zero_req_lim);
2510 }
2511
2512 static ssize_t show_local_ib_port(struct device *dev,
2513                                   struct device_attribute *attr, char *buf)
2514 {
2515         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2516
2517         return sprintf(buf, "%d\n", target->srp_host->port);
2518 }
2519
2520 static ssize_t show_local_ib_device(struct device *dev,
2521                                     struct device_attribute *attr, char *buf)
2522 {
2523         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2524
2525         return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name);
2526 }
2527
2528 static ssize_t show_comp_vector(struct device *dev,
2529                                 struct device_attribute *attr, char *buf)
2530 {
2531         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2532
2533         return sprintf(buf, "%d\n", target->comp_vector);
2534 }
2535
2536 static ssize_t show_tl_retry_count(struct device *dev,
2537                                    struct device_attribute *attr, char *buf)
2538 {
2539         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2540
2541         return sprintf(buf, "%d\n", target->tl_retry_count);
2542 }
2543
2544 static ssize_t show_cmd_sg_entries(struct device *dev,
2545                                    struct device_attribute *attr, char *buf)
2546 {
2547         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2548
2549         return sprintf(buf, "%u\n", target->cmd_sg_cnt);
2550 }
2551
2552 static ssize_t show_allow_ext_sg(struct device *dev,
2553                                  struct device_attribute *attr, char *buf)
2554 {
2555         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2556
2557         return sprintf(buf, "%s\n", target->allow_ext_sg ? "true" : "false");
2558 }
2559
2560 static DEVICE_ATTR(id_ext,          S_IRUGO, show_id_ext,          NULL);
2561 static DEVICE_ATTR(ioc_guid,        S_IRUGO, show_ioc_guid,        NULL);
2562 static DEVICE_ATTR(service_id,      S_IRUGO, show_service_id,      NULL);
2563 static DEVICE_ATTR(pkey,            S_IRUGO, show_pkey,            NULL);
2564 static DEVICE_ATTR(sgid,            S_IRUGO, show_sgid,            NULL);
2565 static DEVICE_ATTR(dgid,            S_IRUGO, show_dgid,            NULL);
2566 static DEVICE_ATTR(orig_dgid,       S_IRUGO, show_orig_dgid,       NULL);
2567 static DEVICE_ATTR(req_lim,         S_IRUGO, show_req_lim,         NULL);
2568 static DEVICE_ATTR(zero_req_lim,    S_IRUGO, show_zero_req_lim,    NULL);
2569 static DEVICE_ATTR(local_ib_port,   S_IRUGO, show_local_ib_port,   NULL);
2570 static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
2571 static DEVICE_ATTR(comp_vector,     S_IRUGO, show_comp_vector,     NULL);
2572 static DEVICE_ATTR(tl_retry_count,  S_IRUGO, show_tl_retry_count,  NULL);
2573 static DEVICE_ATTR(cmd_sg_entries,  S_IRUGO, show_cmd_sg_entries,  NULL);
2574 static DEVICE_ATTR(allow_ext_sg,    S_IRUGO, show_allow_ext_sg,    NULL);
2575
2576 static struct device_attribute *srp_host_attrs[] = {
2577         &dev_attr_id_ext,
2578         &dev_attr_ioc_guid,
2579         &dev_attr_service_id,
2580         &dev_attr_pkey,
2581         &dev_attr_sgid,
2582         &dev_attr_dgid,
2583         &dev_attr_orig_dgid,
2584         &dev_attr_req_lim,
2585         &dev_attr_zero_req_lim,
2586         &dev_attr_local_ib_port,
2587         &dev_attr_local_ib_device,
2588         &dev_attr_comp_vector,
2589         &dev_attr_tl_retry_count,
2590         &dev_attr_cmd_sg_entries,
2591         &dev_attr_allow_ext_sg,
2592         NULL
2593 };
2594
2595 static struct scsi_host_template srp_template = {
2596         .module                         = THIS_MODULE,
2597         .name                           = "InfiniBand SRP initiator",
2598         .proc_name                      = DRV_NAME,
2599         .slave_configure                = srp_slave_configure,
2600         .info                           = srp_target_info,
2601         .queuecommand                   = srp_queuecommand,
2602         .change_queue_depth             = srp_change_queue_depth,
2603         .change_queue_type              = srp_change_queue_type,
2604         .eh_abort_handler               = srp_abort,
2605         .eh_device_reset_handler        = srp_reset_device,
2606         .eh_host_reset_handler          = srp_reset_host,
2607         .skip_settle_delay              = true,
2608         .sg_tablesize                   = SRP_DEF_SG_TABLESIZE,
2609         .can_queue                      = SRP_DEFAULT_CMD_SQ_SIZE,
2610         .this_id                        = -1,
2611         .cmd_per_lun                    = SRP_DEFAULT_CMD_SQ_SIZE,
2612         .use_clustering                 = ENABLE_CLUSTERING,
2613         .shost_attrs                    = srp_host_attrs
2614 };
2615
2616 static int srp_add_target(struct srp_host *host, struct srp_target_port *target)
2617 {
2618         struct srp_rport_identifiers ids;
2619         struct srp_rport *rport;
2620
2621         sprintf(target->target_name, "SRP.T10:%016llX",
2622                  (unsigned long long) be64_to_cpu(target->id_ext));
2623
2624         if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dma_device))
2625                 return -ENODEV;
2626
2627         memcpy(ids.port_id, &target->id_ext, 8);
2628         memcpy(ids.port_id + 8, &target->ioc_guid, 8);
2629         ids.roles = SRP_RPORT_ROLE_TARGET;
2630         rport = srp_rport_add(target->scsi_host, &ids);
2631         if (IS_ERR(rport)) {
2632                 scsi_remove_host(target->scsi_host);
2633                 return PTR_ERR(rport);
2634         }
2635
2636         rport->lld_data = target;
2637         target->rport = rport;
2638
2639         spin_lock(&host->target_lock);
2640         list_add_tail(&target->list, &host->target_list);
2641         spin_unlock(&host->target_lock);
2642
2643         target->state = SRP_TARGET_LIVE;
2644
2645         scsi_scan_target(&target->scsi_host->shost_gendev,
2646                          0, target->scsi_id, SCAN_WILD_CARD, 0);
2647
2648         return 0;
2649 }
2650
2651 static void srp_release_dev(struct device *dev)
2652 {
2653         struct srp_host *host =
2654                 container_of(dev, struct srp_host, dev);
2655
2656         complete(&host->released);
2657 }
2658
2659 static struct class srp_class = {
2660         .name    = "infiniband_srp",
2661         .dev_release = srp_release_dev
2662 };
2663
2664 /**
2665  * srp_conn_unique() - check whether the connection to a target is unique
2666  * @host:   SRP host.
2667  * @target: SRP target port.
2668  */
2669 static bool srp_conn_unique(struct srp_host *host,
2670                             struct srp_target_port *target)
2671 {
2672         struct srp_target_port *t;
2673         bool ret = false;
2674
2675         if (target->state == SRP_TARGET_REMOVED)
2676                 goto out;
2677
2678         ret = true;
2679
2680         spin_lock(&host->target_lock);
2681         list_for_each_entry(t, &host->target_list, list) {
2682                 if (t != target &&
2683                     target->id_ext == t->id_ext &&
2684                     target->ioc_guid == t->ioc_guid &&
2685                     target->initiator_ext == t->initiator_ext) {
2686                         ret = false;
2687                         break;
2688                 }
2689         }
2690         spin_unlock(&host->target_lock);
2691
2692 out:
2693         return ret;
2694 }
2695
2696 /*
2697  * Target ports are added by writing
2698  *
2699  *     id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>,
2700  *     pkey=<P_Key>,service_id=<service ID>
2701  *
2702  * to the add_target sysfs attribute.
2703  */
2704 enum {
2705         SRP_OPT_ERR             = 0,
2706         SRP_OPT_ID_EXT          = 1 << 0,
2707         SRP_OPT_IOC_GUID        = 1 << 1,
2708         SRP_OPT_DGID            = 1 << 2,
2709         SRP_OPT_PKEY            = 1 << 3,
2710         SRP_OPT_SERVICE_ID      = 1 << 4,
2711         SRP_OPT_MAX_SECT        = 1 << 5,
2712         SRP_OPT_MAX_CMD_PER_LUN = 1 << 6,
2713         SRP_OPT_IO_CLASS        = 1 << 7,
2714         SRP_OPT_INITIATOR_EXT   = 1 << 8,
2715         SRP_OPT_CMD_SG_ENTRIES  = 1 << 9,
2716         SRP_OPT_ALLOW_EXT_SG    = 1 << 10,
2717         SRP_OPT_SG_TABLESIZE    = 1 << 11,
2718         SRP_OPT_COMP_VECTOR     = 1 << 12,
2719         SRP_OPT_TL_RETRY_COUNT  = 1 << 13,
2720         SRP_OPT_QUEUE_SIZE      = 1 << 14,
2721         SRP_OPT_ALL             = (SRP_OPT_ID_EXT       |
2722                                    SRP_OPT_IOC_GUID     |
2723                                    SRP_OPT_DGID         |
2724                                    SRP_OPT_PKEY         |
2725                                    SRP_OPT_SERVICE_ID),
2726 };
2727
2728 static const match_table_t srp_opt_tokens = {
2729         { SRP_OPT_ID_EXT,               "id_ext=%s"             },
2730         { SRP_OPT_IOC_GUID,             "ioc_guid=%s"           },
2731         { SRP_OPT_DGID,                 "dgid=%s"               },
2732         { SRP_OPT_PKEY,                 "pkey=%x"               },
2733         { SRP_OPT_SERVICE_ID,           "service_id=%s"         },
2734         { SRP_OPT_MAX_SECT,             "max_sect=%d"           },
2735         { SRP_OPT_MAX_CMD_PER_LUN,      "max_cmd_per_lun=%d"    },
2736         { SRP_OPT_IO_CLASS,             "io_class=%x"           },
2737         { SRP_OPT_INITIATOR_EXT,        "initiator_ext=%s"      },
2738         { SRP_OPT_CMD_SG_ENTRIES,       "cmd_sg_entries=%u"     },
2739         { SRP_OPT_ALLOW_EXT_SG,         "allow_ext_sg=%u"       },
2740         { SRP_OPT_SG_TABLESIZE,         "sg_tablesize=%u"       },
2741         { SRP_OPT_COMP_VECTOR,          "comp_vector=%u"        },
2742         { SRP_OPT_TL_RETRY_COUNT,       "tl_retry_count=%u"     },
2743         { SRP_OPT_QUEUE_SIZE,           "queue_size=%d"         },
2744         { SRP_OPT_ERR,                  NULL                    }
2745 };
2746
2747 static int srp_parse_options(const char *buf, struct srp_target_port *target)
2748 {
2749         char *options, *sep_opt;
2750         char *p;
2751         char dgid[3];
2752         substring_t args[MAX_OPT_ARGS];
2753         int opt_mask = 0;
2754         int token;
2755         int ret = -EINVAL;
2756         int i;
2757
2758         options = kstrdup(buf, GFP_KERNEL);
2759         if (!options)
2760                 return -ENOMEM;
2761
2762         sep_opt = options;
2763         while ((p = strsep(&sep_opt, ",")) != NULL) {
2764                 if (!*p)
2765                         continue;
2766
2767                 token = match_token(p, srp_opt_tokens, args);
2768                 opt_mask |= token;
2769
2770                 switch (token) {
2771                 case SRP_OPT_ID_EXT:
2772                         p = match_strdup(args);
2773                         if (!p) {
2774                                 ret = -ENOMEM;
2775                                 goto out;
2776                         }
2777                         target->id_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
2778                         kfree(p);
2779                         break;
2780
2781                 case SRP_OPT_IOC_GUID:
2782                         p = match_strdup(args);
2783                         if (!p) {
2784                                 ret = -ENOMEM;
2785                                 goto out;
2786                         }
2787                         target->ioc_guid = cpu_to_be64(simple_strtoull(p, NULL, 16));
2788                         kfree(p);
2789                         break;
2790
2791                 case SRP_OPT_DGID:
2792                         p = match_strdup(args);
2793                         if (!p) {
2794                                 ret = -ENOMEM;
2795                                 goto out;
2796                         }
2797                         if (strlen(p) != 32) {
2798                                 pr_warn("bad dest GID parameter '%s'\n", p);
2799                                 kfree(p);
2800                                 goto out;
2801                         }
2802
2803                         for (i = 0; i < 16; ++i) {
2804                                 strlcpy(dgid, p + i * 2, 3);
2805                                 target->path.dgid.raw[i] = simple_strtoul(dgid, NULL, 16);
2806                         }
2807                         kfree(p);
2808                         memcpy(target->orig_dgid, target->path.dgid.raw, 16);
2809                         break;
2810
2811                 case SRP_OPT_PKEY:
2812                         if (match_hex(args, &token)) {
2813                                 pr_warn("bad P_Key parameter '%s'\n", p);
2814                                 goto out;
2815                         }
2816                         target->path.pkey = cpu_to_be16(token);
2817                         break;
2818
2819                 case SRP_OPT_SERVICE_ID:
2820                         p = match_strdup(args);
2821                         if (!p) {
2822                                 ret = -ENOMEM;
2823                                 goto out;
2824                         }
2825                         target->service_id = cpu_to_be64(simple_strtoull(p, NULL, 16));
2826                         target->path.service_id = target->service_id;
2827                         kfree(p);
2828                         break;
2829
2830                 case SRP_OPT_MAX_SECT:
2831                         if (match_int(args, &token)) {
2832                                 pr_warn("bad max sect parameter '%s'\n", p);
2833                                 goto out;
2834                         }
2835                         target->scsi_host->max_sectors = token;
2836                         break;
2837
2838                 case SRP_OPT_QUEUE_SIZE:
2839                         if (match_int(args, &token) || token < 1) {
2840                                 pr_warn("bad queue_size parameter '%s'\n", p);
2841                                 goto out;
2842                         }
2843                         target->scsi_host->can_queue = token;
2844                         target->queue_size = token + SRP_RSP_SQ_SIZE +
2845                                              SRP_TSK_MGMT_SQ_SIZE;
2846                         if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
2847                                 target->scsi_host->cmd_per_lun = token;
2848                         break;
2849
2850                 case SRP_OPT_MAX_CMD_PER_LUN:
2851                         if (match_int(args, &token) || token < 1) {
2852                                 pr_warn("bad max cmd_per_lun parameter '%s'\n",
2853                                         p);
2854                                 goto out;
2855                         }
2856                         target->scsi_host->cmd_per_lun = token;
2857                         break;
2858
2859                 case SRP_OPT_IO_CLASS:
2860                         if (match_hex(args, &token)) {
2861                                 pr_warn("bad IO class parameter '%s'\n", p);
2862                                 goto out;
2863                         }
2864                         if (token != SRP_REV10_IB_IO_CLASS &&
2865                             token != SRP_REV16A_IB_IO_CLASS) {
2866                                 pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n",
2867                                         token, SRP_REV10_IB_IO_CLASS,
2868                                         SRP_REV16A_IB_IO_CLASS);
2869                                 goto out;
2870                         }
2871                         target->io_class = token;
2872                         break;
2873
2874                 case SRP_OPT_INITIATOR_EXT:
2875                         p = match_strdup(args);
2876                         if (!p) {
2877                                 ret = -ENOMEM;
2878                                 goto out;
2879                         }
2880                         target->initiator_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
2881                         kfree(p);
2882                         break;
2883
2884                 case SRP_OPT_CMD_SG_ENTRIES:
2885                         if (match_int(args, &token) || token < 1 || token > 255) {
2886                                 pr_warn("bad max cmd_sg_entries parameter '%s'\n",
2887                                         p);
2888                                 goto out;
2889                         }
2890                         target->cmd_sg_cnt = token;
2891                         break;
2892
2893                 case SRP_OPT_ALLOW_EXT_SG:
2894                         if (match_int(args, &token)) {
2895                                 pr_warn("bad allow_ext_sg parameter '%s'\n", p);
2896                                 goto out;
2897                         }
2898                         target->allow_ext_sg = !!token;
2899                         break;
2900
2901                 case SRP_OPT_SG_TABLESIZE:
2902                         if (match_int(args, &token) || token < 1 ||
2903                                         token > SCSI_MAX_SG_CHAIN_SEGMENTS) {
2904                                 pr_warn("bad max sg_tablesize parameter '%s'\n",
2905                                         p);
2906                                 goto out;
2907                         }
2908                         target->sg_tablesize = token;
2909                         break;
2910
2911                 case SRP_OPT_COMP_VECTOR:
2912                         if (match_int(args, &token) || token < 0) {
2913                                 pr_warn("bad comp_vector parameter '%s'\n", p);
2914                                 goto out;
2915                         }
2916                         target->comp_vector = token;
2917                         break;
2918
2919                 case SRP_OPT_TL_RETRY_COUNT:
2920                         if (match_int(args, &token) || token < 2 || token > 7) {
2921                                 pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n",
2922                                         p);
2923                                 goto out;
2924                         }
2925                         target->tl_retry_count = token;
2926                         break;
2927
2928                 default:
2929                         pr_warn("unknown parameter or missing value '%s' in target creation request\n",
2930                                 p);
2931                         goto out;
2932                 }
2933         }
2934
2935         if ((opt_mask & SRP_OPT_ALL) == SRP_OPT_ALL)
2936                 ret = 0;
2937         else
2938                 for (i = 0; i < ARRAY_SIZE(srp_opt_tokens); ++i)
2939                         if ((srp_opt_tokens[i].token & SRP_OPT_ALL) &&
2940                             !(srp_opt_tokens[i].token & opt_mask))
2941                                 pr_warn("target creation request is missing parameter '%s'\n",
2942                                         srp_opt_tokens[i].pattern);
2943
2944         if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue
2945             && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
2946                 pr_warn("cmd_per_lun = %d > queue_size = %d\n",
2947                         target->scsi_host->cmd_per_lun,
2948                         target->scsi_host->can_queue);
2949
2950 out:
2951         kfree(options);
2952         return ret;
2953 }
2954
2955 static ssize_t srp_create_target(struct device *dev,
2956                                  struct device_attribute *attr,
2957                                  const char *buf, size_t count)
2958 {
2959         struct srp_host *host =
2960                 container_of(dev, struct srp_host, dev);
2961         struct Scsi_Host *target_host;
2962         struct srp_target_port *target;
2963         struct srp_device *srp_dev = host->srp_dev;
2964         struct ib_device *ibdev = srp_dev->dev;
2965         int ret;
2966
2967         target_host = scsi_host_alloc(&srp_template,
2968                                       sizeof (struct srp_target_port));
2969         if (!target_host)
2970                 return -ENOMEM;
2971
2972         target_host->transportt  = ib_srp_transport_template;
2973         target_host->max_channel = 0;
2974         target_host->max_id      = 1;
2975         target_host->max_lun     = SRP_MAX_LUN;
2976         target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb;
2977
2978         target = host_to_target(target_host);
2979
2980         target->io_class        = SRP_REV16A_IB_IO_CLASS;
2981         target->scsi_host       = target_host;
2982         target->srp_host        = host;
2983         target->lkey            = host->srp_dev->mr->lkey;
2984         target->rkey            = host->srp_dev->mr->rkey;
2985         target->cmd_sg_cnt      = cmd_sg_entries;
2986         target->sg_tablesize    = indirect_sg_entries ? : cmd_sg_entries;
2987         target->allow_ext_sg    = allow_ext_sg;
2988         target->tl_retry_count  = 7;
2989         target->queue_size      = SRP_DEFAULT_QUEUE_SIZE;
2990
2991         mutex_lock(&host->add_target_mutex);
2992
2993         ret = srp_parse_options(buf, target);
2994         if (ret)
2995                 goto err;
2996
2997         target->req_ring_size = target->queue_size - SRP_TSK_MGMT_SQ_SIZE;
2998
2999         if (!srp_conn_unique(target->srp_host, target)) {
3000                 shost_printk(KERN_INFO, target->scsi_host,
3001                              PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n",
3002                              be64_to_cpu(target->id_ext),
3003                              be64_to_cpu(target->ioc_guid),
3004                              be64_to_cpu(target->initiator_ext));
3005                 ret = -EEXIST;
3006                 goto err;
3007         }
3008
3009         if (!srp_dev->has_fmr && !srp_dev->has_fr && !target->allow_ext_sg &&
3010             target->cmd_sg_cnt < target->sg_tablesize) {
3011                 pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n");
3012                 target->sg_tablesize = target->cmd_sg_cnt;
3013         }
3014
3015         target_host->sg_tablesize = target->sg_tablesize;
3016         target->indirect_size = target->sg_tablesize *
3017                                 sizeof (struct srp_direct_buf);
3018         target->max_iu_len = sizeof (struct srp_cmd) +
3019                              sizeof (struct srp_indirect_buf) +
3020                              target->cmd_sg_cnt * sizeof (struct srp_direct_buf);
3021
3022         INIT_WORK(&target->tl_err_work, srp_tl_err_work);
3023         INIT_WORK(&target->remove_work, srp_remove_work);
3024         spin_lock_init(&target->lock);
3025         INIT_LIST_HEAD(&target->free_tx);
3026         ret = srp_alloc_req_data(target);
3027         if (ret)
3028                 goto err_free_mem;
3029
3030         ret = ib_query_gid(ibdev, host->port, 0, &target->path.sgid);
3031         if (ret)
3032                 goto err_free_mem;
3033
3034         ret = srp_create_target_ib(target);
3035         if (ret)
3036                 goto err_free_mem;
3037
3038         ret = srp_new_cm_id(target);
3039         if (ret)
3040                 goto err_free_ib;
3041
3042         ret = srp_connect_target(target);
3043         if (ret) {
3044                 shost_printk(KERN_ERR, target->scsi_host,
3045                              PFX "Connection failed\n");
3046                 goto err_cm_id;
3047         }
3048
3049         ret = srp_add_target(host, target);
3050         if (ret)
3051                 goto err_disconnect;
3052
3053         shost_printk(KERN_DEBUG, target->scsi_host, PFX
3054                      "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n",
3055                      be64_to_cpu(target->id_ext),
3056                      be64_to_cpu(target->ioc_guid),
3057                      be16_to_cpu(target->path.pkey),
3058                      be64_to_cpu(target->service_id),
3059                      target->path.sgid.raw, target->path.dgid.raw);
3060
3061         ret = count;
3062
3063 out:
3064         mutex_unlock(&host->add_target_mutex);
3065         return ret;
3066
3067 err_disconnect:
3068         srp_disconnect_target(target);
3069
3070 err_cm_id:
3071         ib_destroy_cm_id(target->cm_id);
3072
3073 err_free_ib:
3074         srp_free_target_ib(target);
3075
3076 err_free_mem:
3077         srp_free_req_data(target);
3078
3079 err:
3080         scsi_host_put(target_host);
3081         goto out;
3082 }
3083
3084 static DEVICE_ATTR(add_target, S_IWUSR, NULL, srp_create_target);
3085
3086 static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr,
3087                           char *buf)
3088 {
3089         struct srp_host *host = container_of(dev, struct srp_host, dev);
3090
3091         return sprintf(buf, "%s\n", host->srp_dev->dev->name);
3092 }
3093
3094 static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
3095
3096 static ssize_t show_port(struct device *dev, struct device_attribute *attr,
3097                          char *buf)
3098 {
3099         struct srp_host *host = container_of(dev, struct srp_host, dev);
3100
3101         return sprintf(buf, "%d\n", host->port);
3102 }
3103
3104 static DEVICE_ATTR(port, S_IRUGO, show_port, NULL);
3105
3106 static struct srp_host *srp_add_port(struct srp_device *device, u8 port)
3107 {
3108         struct srp_host *host;
3109
3110         host = kzalloc(sizeof *host, GFP_KERNEL);
3111         if (!host)
3112                 return NULL;
3113
3114         INIT_LIST_HEAD(&host->target_list);
3115         spin_lock_init(&host->target_lock);
3116         init_completion(&host->released);
3117         mutex_init(&host->add_target_mutex);
3118         host->srp_dev = device;
3119         host->port = port;
3120
3121         host->dev.class = &srp_class;
3122         host->dev.parent = device->dev->dma_device;
3123         dev_set_name(&host->dev, "srp-%s-%d", device->dev->name, port);
3124
3125         if (device_register(&host->dev))
3126                 goto free_host;
3127         if (device_create_file(&host->dev, &dev_attr_add_target))
3128                 goto err_class;
3129         if (device_create_file(&host->dev, &dev_attr_ibdev))
3130                 goto err_class;
3131         if (device_create_file(&host->dev, &dev_attr_port))
3132                 goto err_class;
3133
3134         return host;
3135
3136 err_class:
3137         device_unregister(&host->dev);
3138
3139 free_host:
3140         kfree(host);
3141
3142         return NULL;
3143 }
3144
3145 static void srp_add_one(struct ib_device *device)
3146 {
3147         struct srp_device *srp_dev;
3148         struct ib_device_attr *dev_attr;
3149         struct srp_host *host;
3150         int mr_page_shift, s, e, p;
3151         u64 max_pages_per_mr;
3152
3153         dev_attr = kmalloc(sizeof *dev_attr, GFP_KERNEL);
3154         if (!dev_attr)
3155                 return;
3156
3157         if (ib_query_device(device, dev_attr)) {
3158                 pr_warn("Query device failed for %s\n", device->name);
3159                 goto free_attr;
3160         }
3161
3162         srp_dev = kmalloc(sizeof *srp_dev, GFP_KERNEL);
3163         if (!srp_dev)
3164                 goto free_attr;
3165
3166         srp_dev->has_fmr = (device->alloc_fmr && device->dealloc_fmr &&
3167                             device->map_phys_fmr && device->unmap_fmr);
3168         srp_dev->has_fr = (dev_attr->device_cap_flags &
3169                            IB_DEVICE_MEM_MGT_EXTENSIONS);
3170         if (!srp_dev->has_fmr && !srp_dev->has_fr)
3171                 dev_warn(&device->dev, "neither FMR nor FR is supported\n");
3172
3173         srp_dev->use_fast_reg = (srp_dev->has_fr &&
3174                                  (!srp_dev->has_fmr || prefer_fr));
3175
3176         /*
3177          * Use the smallest page size supported by the HCA, down to a
3178          * minimum of 4096 bytes. We're unlikely to build large sglists
3179          * out of smaller entries.
3180          */
3181         mr_page_shift           = max(12, ffs(dev_attr->page_size_cap) - 1);
3182         srp_dev->mr_page_size   = 1 << mr_page_shift;
3183         srp_dev->mr_page_mask   = ~((u64) srp_dev->mr_page_size - 1);
3184         max_pages_per_mr        = dev_attr->max_mr_size;
3185         do_div(max_pages_per_mr, srp_dev->mr_page_size);
3186         srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR,
3187                                           max_pages_per_mr);
3188         if (srp_dev->use_fast_reg) {
3189                 srp_dev->max_pages_per_mr =
3190                         min_t(u32, srp_dev->max_pages_per_mr,
3191                               dev_attr->max_fast_reg_page_list_len);
3192         }
3193         srp_dev->mr_max_size    = srp_dev->mr_page_size *
3194                                    srp_dev->max_pages_per_mr;
3195         pr_debug("%s: mr_page_shift = %d, dev_attr->max_mr_size = %#llx, dev_attr->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n",
3196                  device->name, mr_page_shift, dev_attr->max_mr_size,
3197                  dev_attr->max_fast_reg_page_list_len,
3198                  srp_dev->max_pages_per_mr, srp_dev->mr_max_size);
3199
3200         INIT_LIST_HEAD(&srp_dev->dev_list);
3201
3202         srp_dev->dev = device;
3203         srp_dev->pd  = ib_alloc_pd(device);
3204         if (IS_ERR(srp_dev->pd))
3205                 goto free_dev;
3206
3207         srp_dev->mr = ib_get_dma_mr(srp_dev->pd,
3208                                     IB_ACCESS_LOCAL_WRITE |
3209                                     IB_ACCESS_REMOTE_READ |
3210                                     IB_ACCESS_REMOTE_WRITE);
3211         if (IS_ERR(srp_dev->mr))
3212                 goto err_pd;
3213
3214         if (device->node_type == RDMA_NODE_IB_SWITCH) {
3215                 s = 0;
3216                 e = 0;
3217         } else {
3218                 s = 1;
3219                 e = device->phys_port_cnt;
3220         }
3221
3222         for (p = s; p <= e; ++p) {
3223                 host = srp_add_port(srp_dev, p);
3224                 if (host)
3225                         list_add_tail(&host->list, &srp_dev->dev_list);
3226         }
3227
3228         ib_set_client_data(device, &srp_client, srp_dev);
3229
3230         goto free_attr;
3231
3232 err_pd:
3233         ib_dealloc_pd(srp_dev->pd);
3234
3235 free_dev:
3236         kfree(srp_dev);
3237
3238 free_attr:
3239         kfree(dev_attr);
3240 }
3241
3242 static void srp_remove_one(struct ib_device *device)
3243 {
3244         struct srp_device *srp_dev;
3245         struct srp_host *host, *tmp_host;
3246         struct srp_target_port *target;
3247
3248         srp_dev = ib_get_client_data(device, &srp_client);
3249         if (!srp_dev)
3250                 return;
3251
3252         list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
3253                 device_unregister(&host->dev);
3254                 /*
3255                  * Wait for the sysfs entry to go away, so that no new
3256                  * target ports can be created.
3257                  */
3258                 wait_for_completion(&host->released);
3259
3260                 /*
3261                  * Remove all target ports.
3262                  */
3263                 spin_lock(&host->target_lock);
3264                 list_for_each_entry(target, &host->target_list, list)
3265                         srp_queue_remove_work(target);
3266                 spin_unlock(&host->target_lock);
3267
3268                 /*
3269                  * Wait for tl_err and target port removal tasks.
3270                  */
3271                 flush_workqueue(system_long_wq);
3272                 flush_workqueue(srp_remove_wq);
3273
3274                 kfree(host);
3275         }
3276
3277         ib_dereg_mr(srp_dev->mr);
3278         ib_dealloc_pd(srp_dev->pd);
3279
3280         kfree(srp_dev);
3281 }
3282
3283 static struct srp_function_template ib_srp_transport_functions = {
3284         .has_rport_state         = true,
3285         .reset_timer_if_blocked  = true,
3286         .reconnect_delay         = &srp_reconnect_delay,
3287         .fast_io_fail_tmo        = &srp_fast_io_fail_tmo,
3288         .dev_loss_tmo            = &srp_dev_loss_tmo,
3289         .reconnect               = srp_rport_reconnect,
3290         .rport_delete            = srp_rport_delete,
3291         .terminate_rport_io      = srp_terminate_io,
3292 };
3293
3294 static int __init srp_init_module(void)
3295 {
3296         int ret;
3297
3298         BUILD_BUG_ON(FIELD_SIZEOF(struct ib_wc, wr_id) < sizeof(void *));
3299
3300         if (srp_sg_tablesize) {
3301                 pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n");
3302                 if (!cmd_sg_entries)
3303                         cmd_sg_entries = srp_sg_tablesize;
3304         }
3305
3306         if (!cmd_sg_entries)
3307                 cmd_sg_entries = SRP_DEF_SG_TABLESIZE;
3308
3309         if (cmd_sg_entries > 255) {
3310                 pr_warn("Clamping cmd_sg_entries to 255\n");
3311                 cmd_sg_entries = 255;
3312         }
3313
3314         if (!indirect_sg_entries)
3315                 indirect_sg_entries = cmd_sg_entries;
3316         else if (indirect_sg_entries < cmd_sg_entries) {
3317                 pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n",
3318                         cmd_sg_entries);
3319                 indirect_sg_entries = cmd_sg_entries;
3320         }
3321
3322         srp_remove_wq = create_workqueue("srp_remove");
3323         if (!srp_remove_wq) {
3324                 ret = -ENOMEM;
3325                 goto out;
3326         }
3327
3328         ret = -ENOMEM;
3329         ib_srp_transport_template =
3330                 srp_attach_transport(&ib_srp_transport_functions);
3331         if (!ib_srp_transport_template)
3332                 goto destroy_wq;
3333
3334         ret = class_register(&srp_class);
3335         if (ret) {
3336                 pr_err("couldn't register class infiniband_srp\n");
3337                 goto release_tr;
3338         }
3339
3340         ib_sa_register_client(&srp_sa_client);
3341
3342         ret = ib_register_client(&srp_client);
3343         if (ret) {
3344                 pr_err("couldn't register IB client\n");
3345                 goto unreg_sa;
3346         }
3347
3348 out:
3349         return ret;
3350
3351 unreg_sa:
3352         ib_sa_unregister_client(&srp_sa_client);
3353         class_unregister(&srp_class);
3354
3355 release_tr:
3356         srp_release_transport(ib_srp_transport_template);
3357
3358 destroy_wq:
3359         destroy_workqueue(srp_remove_wq);
3360         goto out;
3361 }
3362
3363 static void __exit srp_cleanup_module(void)
3364 {
3365         ib_unregister_client(&srp_client);
3366         ib_sa_unregister_client(&srp_sa_client);
3367         class_unregister(&srp_class);
3368         srp_release_transport(ib_srp_transport_template);
3369         destroy_workqueue(srp_remove_wq);
3370 }
3371
3372 module_init(srp_init_module);
3373 module_exit(srp_cleanup_module);