drivers/vhost/scsi.c

   1 /*******************************************************************************
   2  * Vhost kernel TCM fabric driver for virtio SCSI initiators
   3  *
   4  * (C) Copyright 2010-2013 Datera, Inc.
   5  * (C) Copyright 2010-2012 IBM Corp.
   6  *
   7  * Licensed to the Linux Foundation under the General Public License (GPL) version 2.
   8  *
   9  * Authors: Nicholas A. Bellinger <nab@daterainc.com>
  10  *          Stefan Hajnoczi <stefanha@linux.vnet.ibm.com>
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License as published by
  14  * the Free Software Foundation; either version 2 of the License, or
  15  * (at your option) any later version.
  16  *
  17  * This program is distributed in the hope that it will be useful,
  18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20  * GNU General Public License for more details.
  21  *
  22  ****************************************************************************/
  23
  24 #include <linux/module.h>
  25 #include <linux/moduleparam.h>
  26 #include <generated/utsrelease.h>
  27 #include <linux/utsname.h>
  28 #include <linux/init.h>
  29 #include <linux/slab.h>
  30 #include <linux/kthread.h>
  31 #include <linux/types.h>
  32 #include <linux/string.h>
  33 #include <linux/configfs.h>
  34 #include <linux/ctype.h>
  35 #include <linux/compat.h>
  36 #include <linux/eventfd.h>
  37 #include <linux/fs.h>
  38 #include <linux/miscdevice.h>
  39 #include <asm/unaligned.h>
  40 #include <scsi/scsi.h>
  41 #include <scsi/scsi_tcq.h>
  42 #include <target/target_core_base.h>
  43 #include <target/target_core_fabric.h>
  44 #include <target/target_core_fabric_configfs.h>
  45 #include <target/target_core_configfs.h>
  46 #include <target/configfs_macros.h>
  47 #include <linux/vhost.h>
  48 #include <linux/virtio_scsi.h>
  49 #include <linux/llist.h>
  50 #include <linux/bitmap.h>
  51 #include <linux/percpu_ida.h>
  52
  53 #include "vhost.h"
  54
  55 #define TCM_VHOST_VERSION  "v0.1"
  56 #define TCM_VHOST_NAMELEN 256
  57 #define TCM_VHOST_MAX_CDB_SIZE 32
  58 #define TCM_VHOST_DEFAULT_TAGS 256
  59 #define TCM_VHOST_PREALLOC_SGLS 2048
  60 #define TCM_VHOST_PREALLOC_PAGES 2048
  61
  62 struct vhost_scsi_inflight {
  63         /* Wait for the flush operation to finish */
  64         struct completion comp;
  65         /* Refcount for the inflight reqs */
  66         struct kref kref;
  67 };
  68
  69 struct tcm_vhost_cmd {
  70         /* Descriptor from vhost_get_vq_desc() for virt_queue segment */
  71         int tvc_vq_desc;
  72         /* virtio-scsi initiator task attribute */
  73         int tvc_task_attr;
  74         /* virtio-scsi initiator data direction */
  75         enum dma_data_direction tvc_data_direction;
  76         /* Expected data transfer length from virtio-scsi header */
  77         u32 tvc_exp_data_len;
  78         /* The Tag from include/linux/virtio_scsi.h:struct virtio_scsi_cmd_req */
  79         u64 tvc_tag;
  80         /* The number of scatterlists associated with this cmd */
  81         u32 tvc_sgl_count;
  82         /* Saved unpacked SCSI LUN for tcm_vhost_submission_work() */
  83         u32 tvc_lun;
  84         /* Pointer to the SGL formatted memory from virtio-scsi */
  85         struct scatterlist *tvc_sgl;
  86         struct page **tvc_upages;
  87         /* Pointer to response */
  88         struct virtio_scsi_cmd_resp __user *tvc_resp;
  89         /* Pointer to vhost_scsi for our device */
  90         struct vhost_scsi *tvc_vhost;
  91         /* Pointer to vhost_virtqueue for the cmd */
  92         struct vhost_virtqueue *tvc_vq;
  93         /* Pointer to vhost nexus memory */
  94         struct tcm_vhost_nexus *tvc_nexus;
  95         /* The TCM I/O descriptor that is accessed via container_of() */
  96         struct se_cmd tvc_se_cmd;
  97         /* work item used for cmwq dispatch to tcm_vhost_submission_work() */
  98         struct work_struct work;
  99         /* Copy of the incoming SCSI command descriptor block (CDB) */
 100         unsigned char tvc_cdb[TCM_VHOST_MAX_CDB_SIZE];
 101         /* Sense buffer that will be mapped into outgoing status */
 102         unsigned char tvc_sense_buf[TRANSPORT_SENSE_BUFFER];
 103         /* Completed commands list, serviced from vhost worker thread */
 104         struct llist_node tvc_completion_list;
 105         /* Used to track inflight cmd */
 106         struct vhost_scsi_inflight *inflight;
 107 };
 108
 109 struct tcm_vhost_nexus {
 110         /* Pointer to TCM session for I_T Nexus */
 111         struct se_session *tvn_se_sess;
 112 };
 113
 114 struct tcm_vhost_nacl {
 115         /* Binary World Wide unique Port Name for Vhost Initiator port */
 116         u64 iport_wwpn;
 117         /* ASCII formatted WWPN for Sas Initiator port */
 118         char iport_name[TCM_VHOST_NAMELEN];
 119         /* Returned by tcm_vhost_make_nodeacl() */
 120         struct se_node_acl se_node_acl;
 121 };
 122
 123 struct tcm_vhost_tpg {
 124         /* Vhost port target portal group tag for TCM */
 125         u16 tport_tpgt;
 126         /* Used to track number of TPG Port/Lun Links wrt to explict I_T Nexus shutdown */
 127         int tv_tpg_port_count;
 128         /* Used for vhost_scsi device reference to tpg_nexus, protected by tv_tpg_mutex */
 129         int tv_tpg_vhost_count;
 130         /* list for tcm_vhost_list */
 131         struct list_head tv_tpg_list;
 132         /* Used to protect access for tpg_nexus */
 133         struct mutex tv_tpg_mutex;
 134         /* Pointer to the TCM VHost I_T Nexus for this TPG endpoint */
 135         struct tcm_vhost_nexus *tpg_nexus;
 136         /* Pointer back to tcm_vhost_tport */
 137         struct tcm_vhost_tport *tport;
 138         /* Returned by tcm_vhost_make_tpg() */
 139         struct se_portal_group se_tpg;
 140         /* Pointer back to vhost_scsi, protected by tv_tpg_mutex */
 141         struct vhost_scsi *vhost_scsi;
 142 };
 143
 144 struct tcm_vhost_tport {
 145         /* SCSI protocol the tport is providing */
 146         u8 tport_proto_id;
 147         /* Binary World Wide unique Port Name for Vhost Target port */
 148         u64 tport_wwpn;
 149         /* ASCII formatted WWPN for Vhost Target port */
 150         char tport_name[TCM_VHOST_NAMELEN];
 151         /* Returned by tcm_vhost_make_tport() */
 152         struct se_wwn tport_wwn;
 153 };
 154
 155 struct tcm_vhost_evt {
 156         /* event to be sent to guest */
 157         struct virtio_scsi_event event;
 158         /* event list, serviced from vhost worker thread */
 159         struct llist_node list;
 160 };
 161
 162 enum {
 163         VHOST_SCSI_VQ_CTL = 0,
 164         VHOST_SCSI_VQ_EVT = 1,
 165         VHOST_SCSI_VQ_IO = 2,
 166 };
 167
 168 enum {
 169         VHOST_SCSI_FEATURES = VHOST_FEATURES | (1ULL << VIRTIO_SCSI_F_HOTPLUG)
 170 };
 171
 172 #define VHOST_SCSI_MAX_TARGET   256
 173 #define VHOST_SCSI_MAX_VQ       128
 174 #define VHOST_SCSI_MAX_EVENT    128
 175
 176 struct vhost_scsi_virtqueue {
 177         struct vhost_virtqueue vq;
 178         /*
 179          * Reference counting for inflight reqs, used for flush operation. At
 180          * each time, one reference tracks new commands submitted, while we
 181          * wait for another one to reach 0.
 182          */
 183         struct vhost_scsi_inflight inflights[2];
 184         /*
 185          * Indicate current inflight in use, protected by vq->mutex.
 186          * Writers must also take dev mutex and flush under it.
 187          */
 188         int inflight_idx;
 189 };
 190
 191 struct vhost_scsi {
 192         /* Protected by vhost_scsi->dev.mutex */
 193         struct tcm_vhost_tpg **vs_tpg;
 194         char vs_vhost_wwpn[TRANSPORT_IQN_LEN];
 195
 196         struct vhost_dev dev;
 197         struct vhost_scsi_virtqueue vqs[VHOST_SCSI_MAX_VQ];
 198
 199         struct vhost_work vs_completion_work; /* cmd completion work item */
 200         struct llist_head vs_completion_list; /* cmd completion queue */
 201
 202         struct vhost_work vs_event_work; /* evt injection work item */
 203         struct llist_head vs_event_list; /* evt injection queue */
 204
 205         bool vs_events_missed; /* any missed events, protected by vq->mutex */
 206         int vs_events_nr; /* num of pending events, protected by vq->mutex */
 207 };
 208
 209 /* Local pointer to allocated TCM configfs fabric module */
 210 static struct target_fabric_configfs *tcm_vhost_fabric_configfs;
 211
 212 static struct workqueue_struct *tcm_vhost_workqueue;
 213
 214 /* Global spinlock to protect tcm_vhost TPG list for vhost IOCTL access */
 215 static DEFINE_MUTEX(tcm_vhost_mutex);
 216 static LIST_HEAD(tcm_vhost_list);
 217
 218 static int iov_num_pages(struct iovec *iov)
 219 {
 220         return (PAGE_ALIGN((unsigned long)iov->iov_base + iov->iov_len) -
 221                ((unsigned long)iov->iov_base & PAGE_MASK)) >> PAGE_SHIFT;
 222 }
 223
 224 static void tcm_vhost_done_inflight(struct kref *kref)
 225 {
 226         struct vhost_scsi_inflight *inflight;
 227
 228         inflight = container_of(kref, struct vhost_scsi_inflight, kref);
 229         complete(&inflight->comp);
 230 }
 231
 232 static void tcm_vhost_init_inflight(struct vhost_scsi *vs,
 233                                     struct vhost_scsi_inflight *old_inflight[])
 234 {
 235         struct vhost_scsi_inflight *new_inflight;
 236         struct vhost_virtqueue *vq;
 237         int idx, i;
 238
 239         for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
 240                 vq = &vs->vqs[i].vq;
 241
 242                 mutex_lock(&vq->mutex);
 243
 244                 /* store old infight */
 245                 idx = vs->vqs[i].inflight_idx;
 246                 if (old_inflight)
 247                         old_inflight[i] = &vs->vqs[i].inflights[idx];
 248
 249                 /* setup new infight */
 250                 vs->vqs[i].inflight_idx = idx ^ 1;
 251                 new_inflight = &vs->vqs[i].inflights[idx ^ 1];
 252                 kref_init(&new_inflight->kref);
 253                 init_completion(&new_inflight->comp);
 254
 255                 mutex_unlock(&vq->mutex);
 256         }
 257 }
 258
 259 static struct vhost_scsi_inflight *
 260 tcm_vhost_get_inflight(struct vhost_virtqueue *vq)
 261 {
 262         struct vhost_scsi_inflight *inflight;
 263         struct vhost_scsi_virtqueue *svq;
 264
 265         svq = container_of(vq, struct vhost_scsi_virtqueue, vq);
 266         inflight = &svq->inflights[svq->inflight_idx];
 267         kref_get(&inflight->kref);
 268
 269         return inflight;
 270 }
 271
 272 static void tcm_vhost_put_inflight(struct vhost_scsi_inflight *inflight)
 273 {
 274         kref_put(&inflight->kref, tcm_vhost_done_inflight);
 275 }
 276
 277 static int tcm_vhost_check_true(struct se_portal_group *se_tpg)
 278 {
 279         return 1;
 280 }
 281
 282 static int tcm_vhost_check_false(struct se_portal_group *se_tpg)
 283 {
 284         return 0;
 285 }
 286
 287 static char *tcm_vhost_get_fabric_name(void)
 288 {
 289         return "vhost";
 290 }
 291
 292 static u8 tcm_vhost_get_fabric_proto_ident(struct se_portal_group *se_tpg)
 293 {
 294         struct tcm_vhost_tpg *tpg = container_of(se_tpg,
 295                                 struct tcm_vhost_tpg, se_tpg);
 296         struct tcm_vhost_tport *tport = tpg->tport;
 297
 298         switch (tport->tport_proto_id) {
 299         case SCSI_PROTOCOL_SAS:
 300                 return sas_get_fabric_proto_ident(se_tpg);
 301         case SCSI_PROTOCOL_FCP:
 302                 return fc_get_fabric_proto_ident(se_tpg);
 303         case SCSI_PROTOCOL_ISCSI:
 304                 return iscsi_get_fabric_proto_ident(se_tpg);
 305         default:
 306                 pr_err("Unknown tport_proto_id: 0x%02x, using"
 307                         " SAS emulation\n", tport->tport_proto_id);
 308                 break;
 309         }
 310
 311         return sas_get_fabric_proto_ident(se_tpg);
 312 }
 313
 314 static char *tcm_vhost_get_fabric_wwn(struct se_portal_group *se_tpg)
 315 {
 316         struct tcm_vhost_tpg *tpg = container_of(se_tpg,
 317                                 struct tcm_vhost_tpg, se_tpg);
 318         struct tcm_vhost_tport *tport = tpg->tport;
 319
 320         return &tport->tport_name[0];
 321 }
 322
 323 static u16 tcm_vhost_get_tag(struct se_portal_group *se_tpg)
 324 {
 325         struct tcm_vhost_tpg *tpg = container_of(se_tpg,
 326                                 struct tcm_vhost_tpg, se_tpg);
 327         return tpg->tport_tpgt;
 328 }
 329
 330 static u32 tcm_vhost_get_default_depth(struct se_portal_group *se_tpg)
 331 {
 332         return 1;
 333 }
 334
 335 static u32
 336 tcm_vhost_get_pr_transport_id(struct se_portal_group *se_tpg,
 337                               struct se_node_acl *se_nacl,
 338                               struct t10_pr_registration *pr_reg,
 339                               int *format_code,
 340                               unsigned char *buf)
 341 {
 342         struct tcm_vhost_tpg *tpg = container_of(se_tpg,
 343                                 struct tcm_vhost_tpg, se_tpg);
 344         struct tcm_vhost_tport *tport = tpg->tport;
 345
 346         switch (tport->tport_proto_id) {
 347         case SCSI_PROTOCOL_SAS:
 348                 return sas_get_pr_transport_id(se_tpg, se_nacl, pr_reg,
 349                                         format_code, buf);
 350         case SCSI_PROTOCOL_FCP:
 351                 return fc_get_pr_transport_id(se_tpg, se_nacl, pr_reg,
 352                                         format_code, buf);
 353         case SCSI_PROTOCOL_ISCSI:
 354                 return iscsi_get_pr_transport_id(se_tpg, se_nacl, pr_reg,
 355                                         format_code, buf);
 356         default:
 357                 pr_err("Unknown tport_proto_id: 0x%02x, using"
 358                         " SAS emulation\n", tport->tport_proto_id);
 359                 break;
 360         }
 361
 362         return sas_get_pr_transport_id(se_tpg, se_nacl, pr_reg,
 363                         format_code, buf);
 364 }
 365
 366 static u32
 367 tcm_vhost_get_pr_transport_id_len(struct se_portal_group *se_tpg,
 368                                   struct se_node_acl *se_nacl,
 369                                   struct t10_pr_registration *pr_reg,
 370                                   int *format_code)
 371 {
 372         struct tcm_vhost_tpg *tpg = container_of(se_tpg,
 373                                 struct tcm_vhost_tpg, se_tpg);
 374         struct tcm_vhost_tport *tport = tpg->tport;
 375
 376         switch (tport->tport_proto_id) {
 377         case SCSI_PROTOCOL_SAS:
 378                 return sas_get_pr_transport_id_len(se_tpg, se_nacl, pr_reg,
 379                                         format_code);
 380         case SCSI_PROTOCOL_FCP:
 381                 return fc_get_pr_transport_id_len(se_tpg, se_nacl, pr_reg,
 382                                         format_code);
 383         case SCSI_PROTOCOL_ISCSI:
 384                 return iscsi_get_pr_transport_id_len(se_tpg, se_nacl, pr_reg,
 385                                         format_code);
 386         default:
 387                 pr_err("Unknown tport_proto_id: 0x%02x, using"
 388                         " SAS emulation\n", tport->tport_proto_id);
 389                 break;
 390         }
 391
 392         return sas_get_pr_transport_id_len(se_tpg, se_nacl, pr_reg,
 393                         format_code);
 394 }
 395
 396 static char *
 397 tcm_vhost_parse_pr_out_transport_id(struct se_portal_group *se_tpg,
 398                                     const char *buf,
 399                                     u32 *out_tid_len,
 400                                     char **port_nexus_ptr)
 401 {
 402         struct tcm_vhost_tpg *tpg = container_of(se_tpg,
 403                                 struct tcm_vhost_tpg, se_tpg);
 404         struct tcm_vhost_tport *tport = tpg->tport;
 405
 406         switch (tport->tport_proto_id) {
 407         case SCSI_PROTOCOL_SAS:
 408                 return sas_parse_pr_out_transport_id(se_tpg, buf, out_tid_len,
 409                                         port_nexus_ptr);
 410         case SCSI_PROTOCOL_FCP:
 411                 return fc_parse_pr_out_transport_id(se_tpg, buf, out_tid_len,
 412                                         port_nexus_ptr);
 413         case SCSI_PROTOCOL_ISCSI:
 414                 return iscsi_parse_pr_out_transport_id(se_tpg, buf, out_tid_len,
 415                                         port_nexus_ptr);
 416         default:
 417                 pr_err("Unknown tport_proto_id: 0x%02x, using"
 418                         " SAS emulation\n", tport->tport_proto_id);
 419                 break;
 420         }
 421
 422         return sas_parse_pr_out_transport_id(se_tpg, buf, out_tid_len,
 423                         port_nexus_ptr);
 424 }
 425
 426 static struct se_node_acl *
 427 tcm_vhost_alloc_fabric_acl(struct se_portal_group *se_tpg)
 428 {
 429         struct tcm_vhost_nacl *nacl;
 430
 431         nacl = kzalloc(sizeof(struct tcm_vhost_nacl), GFP_KERNEL);
 432         if (!nacl) {
 433                 pr_err("Unable to allocate struct tcm_vhost_nacl\n");
 434                 return NULL;
 435         }
 436
 437         return &nacl->se_node_acl;
 438 }
 439
 440 static void
 441 tcm_vhost_release_fabric_acl(struct se_portal_group *se_tpg,
 442                              struct se_node_acl *se_nacl)
 443 {
 444         struct tcm_vhost_nacl *nacl = container_of(se_nacl,
 445                         struct tcm_vhost_nacl, se_node_acl);
 446         kfree(nacl);
 447 }
 448
 449 static u32 tcm_vhost_tpg_get_inst_index(struct se_portal_group *se_tpg)
 450 {
 451         return 1;
 452 }
 453
 454 static void tcm_vhost_release_cmd(struct se_cmd *se_cmd)
 455 {
 456         struct tcm_vhost_cmd *tv_cmd = container_of(se_cmd,
 457                                 struct tcm_vhost_cmd, tvc_se_cmd);
 458         struct se_session *se_sess = se_cmd->se_sess;
 459
 460         if (tv_cmd->tvc_sgl_count) {
 461                 u32 i;
 462                 for (i = 0; i < tv_cmd->tvc_sgl_count; i++)
 463                         put_page(sg_page(&tv_cmd->tvc_sgl[i]));
 464         }
 465
 466         tcm_vhost_put_inflight(tv_cmd->inflight);
 467         percpu_ida_free(&se_sess->sess_tag_pool, se_cmd->map_tag);
 468 }
 469
 470 static int tcm_vhost_shutdown_session(struct se_session *se_sess)
 471 {
 472         return 0;
 473 }
 474
 475 static void tcm_vhost_close_session(struct se_session *se_sess)
 476 {
 477         return;
 478 }
 479
 480 static u32 tcm_vhost_sess_get_index(struct se_session *se_sess)
 481 {
 482         return 0;
 483 }
 484
 485 static int tcm_vhost_write_pending(struct se_cmd *se_cmd)
 486 {
 487         /* Go ahead and process the write immediately */
 488         target_execute_cmd(se_cmd);
 489         return 0;
 490 }
 491
 492 static int tcm_vhost_write_pending_status(struct se_cmd *se_cmd)
 493 {
 494         return 0;
 495 }
 496
 497 static void tcm_vhost_set_default_node_attrs(struct se_node_acl *nacl)
 498 {
 499         return;
 500 }
 501
 502 static u32 tcm_vhost_get_task_tag(struct se_cmd *se_cmd)
 503 {
 504         return 0;
 505 }
 506
 507 static int tcm_vhost_get_cmd_state(struct se_cmd *se_cmd)
 508 {
 509         return 0;
 510 }
 511
 512 static void vhost_scsi_complete_cmd(struct tcm_vhost_cmd *cmd)
 513 {
 514         struct vhost_scsi *vs = cmd->tvc_vhost;
 515
 516         llist_add(&cmd->tvc_completion_list, &vs->vs_completion_list);
 517
 518         vhost_work_queue(&vs->dev, &vs->vs_completion_work);
 519 }
 520
 521 static int tcm_vhost_queue_data_in(struct se_cmd *se_cmd)
 522 {
 523         struct tcm_vhost_cmd *cmd = container_of(se_cmd,
 524                                 struct tcm_vhost_cmd, tvc_se_cmd);
 525         vhost_scsi_complete_cmd(cmd);
 526         return 0;
 527 }
 528
 529 static int tcm_vhost_queue_status(struct se_cmd *se_cmd)
 530 {
 531         struct tcm_vhost_cmd *cmd = container_of(se_cmd,
 532                                 struct tcm_vhost_cmd, tvc_se_cmd);
 533         vhost_scsi_complete_cmd(cmd);
 534         return 0;
 535 }
 536
 537 static void tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd)
 538 {
 539         return;
 540 }
 541
 542 static void tcm_vhost_free_evt(struct vhost_scsi *vs, struct tcm_vhost_evt *evt)
 543 {
 544         vs->vs_events_nr--;
 545         kfree(evt);
 546 }
 547
 548 static struct tcm_vhost_evt *
 549 tcm_vhost_allocate_evt(struct vhost_scsi *vs,
 550                        u32 event, u32 reason)
 551 {
 552         struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
 553         struct tcm_vhost_evt *evt;
 554
 555         if (vs->vs_events_nr > VHOST_SCSI_MAX_EVENT) {
 556                 vs->vs_events_missed = true;
 557                 return NULL;
 558         }
 559
 560         evt = kzalloc(sizeof(*evt), GFP_KERNEL);
 561         if (!evt) {
 562                 vq_err(vq, "Failed to allocate tcm_vhost_evt\n");
 563                 vs->vs_events_missed = true;
 564                 return NULL;
 565         }
 566
 567         evt->event.event = event;
 568         evt->event.reason = reason;
 569         vs->vs_events_nr++;
 570
 571         return evt;
 572 }
 573
 574 static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *cmd)
 575 {
 576         struct se_cmd *se_cmd = &cmd->tvc_se_cmd;
 577
 578         /* TODO locking against target/backend threads? */
 579         transport_generic_free_cmd(se_cmd, 0);
 580
 581 }
 582
 583 static int vhost_scsi_check_stop_free(struct se_cmd *se_cmd)
 584 {
 585         return target_put_sess_cmd(se_cmd->se_sess, se_cmd);
 586 }
 587
 588 static void
 589 tcm_vhost_do_evt_work(struct vhost_scsi *vs, struct tcm_vhost_evt *evt)
 590 {
 591         struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
 592         struct virtio_scsi_event *event = &evt->event;
 593         struct virtio_scsi_event __user *eventp;
 594         unsigned out, in;
 595         int head, ret;
 596
 597         if (!vq->private_data) {
 598                 vs->vs_events_missed = true;
 599                 return;
 600         }
 601
 602 again:
 603         vhost_disable_notify(&vs->dev, vq);
 604         head = vhost_get_vq_desc(&vs->dev, vq, vq->iov,
 605                         ARRAY_SIZE(vq->iov), &out, &in,
 606                         NULL, NULL);
 607         if (head < 0) {
 608                 vs->vs_events_missed = true;
 609                 return;
 610         }
 611         if (head == vq->num) {
 612                 if (vhost_enable_notify(&vs->dev, vq))
 613                         goto again;
 614                 vs->vs_events_missed = true;
 615                 return;
 616         }
 617
 618         if ((vq->iov[out].iov_len != sizeof(struct virtio_scsi_event))) {
 619                 vq_err(vq, "Expecting virtio_scsi_event, got %zu bytes\n",
 620                                 vq->iov[out].iov_len);
 621                 vs->vs_events_missed = true;
 622                 return;
 623         }
 624
 625         if (vs->vs_events_missed) {
 626                 event->event |= VIRTIO_SCSI_T_EVENTS_MISSED;
 627                 vs->vs_events_missed = false;
 628         }
 629
 630         eventp = vq->iov[out].iov_base;
 631         ret = __copy_to_user(eventp, event, sizeof(*event));
 632         if (!ret)
 633                 vhost_add_used_and_signal(&vs->dev, vq, head, 0);
 634         else
 635                 vq_err(vq, "Faulted on tcm_vhost_send_event\n");
 636 }
 637
 638 static void tcm_vhost_evt_work(struct vhost_work *work)
 639 {
 640         struct vhost_scsi *vs = container_of(work, struct vhost_scsi,
 641                                         vs_event_work);
 642         struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
 643         struct tcm_vhost_evt *evt;
 644         struct llist_node *llnode;
 645
 646         mutex_lock(&vq->mutex);
 647         llnode = llist_del_all(&vs->vs_event_list);
 648         while (llnode) {
 649                 evt = llist_entry(llnode, struct tcm_vhost_evt, list);
 650                 llnode = llist_next(llnode);
 651                 tcm_vhost_do_evt_work(vs, evt);
 652                 tcm_vhost_free_evt(vs, evt);
 653         }
 654         mutex_unlock(&vq->mutex);
 655 }
 656
 657 /* Fill in status and signal that we are done processing this command
 658  *
 659  * This is scheduled in the vhost work queue so we are called with the owner
 660  * process mm and can access the vring.
 661  */
 662 static void vhost_scsi_complete_cmd_work(struct vhost_work *work)
 663 {
 664         struct vhost_scsi *vs = container_of(work, struct vhost_scsi,
 665                                         vs_completion_work);
 666         DECLARE_BITMAP(signal, VHOST_SCSI_MAX_VQ);
 667         struct virtio_scsi_cmd_resp v_rsp;
 668         struct tcm_vhost_cmd *cmd;
 669         struct llist_node *llnode;
 670         struct se_cmd *se_cmd;
 671         int ret, vq;
 672
 673         bitmap_zero(signal, VHOST_SCSI_MAX_VQ);
 674         llnode = llist_del_all(&vs->vs_completion_list);
 675         while (llnode) {
 676                 cmd = llist_entry(llnode, struct tcm_vhost_cmd,
 677                                      tvc_completion_list);
 678                 llnode = llist_next(llnode);
 679                 se_cmd = &cmd->tvc_se_cmd;
 680
 681                 pr_debug("%s tv_cmd %p resid %u status %#02x\n", __func__,
 682                         cmd, se_cmd->residual_count, se_cmd->scsi_status);
 683
 684                 memset(&v_rsp, 0, sizeof(v_rsp));
 685                 v_rsp.resid = se_cmd->residual_count;
 686                 /* TODO is status_qualifier field needed? */
 687                 v_rsp.status = se_cmd->scsi_status;
 688                 v_rsp.sense_len = se_cmd->scsi_sense_length;
 689                 memcpy(v_rsp.sense, cmd->tvc_sense_buf,
 690                        v_rsp.sense_len);
 691                 ret = copy_to_user(cmd->tvc_resp, &v_rsp, sizeof(v_rsp));
 692                 if (likely(ret == 0)) {
 693                         struct vhost_scsi_virtqueue *q;
 694                         vhost_add_used(cmd->tvc_vq, cmd->tvc_vq_desc, 0);
 695                         q = container_of(cmd->tvc_vq, struct vhost_scsi_virtqueue, vq);
 696                         vq = q - vs->vqs;
 697                         __set_bit(vq, signal);
 698                 } else
 699                         pr_err("Faulted on virtio_scsi_cmd_resp\n");
 700
 701                 vhost_scsi_free_cmd(cmd);
 702         }
 703
 704         vq = -1;
 705         while ((vq = find_next_bit(signal, VHOST_SCSI_MAX_VQ, vq + 1))
 706                 < VHOST_SCSI_MAX_VQ)
 707                 vhost_signal(&vs->dev, &vs->vqs[vq].vq);
 708 }
 709
 710 static struct tcm_vhost_cmd *
 711 vhost_scsi_get_tag(struct vhost_virtqueue *vq,
 712                         struct tcm_vhost_tpg *tpg,
 713                         struct virtio_scsi_cmd_req *v_req,
 714                         u32 exp_data_len,
 715                         int data_direction)
 716 {
 717         struct tcm_vhost_cmd *cmd;
 718         struct tcm_vhost_nexus *tv_nexus;
 719         struct se_session *se_sess;
 720         struct scatterlist *sg;
 721         struct page **pages;
 722         int tag;
 723
 724         tv_nexus = tpg->tpg_nexus;
 725         if (!tv_nexus) {
 726                 pr_err("Unable to locate active struct tcm_vhost_nexus\n");
 727                 return ERR_PTR(-EIO);
 728         }
 729         se_sess = tv_nexus->tvn_se_sess;
 730
 731         tag = percpu_ida_alloc(&se_sess->sess_tag_pool, GFP_KERNEL);
 732         cmd = &((struct tcm_vhost_cmd *)se_sess->sess_cmd_map)[tag];
 733         sg = cmd->tvc_sgl;
 734         pages = cmd->tvc_upages;
 735         memset(cmd, 0, sizeof(struct tcm_vhost_cmd));
 736
 737         cmd->tvc_sgl = sg;
 738         cmd->tvc_upages = pages;
 739         cmd->tvc_se_cmd.map_tag = tag;
 740         cmd->tvc_tag = v_req->tag;
 741         cmd->tvc_task_attr = v_req->task_attr;
 742         cmd->tvc_exp_data_len = exp_data_len;
 743         cmd->tvc_data_direction = data_direction;
 744         cmd->tvc_nexus = tv_nexus;
 745         cmd->inflight = tcm_vhost_get_inflight(vq);
 746
 747         return cmd;
 748 }
 749
 750 /*
 751  * Map a user memory range into a scatterlist
 752  *
 753  * Returns the number of scatterlist entries used or -errno on error.
 754  */
 755 static int
 756 vhost_scsi_map_to_sgl(struct tcm_vhost_cmd *tv_cmd,
 757                       struct scatterlist *sgl,
 758                       unsigned int sgl_count,
 759                       struct iovec *iov,
 760                       int write)
 761 {
 762         unsigned int npages = 0, pages_nr, offset, nbytes;
 763         struct scatterlist *sg = sgl;
 764         void __user *ptr = iov->iov_base;
 765         size_t len = iov->iov_len;
 766         struct page **pages;
 767         int ret, i;
 768
 769         if (sgl_count > TCM_VHOST_PREALLOC_SGLS) {
 770                 pr_err("vhost_scsi_map_to_sgl() psgl_count: %u greater than"
 771                        " preallocated TCM_VHOST_PREALLOC_SGLS: %u\n",
 772                         sgl_count, TCM_VHOST_PREALLOC_SGLS);
 773                 return -ENOBUFS;
 774         }
 775
 776         pages_nr = iov_num_pages(iov);
 777         if (pages_nr > sgl_count)
 778                 return -ENOBUFS;
 779
 780         if (pages_nr > TCM_VHOST_PREALLOC_PAGES) {
 781                 pr_err("vhost_scsi_map_to_sgl() pages_nr: %u greater than"
 782                        " preallocated TCM_VHOST_PREALLOC_PAGES: %u\n",
 783                         pages_nr, TCM_VHOST_PREALLOC_PAGES);
 784                 return -ENOBUFS;
 785         }
 786
 787         pages = tv_cmd->tvc_upages;
 788
 789         ret = get_user_pages_fast((unsigned long)ptr, pages_nr, write, pages);
 790         /* No pages were pinned */
 791         if (ret < 0)
 792                 goto out;
 793         /* Less pages pinned than wanted */
 794         if (ret != pages_nr) {
 795                 for (i = 0; i < ret; i++)
 796                         put_page(pages[i]);
 797                 ret = -EFAULT;
 798                 goto out;
 799         }
 800
 801         while (len > 0) {
 802                 offset = (uintptr_t)ptr & ~PAGE_MASK;
 803                 nbytes = min_t(unsigned int, PAGE_SIZE - offset, len);
 804                 sg_set_page(sg, pages[npages], nbytes, offset);
 805                 ptr += nbytes;
 806                 len -= nbytes;
 807                 sg++;
 808                 npages++;
 809         }
 810
 811 out:
 812         return ret;
 813 }
 814
 815 static int
 816 vhost_scsi_map_iov_to_sgl(struct tcm_vhost_cmd *cmd,
 817                           struct iovec *iov,
 818                           unsigned int niov,
 819                           int write)
 820 {
 821         int ret;
 822         unsigned int i;
 823         u32 sgl_count;
 824         struct scatterlist *sg;
 825
 826         /*
 827          * Find out how long sglist needs to be
 828          */
 829         sgl_count = 0;
 830         for (i = 0; i < niov; i++)
 831                 sgl_count += iov_num_pages(&iov[i]);
 832
 833         /* TODO overflow checking */
 834
 835         sg = cmd->tvc_sgl;
 836         pr_debug("%s sg %p sgl_count %u\n", __func__, sg, sgl_count);
 837         sg_init_table(sg, sgl_count);
 838
 839         cmd->tvc_sgl_count = sgl_count;
 840
 841         pr_debug("Mapping %u iovecs for %u pages\n", niov, sgl_count);
 842         for (i = 0; i < niov; i++) {
 843                 ret = vhost_scsi_map_to_sgl(cmd, sg, sgl_count, &iov[i],
 844                                             write);
 845                 if (ret < 0) {
 846                         for (i = 0; i < cmd->tvc_sgl_count; i++)
 847                                 put_page(sg_page(&cmd->tvc_sgl[i]));
 848
 849                         cmd->tvc_sgl_count = 0;
 850                         return ret;
 851                 }
 852
 853                 sg += ret;
 854                 sgl_count -= ret;
 855         }
 856         return 0;
 857 }
 858
 859 static void tcm_vhost_submission_work(struct work_struct *work)
 860 {
 861         struct tcm_vhost_cmd *cmd =
 862                 container_of(work, struct tcm_vhost_cmd, work);
 863         struct tcm_vhost_nexus *tv_nexus;
 864         struct se_cmd *se_cmd = &cmd->tvc_se_cmd;
 865         struct scatterlist *sg_ptr, *sg_bidi_ptr = NULL;
 866         int rc, sg_no_bidi = 0;
 867
 868         if (cmd->tvc_sgl_count) {
 869                 sg_ptr = cmd->tvc_sgl;
 870 /* FIXME: Fix BIDI operation in tcm_vhost_submission_work() */
 871 #if 0
 872                 if (se_cmd->se_cmd_flags & SCF_BIDI) {
 873                         sg_bidi_ptr = NULL;
 874                         sg_no_bidi = 0;
 875                 }
 876 #endif
 877         } else {
 878                 sg_ptr = NULL;
 879         }
 880         tv_nexus = cmd->tvc_nexus;
 881
 882         rc = target_submit_cmd_map_sgls(se_cmd, tv_nexus->tvn_se_sess,
 883                         cmd->tvc_cdb, &cmd->tvc_sense_buf[0],
 884                         cmd->tvc_lun, cmd->tvc_exp_data_len,
 885                         cmd->tvc_task_attr, cmd->tvc_data_direction,
 886                         TARGET_SCF_ACK_KREF, sg_ptr, cmd->tvc_sgl_count,
 887                         sg_bidi_ptr, sg_no_bidi);
 888         if (rc < 0) {
 889                 transport_send_check_condition_and_sense(se_cmd,
 890                                 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE, 0);
 891                 transport_generic_free_cmd(se_cmd, 0);
 892         }
 893 }
 894
 895 static void
 896 vhost_scsi_send_bad_target(struct vhost_scsi *vs,
 897                            struct vhost_virtqueue *vq,
 898                            int head, unsigned out)
 899 {
 900         struct virtio_scsi_cmd_resp __user *resp;
 901         struct virtio_scsi_cmd_resp rsp;
 902         int ret;
 903
 904         memset(&rsp, 0, sizeof(rsp));
 905         rsp.response = VIRTIO_SCSI_S_BAD_TARGET;
 906         resp = vq->iov[out].iov_base;
 907         ret = __copy_to_user(resp, &rsp, sizeof(rsp));
 908         if (!ret)
 909                 vhost_add_used_and_signal(&vs->dev, vq, head, 0);
 910         else
 911                 pr_err("Faulted on virtio_scsi_cmd_resp\n");
 912 }
 913
 914 static void
 915 vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)
 916 {
 917         struct tcm_vhost_tpg **vs_tpg;
 918         struct virtio_scsi_cmd_req v_req;
 919         struct tcm_vhost_tpg *tpg;
 920         struct tcm_vhost_cmd *cmd;
 921         u32 exp_data_len, data_first, data_num, data_direction;
 922         unsigned out, in, i;
 923         int head, ret;
 924         u8 target;
 925
 926         mutex_lock(&vq->mutex);
 927         /*
 928          * We can handle the vq only after the endpoint is setup by calling the
 929          * VHOST_SCSI_SET_ENDPOINT ioctl.
 930          */
 931         vs_tpg = vq->private_data;
 932         if (!vs_tpg)
 933                 goto out;
 934
 935         vhost_disable_notify(&vs->dev, vq);
 936
 937         for (;;) {
 938                 head = vhost_get_vq_desc(&vs->dev, vq, vq->iov,
 939                                         ARRAY_SIZE(vq->iov), &out, &in,
 940                                         NULL, NULL);
 941                 pr_debug("vhost_get_vq_desc: head: %d, out: %u in: %u\n",
 942                                         head, out, in);
 943                 /* On error, stop handling until the next kick. */
 944                 if (unlikely(head < 0))
 945                         break;
 946                 /* Nothing new?  Wait for eventfd to tell us they refilled. */
 947                 if (head == vq->num) {
 948                         if (unlikely(vhost_enable_notify(&vs->dev, vq))) {
 949                                 vhost_disable_notify(&vs->dev, vq);
 950                                 continue;
 951                         }
 952                         break;
 953                 }
 954
 955 /* FIXME: BIDI operation */
 956                 if (out == 1 && in == 1) {
 957                         data_direction = DMA_NONE;
 958                         data_first = 0;
 959                         data_num = 0;
 960                 } else if (out == 1 && in > 1) {
 961                         data_direction = DMA_FROM_DEVICE;
 962                         data_first = out + 1;
 963                         data_num = in - 1;
 964                 } else if (out > 1 && in == 1) {
 965                         data_direction = DMA_TO_DEVICE;
 966                         data_first = 1;
 967                         data_num = out - 1;
 968                 } else {
 969                         vq_err(vq, "Invalid buffer layout out: %u in: %u\n",
 970                                         out, in);
 971                         break;
 972                 }
 973
 974                 /*
 975                  * Check for a sane resp buffer so we can report errors to
 976                  * the guest.
 977                  */
 978                 if (unlikely(vq->iov[out].iov_len !=
 979                                         sizeof(struct virtio_scsi_cmd_resp))) {
 980                         vq_err(vq, "Expecting virtio_scsi_cmd_resp, got %zu"
 981                                 " bytes\n", vq->iov[out].iov_len);
 982                         break;
 983                 }
 984
 985                 if (unlikely(vq->iov[0].iov_len != sizeof(v_req))) {
 986                         vq_err(vq, "Expecting virtio_scsi_cmd_req, got %zu"
 987                                 " bytes\n", vq->iov[0].iov_len);
 988                         break;
 989                 }
 990                 pr_debug("Calling __copy_from_user: vq->iov[0].iov_base: %p,"
 991                         " len: %zu\n", vq->iov[0].iov_base, sizeof(v_req));
 992                 ret = __copy_from_user(&v_req, vq->iov[0].iov_base,
 993                                 sizeof(v_req));
 994                 if (unlikely(ret)) {
 995                         vq_err(vq, "Faulted on virtio_scsi_cmd_req\n");
 996                         break;
 997                 }
 998
 999                 /* Extract the tpgt */
1000                 target = v_req.lun[1];
1001                 tpg = ACCESS_ONCE(vs_tpg[target]);
1002
1003                 /* Target does not exist, fail the request */
1004                 if (unlikely(!tpg)) {
1005                         vhost_scsi_send_bad_target(vs, vq, head, out);
1006                         continue;
1007                 }
1008
1009                 exp_data_len = 0;
1010                 for (i = 0; i < data_num; i++)
1011                         exp_data_len += vq->iov[data_first + i].iov_len;
1012
1013                 cmd = vhost_scsi_get_tag(vq, tpg, &v_req,
1014                                          exp_data_len, data_direction);
1015                 if (IS_ERR(cmd)) {
1016                         vq_err(vq, "vhost_scsi_get_tag failed %ld\n",
1017                                         PTR_ERR(cmd));
1018                         goto err_cmd;
1019                 }
1020                 pr_debug("Allocated tv_cmd: %p exp_data_len: %d, data_direction"
1021                         ": %d\n", cmd, exp_data_len, data_direction);
1022
1023                 cmd->tvc_vhost = vs;
1024                 cmd->tvc_vq = vq;
1025                 cmd->tvc_resp = vq->iov[out].iov_base;
1026
1027                 /*
1028                  * Copy in the recieved CDB descriptor into cmd->tvc_cdb
1029                  * that will be used by tcm_vhost_new_cmd_map() and down into
1030                  * target_setup_cmd_from_cdb()
1031                  */
1032                 memcpy(cmd->tvc_cdb, v_req.cdb, TCM_VHOST_MAX_CDB_SIZE);
1033                 /*
1034                  * Check that the recieved CDB size does not exceeded our
1035                  * hardcoded max for tcm_vhost
1036                  */
1037                 /* TODO what if cdb was too small for varlen cdb header? */
1038                 if (unlikely(scsi_command_size(cmd->tvc_cdb) >
1039                                         TCM_VHOST_MAX_CDB_SIZE)) {
1040                         vq_err(vq, "Received SCSI CDB with command_size: %d that"
1041                                 " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n",
1042                                 scsi_command_size(cmd->tvc_cdb),
1043                                 TCM_VHOST_MAX_CDB_SIZE);
1044                         goto err_free;
1045                 }
1046                 cmd->tvc_lun = ((v_req.lun[2] << 8) | v_req.lun[3]) & 0x3FFF;
1047
1048                 pr_debug("vhost_scsi got command opcode: %#02x, lun: %d\n",
1049                         cmd->tvc_cdb[0], cmd->tvc_lun);
1050
1051                 if (data_direction != DMA_NONE) {
1052                         ret = vhost_scsi_map_iov_to_sgl(cmd,
1053                                         &vq->iov[data_first], data_num,
1054                                         data_direction == DMA_TO_DEVICE);
1055                         if (unlikely(ret)) {
1056                                 vq_err(vq, "Failed to map iov to sgl\n");
1057                                 goto err_free;
1058                         }
1059                 }
1060
1061                 /*
1062                  * Save the descriptor from vhost_get_vq_desc() to be used to
1063                  * complete the virtio-scsi request in TCM callback context via
1064                  * tcm_vhost_queue_data_in() and tcm_vhost_queue_status()
1065                  */
1066                 cmd->tvc_vq_desc = head;
1067                 /*
1068                  * Dispatch tv_cmd descriptor for cmwq execution in process
1069                  * context provided by tcm_vhost_workqueue.  This also ensures
1070                  * tv_cmd is executed on the same kworker CPU as this vhost
1071                  * thread to gain positive L2 cache locality effects..
1072                  */
1073                 INIT_WORK(&cmd->work, tcm_vhost_submission_work);
1074                 queue_work(tcm_vhost_workqueue, &cmd->work);
1075         }
1076
1077         mutex_unlock(&vq->mutex);
1078         return;
1079
1080 err_free:
1081         vhost_scsi_free_cmd(cmd);
1082 err_cmd:
1083         vhost_scsi_send_bad_target(vs, vq, head, out);
1084 out:
1085         mutex_unlock(&vq->mutex);
1086 }
1087
1088 static void vhost_scsi_ctl_handle_kick(struct vhost_work *work)
1089 {
1090         pr_debug("%s: The handling func for control queue.\n", __func__);
1091 }
1092
1093 static void
1094 tcm_vhost_send_evt(struct vhost_scsi *vs,
1095                    struct tcm_vhost_tpg *tpg,
1096                    struct se_lun *lun,
1097                    u32 event,
1098                    u32 reason)
1099 {
1100         struct tcm_vhost_evt *evt;
1101
1102         evt = tcm_vhost_allocate_evt(vs, event, reason);
1103         if (!evt)
1104                 return;
1105
1106         if (tpg && lun) {
1107                 /* TODO: share lun setup code with virtio-scsi.ko */
1108                 /*
1109                  * Note: evt->event is zeroed when we allocate it and
1110                  * lun[4-7] need to be zero according to virtio-scsi spec.
1111                  */
1112                 evt->event.lun[0] = 0x01;
1113                 evt->event.lun[1] = tpg->tport_tpgt & 0xFF;
1114                 if (lun->unpacked_lun >= 256)
1115                         evt->event.lun[2] = lun->unpacked_lun >> 8 | 0x40 ;
1116                 evt->event.lun[3] = lun->unpacked_lun & 0xFF;
1117         }
1118
1119         llist_add(&evt->list, &vs->vs_event_list);
1120         vhost_work_queue(&vs->dev, &vs->vs_event_work);
1121 }
1122
1123 static void vhost_scsi_evt_handle_kick(struct vhost_work *work)
1124 {
1125         struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1126                                                 poll.work);
1127         struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);
1128
1129         mutex_lock(&vq->mutex);
1130         if (!vq->private_data)
1131                 goto out;
1132
1133         if (vs->vs_events_missed)
1134                 tcm_vhost_send_evt(vs, NULL, NULL, VIRTIO_SCSI_T_NO_EVENT, 0);
1135 out:
1136         mutex_unlock(&vq->mutex);
1137 }
1138
1139 static void vhost_scsi_handle_kick(struct vhost_work *work)
1140 {
1141         struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1142                                                 poll.work);
1143         struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);
1144
1145         vhost_scsi_handle_vq(vs, vq);
1146 }
1147
1148 static void vhost_scsi_flush_vq(struct vhost_scsi *vs, int index)
1149 {
1150         vhost_poll_flush(&vs->vqs[index].vq.poll);
1151 }
1152
1153 /* Callers must hold dev mutex */
1154 static void vhost_scsi_flush(struct vhost_scsi *vs)
1155 {
1156         struct vhost_scsi_inflight *old_inflight[VHOST_SCSI_MAX_VQ];
1157         int i;
1158
1159         /* Init new inflight and remember the old inflight */
1160         tcm_vhost_init_inflight(vs, old_inflight);
1161
1162         /*
1163          * The inflight->kref was initialized to 1. We decrement it here to
1164          * indicate the start of the flush operation so that it will reach 0
1165          * when all the reqs are finished.
1166          */
1167         for (i = 0; i < VHOST_SCSI_MAX_VQ; i++)
1168                 kref_put(&old_inflight[i]->kref, tcm_vhost_done_inflight);
1169
1170         /* Flush both the vhost poll and vhost work */
1171         for (i = 0; i < VHOST_SCSI_MAX_VQ; i++)
1172                 vhost_scsi_flush_vq(vs, i);
1173         vhost_work_flush(&vs->dev, &vs->vs_completion_work);
1174         vhost_work_flush(&vs->dev, &vs->vs_event_work);
1175
1176         /* Wait for all reqs issued before the flush to be finished */
1177         for (i = 0; i < VHOST_SCSI_MAX_VQ; i++)
1178                 wait_for_completion(&old_inflight[i]->comp);
1179 }
1180
1181 /*
1182  * Called from vhost_scsi_ioctl() context to walk the list of available
1183  * tcm_vhost_tpg with an active struct tcm_vhost_nexus
1184  *
1185  *  The lock nesting rule is:
1186  *    tcm_vhost_mutex -> vs->dev.mutex -> tpg->tv_tpg_mutex -> vq->mutex
1187  */
1188 static int
1189 vhost_scsi_set_endpoint(struct vhost_scsi *vs,
1190                         struct vhost_scsi_target *t)
1191 {
1192         struct tcm_vhost_tport *tv_tport;
1193         struct tcm_vhost_tpg *tpg;
1194         struct tcm_vhost_tpg **vs_tpg;
1195         struct vhost_virtqueue *vq;
1196         int index, ret, i, len;
1197         bool match = false;
1198
1199         mutex_lock(&tcm_vhost_mutex);
1200         mutex_lock(&vs->dev.mutex);
1201
1202         /* Verify that ring has been setup correctly. */
1203         for (index = 0; index < vs->dev.nvqs; ++index) {
1204                 /* Verify that ring has been setup correctly. */
1205                 if (!vhost_vq_access_ok(&vs->vqs[index].vq)) {
1206                         ret = -EFAULT;
1207                         goto out;
1208                 }
1209         }
1210
1211         len = sizeof(vs_tpg[0]) * VHOST_SCSI_MAX_TARGET;
1212         vs_tpg = kzalloc(len, GFP_KERNEL);
1213         if (!vs_tpg) {
1214                 ret = -ENOMEM;
1215                 goto out;
1216         }
1217         if (vs->vs_tpg)
1218                 memcpy(vs_tpg, vs->vs_tpg, len);
1219
1220         list_for_each_entry(tpg, &tcm_vhost_list, tv_tpg_list) {
1221                 mutex_lock(&tpg->tv_tpg_mutex);
1222                 if (!tpg->tpg_nexus) {
1223                         mutex_unlock(&tpg->tv_tpg_mutex);
1224                         continue;
1225                 }
1226                 if (tpg->tv_tpg_vhost_count != 0) {
1227                         mutex_unlock(&tpg->tv_tpg_mutex);
1228                         continue;
1229                 }
1230                 tv_tport = tpg->tport;
1231
1232                 if (!strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
1233                         if (vs->vs_tpg && vs->vs_tpg[tpg->tport_tpgt]) {
1234                                 kfree(vs_tpg);
1235                                 mutex_unlock(&tpg->tv_tpg_mutex);
1236                                 ret = -EEXIST;
1237                                 goto out;
1238                         }
1239                         tpg->tv_tpg_vhost_count++;
1240                         tpg->vhost_scsi = vs;
1241                         vs_tpg[tpg->tport_tpgt] = tpg;
1242                         smp_mb__after_atomic_inc();
1243                         match = true;
1244                 }
1245                 mutex_unlock(&tpg->tv_tpg_mutex);
1246         }
1247
1248         if (match) {
1249                 memcpy(vs->vs_vhost_wwpn, t->vhost_wwpn,
1250                        sizeof(vs->vs_vhost_wwpn));
1251                 for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
1252                         vq = &vs->vqs[i].vq;
1253                         mutex_lock(&vq->mutex);
1254                         vq->private_data = vs_tpg;
1255                         vhost_init_used(vq);
1256                         mutex_unlock(&vq->mutex);
1257                 }
1258                 ret = 0;
1259         } else {
1260                 ret = -EEXIST;
1261         }
1262
1263         /*
1264          * Act as synchronize_rcu to make sure access to
1265          * old vs->vs_tpg is finished.
1266          */
1267         vhost_scsi_flush(vs);
1268         kfree(vs->vs_tpg);
1269         vs->vs_tpg = vs_tpg;
1270
1271 out:
1272         mutex_unlock(&vs->dev.mutex);
1273         mutex_unlock(&tcm_vhost_mutex);
1274         return ret;
1275 }
1276
1277 static int
1278 vhost_scsi_clear_endpoint(struct vhost_scsi *vs,
1279                           struct vhost_scsi_target *t)
1280 {
1281         struct tcm_vhost_tport *tv_tport;
1282         struct tcm_vhost_tpg *tpg;
1283         struct vhost_virtqueue *vq;
1284         bool match = false;
1285         int index, ret, i;
1286         u8 target;
1287
1288         mutex_lock(&tcm_vhost_mutex);
1289         mutex_lock(&vs->dev.mutex);
1290         /* Verify that ring has been setup correctly. */
1291         for (index = 0; index < vs->dev.nvqs; ++index) {
1292                 if (!vhost_vq_access_ok(&vs->vqs[index].vq)) {
1293                         ret = -EFAULT;
1294                         goto err_dev;
1295                 }
1296         }
1297
1298         if (!vs->vs_tpg) {
1299                 ret = 0;
1300                 goto err_dev;
1301         }
1302
1303         for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {
1304                 target = i;
1305                 tpg = vs->vs_tpg[target];
1306                 if (!tpg)
1307                         continue;
1308
1309                 mutex_lock(&tpg->tv_tpg_mutex);
1310                 tv_tport = tpg->tport;
1311                 if (!tv_tport) {
1312                         ret = -ENODEV;
1313                         goto err_tpg;
1314                 }
1315
1316                 if (strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
1317                         pr_warn("tv_tport->tport_name: %s, tpg->tport_tpgt: %hu"
1318                                 " does not match t->vhost_wwpn: %s, t->vhost_tpgt: %hu\n",
1319                                 tv_tport->tport_name, tpg->tport_tpgt,
1320                                 t->vhost_wwpn, t->vhost_tpgt);
1321                         ret = -EINVAL;
1322                         goto err_tpg;
1323                 }
1324                 tpg->tv_tpg_vhost_count--;
1325                 tpg->vhost_scsi = NULL;
1326                 vs->vs_tpg[target] = NULL;
1327                 match = true;
1328                 mutex_unlock(&tpg->tv_tpg_mutex);
1329         }
1330         if (match) {
1331                 for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
1332                         vq = &vs->vqs[i].vq;
1333                         mutex_lock(&vq->mutex);
1334                         vq->private_data = NULL;
1335                         mutex_unlock(&vq->mutex);
1336                 }
1337         }
1338         /*
1339          * Act as synchronize_rcu to make sure access to
1340          * old vs->vs_tpg is finished.
1341          */
1342         vhost_scsi_flush(vs);
1343         kfree(vs->vs_tpg);
1344         vs->vs_tpg = NULL;
1345         WARN_ON(vs->vs_events_nr);
1346         mutex_unlock(&vs->dev.mutex);
1347         mutex_unlock(&tcm_vhost_mutex);
1348         return 0;
1349
1350 err_tpg:
1351         mutex_unlock(&tpg->tv_tpg_mutex);
1352 err_dev:
1353         mutex_unlock(&vs->dev.mutex);
1354         mutex_unlock(&tcm_vhost_mutex);
1355         return ret;
1356 }
1357
1358 static int vhost_scsi_set_features(struct vhost_scsi *vs, u64 features)
1359 {
1360         if (features & ~VHOST_SCSI_FEATURES)
1361                 return -EOPNOTSUPP;
1362
1363         mutex_lock(&vs->dev.mutex);
1364         if ((features & (1 << VHOST_F_LOG_ALL)) &&
1365             !vhost_log_access_ok(&vs->dev)) {
1366                 mutex_unlock(&vs->dev.mutex);
1367                 return -EFAULT;
1368         }
1369         vs->dev.acked_features = features;
1370         smp_wmb();
1371         vhost_scsi_flush(vs);
1372         mutex_unlock(&vs->dev.mutex);
1373         return 0;
1374 }
1375
1376 static int vhost_scsi_open(struct inode *inode, struct file *f)
1377 {
1378         struct vhost_scsi *vs;
1379         struct vhost_virtqueue **vqs;
1380         int r, i;
1381
1382         vs = kzalloc(sizeof(*vs), GFP_KERNEL);
1383         if (!vs)
1384                 return -ENOMEM;
1385
1386         vqs = kmalloc(VHOST_SCSI_MAX_VQ * sizeof(*vqs), GFP_KERNEL);
1387         if (!vqs) {
1388                 kfree(vs);
1389                 return -ENOMEM;
1390         }
1391
1392         vhost_work_init(&vs->vs_completion_work, vhost_scsi_complete_cmd_work);
1393         vhost_work_init(&vs->vs_event_work, tcm_vhost_evt_work);
1394
1395         vs->vs_events_nr = 0;
1396         vs->vs_events_missed = false;
1397
1398         vqs[VHOST_SCSI_VQ_CTL] = &vs->vqs[VHOST_SCSI_VQ_CTL].vq;
1399         vqs[VHOST_SCSI_VQ_EVT] = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
1400         vs->vqs[VHOST_SCSI_VQ_CTL].vq.handle_kick = vhost_scsi_ctl_handle_kick;
1401         vs->vqs[VHOST_SCSI_VQ_EVT].vq.handle_kick = vhost_scsi_evt_handle_kick;
1402         for (i = VHOST_SCSI_VQ_IO; i < VHOST_SCSI_MAX_VQ; i++) {
1403                 vqs[i] = &vs->vqs[i].vq;
1404                 vs->vqs[i].vq.handle_kick = vhost_scsi_handle_kick;
1405         }
1406         r = vhost_dev_init(&vs->dev, vqs, VHOST_SCSI_MAX_VQ);
1407
1408         tcm_vhost_init_inflight(vs, NULL);
1409
1410         if (r < 0) {
1411                 kfree(vqs);
1412                 kfree(vs);
1413                 return r;
1414         }
1415
1416         f->private_data = vs;
1417         return 0;
1418 }
1419
1420 static int vhost_scsi_release(struct inode *inode, struct file *f)
1421 {
1422         struct vhost_scsi *vs = f->private_data;
1423         struct vhost_scsi_target t;
1424
1425         mutex_lock(&vs->dev.mutex);
1426         memcpy(t.vhost_wwpn, vs->vs_vhost_wwpn, sizeof(t.vhost_wwpn));
1427         mutex_unlock(&vs->dev.mutex);
1428         vhost_scsi_clear_endpoint(vs, &t);
1429         vhost_dev_stop(&vs->dev);
1430         vhost_dev_cleanup(&vs->dev, false);
1431         /* Jobs can re-queue themselves in evt kick handler. Do extra flush. */
1432         vhost_scsi_flush(vs);
1433         kfree(vs->dev.vqs);
1434         kfree(vs);
1435         return 0;
1436 }
1437
1438 static long
1439 vhost_scsi_ioctl(struct file *f,
1440                  unsigned int ioctl,
1441                  unsigned long arg)
1442 {
1443         struct vhost_scsi *vs = f->private_data;
1444         struct vhost_scsi_target backend;
1445         void __user *argp = (void __user *)arg;
1446         u64 __user *featurep = argp;
1447         u32 __user *eventsp = argp;
1448         u32 events_missed;
1449         u64 features;
1450         int r, abi_version = VHOST_SCSI_ABI_VERSION;
1451         struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
1452
1453         switch (ioctl) {
1454         case VHOST_SCSI_SET_ENDPOINT:
1455                 if (copy_from_user(&backend, argp, sizeof backend))
1456                         return -EFAULT;
1457                 if (backend.reserved != 0)
1458                         return -EOPNOTSUPP;
1459
1460                 return vhost_scsi_set_endpoint(vs, &backend);
1461         case VHOST_SCSI_CLEAR_ENDPOINT:
1462                 if (copy_from_user(&backend, argp, sizeof backend))
1463                         return -EFAULT;
1464                 if (backend.reserved != 0)
1465                         return -EOPNOTSUPP;
1466
1467                 return vhost_scsi_clear_endpoint(vs, &backend);
1468         case VHOST_SCSI_GET_ABI_VERSION:
1469                 if (copy_to_user(argp, &abi_version, sizeof abi_version))
1470                         return -EFAULT;
1471                 return 0;
1472         case VHOST_SCSI_SET_EVENTS_MISSED:
1473                 if (get_user(events_missed, eventsp))
1474                         return -EFAULT;
1475                 mutex_lock(&vq->mutex);
1476                 vs->vs_events_missed = events_missed;
1477                 mutex_unlock(&vq->mutex);
1478                 return 0;
1479         case VHOST_SCSI_GET_EVENTS_MISSED:
1480                 mutex_lock(&vq->mutex);
1481                 events_missed = vs->vs_events_missed;
1482                 mutex_unlock(&vq->mutex);
1483                 if (put_user(events_missed, eventsp))
1484                         return -EFAULT;
1485                 return 0;
1486         case VHOST_GET_FEATURES:
1487                 features = VHOST_SCSI_FEATURES;
1488                 if (copy_to_user(featurep, &features, sizeof features))
1489                         return -EFAULT;
1490                 return 0;
1491         case VHOST_SET_FEATURES:
1492                 if (copy_from_user(&features, featurep, sizeof features))
1493                         return -EFAULT;
1494                 return vhost_scsi_set_features(vs, features);
1495         default:
1496                 mutex_lock(&vs->dev.mutex);
1497                 r = vhost_dev_ioctl(&vs->dev, ioctl, argp);
1498                 /* TODO: flush backend after dev ioctl. */
1499                 if (r == -ENOIOCTLCMD)
1500                         r = vhost_vring_ioctl(&vs->dev, ioctl, argp);
1501                 mutex_unlock(&vs->dev.mutex);
1502                 return r;
1503         }
1504 }
1505
1506 #ifdef CONFIG_COMPAT
1507 static long vhost_scsi_compat_ioctl(struct file *f, unsigned int ioctl,
1508                                 unsigned long arg)
1509 {
1510         return vhost_scsi_ioctl(f, ioctl, (unsigned long)compat_ptr(arg));
1511 }
1512 #endif
1513
1514 static const struct file_operations vhost_scsi_fops = {
1515         .owner          = THIS_MODULE,
1516         .release        = vhost_scsi_release,
1517         .unlocked_ioctl = vhost_scsi_ioctl,
1518 #ifdef CONFIG_COMPAT
1519         .compat_ioctl   = vhost_scsi_compat_ioctl,
1520 #endif
1521         .open           = vhost_scsi_open,
1522         .llseek         = noop_llseek,
1523 };
1524
1525 static struct miscdevice vhost_scsi_misc = {
1526         MISC_DYNAMIC_MINOR,
1527         "vhost-scsi",
1528         &vhost_scsi_fops,
1529 };
1530
1531 static int __init vhost_scsi_register(void)
1532 {
1533         return misc_register(&vhost_scsi_misc);
1534 }
1535
1536 static int vhost_scsi_deregister(void)
1537 {
1538         return misc_deregister(&vhost_scsi_misc);
1539 }
1540
1541 static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport)
1542 {
1543         switch (tport->tport_proto_id) {
1544         case SCSI_PROTOCOL_SAS:
1545                 return "SAS";
1546         case SCSI_PROTOCOL_FCP:
1547                 return "FCP";
1548         case SCSI_PROTOCOL_ISCSI:
1549                 return "iSCSI";
1550         default:
1551                 break;
1552         }
1553
1554         return "Unknown";
1555 }
1556
1557 static void
1558 tcm_vhost_do_plug(struct tcm_vhost_tpg *tpg,
1559                   struct se_lun *lun, bool plug)
1560 {
1561
1562         struct vhost_scsi *vs = tpg->vhost_scsi;
1563         struct vhost_virtqueue *vq;
1564         u32 reason;
1565
1566         if (!vs)
1567                 return;
1568
1569         mutex_lock(&vs->dev.mutex);
1570         if (!vhost_has_feature(&vs->dev, VIRTIO_SCSI_F_HOTPLUG)) {
1571                 mutex_unlock(&vs->dev.mutex);
1572                 return;
1573         }
1574
1575         if (plug)
1576                 reason = VIRTIO_SCSI_EVT_RESET_RESCAN;
1577         else
1578                 reason = VIRTIO_SCSI_EVT_RESET_REMOVED;
1579
1580         vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
1581         mutex_lock(&vq->mutex);
1582         tcm_vhost_send_evt(vs, tpg, lun,
1583                         VIRTIO_SCSI_T_TRANSPORT_RESET, reason);
1584         mutex_unlock(&vq->mutex);
1585         mutex_unlock(&vs->dev.mutex);
1586 }
1587
1588 static void tcm_vhost_hotplug(struct tcm_vhost_tpg *tpg, struct se_lun *lun)
1589 {
1590         tcm_vhost_do_plug(tpg, lun, true);
1591 }
1592
1593 static void tcm_vhost_hotunplug(struct tcm_vhost_tpg *tpg, struct se_lun *lun)
1594 {
1595         tcm_vhost_do_plug(tpg, lun, false);
1596 }
1597
1598 static int tcm_vhost_port_link(struct se_portal_group *se_tpg,
1599                                struct se_lun *lun)
1600 {
1601         struct tcm_vhost_tpg *tpg = container_of(se_tpg,
1602                                 struct tcm_vhost_tpg, se_tpg);
1603
1604         mutex_lock(&tcm_vhost_mutex);
1605
1606         mutex_lock(&tpg->tv_tpg_mutex);
1607         tpg->tv_tpg_port_count++;
1608         mutex_unlock(&tpg->tv_tpg_mutex);
1609
1610         tcm_vhost_hotplug(tpg, lun);
1611
1612         mutex_unlock(&tcm_vhost_mutex);
1613
1614         return 0;
1615 }
1616
1617 static void tcm_vhost_port_unlink(struct se_portal_group *se_tpg,
1618                                   struct se_lun *lun)
1619 {
1620         struct tcm_vhost_tpg *tpg = container_of(se_tpg,
1621                                 struct tcm_vhost_tpg, se_tpg);
1622
1623         mutex_lock(&tcm_vhost_mutex);
1624
1625         mutex_lock(&tpg->tv_tpg_mutex);
1626         tpg->tv_tpg_port_count--;
1627         mutex_unlock(&tpg->tv_tpg_mutex);
1628
1629         tcm_vhost_hotunplug(tpg, lun);
1630
1631         mutex_unlock(&tcm_vhost_mutex);
1632 }
1633
1634 static struct se_node_acl *
1635 tcm_vhost_make_nodeacl(struct se_portal_group *se_tpg,
1636                        struct config_group *group,
1637                        const char *name)
1638 {
1639         struct se_node_acl *se_nacl, *se_nacl_new;
1640         struct tcm_vhost_nacl *nacl;
1641         u64 wwpn = 0;
1642         u32 nexus_depth;
1643
1644         /* tcm_vhost_parse_wwn(name, &wwpn, 1) < 0)
1645                 return ERR_PTR(-EINVAL); */
1646         se_nacl_new = tcm_vhost_alloc_fabric_acl(se_tpg);
1647         if (!se_nacl_new)
1648                 return ERR_PTR(-ENOMEM);
1649
1650         nexus_depth = 1;
1651         /*
1652          * se_nacl_new may be released by core_tpg_add_initiator_node_acl()
1653          * when converting a NodeACL from demo mode -> explict
1654          */
1655         se_nacl = core_tpg_add_initiator_node_acl(se_tpg, se_nacl_new,
1656                                 name, nexus_depth);
1657         if (IS_ERR(se_nacl)) {
1658                 tcm_vhost_release_fabric_acl(se_tpg, se_nacl_new);
1659                 return se_nacl;
1660         }
1661         /*
1662          * Locate our struct tcm_vhost_nacl and set the FC Nport WWPN
1663          */
1664         nacl = container_of(se_nacl, struct tcm_vhost_nacl, se_node_acl);
1665         nacl->iport_wwpn = wwpn;
1666
1667         return se_nacl;
1668 }
1669
1670 static void tcm_vhost_drop_nodeacl(struct se_node_acl *se_acl)
1671 {
1672         struct tcm_vhost_nacl *nacl = container_of(se_acl,
1673                                 struct tcm_vhost_nacl, se_node_acl);
1674         core_tpg_del_initiator_node_acl(se_acl->se_tpg, se_acl, 1);
1675         kfree(nacl);
1676 }
1677
1678 static void tcm_vhost_free_cmd_map_res(struct tcm_vhost_nexus *nexus,
1679                                        struct se_session *se_sess)
1680 {
1681         struct tcm_vhost_cmd *tv_cmd;
1682         unsigned int i;
1683
1684         if (!se_sess->sess_cmd_map)
1685                 return;
1686
1687         for (i = 0; i < TCM_VHOST_DEFAULT_TAGS; i++) {
1688                 tv_cmd = &((struct tcm_vhost_cmd *)se_sess->sess_cmd_map)[i];
1689
1690                 kfree(tv_cmd->tvc_sgl);
1691                 kfree(tv_cmd->tvc_upages);
1692         }
1693 }
1694
1695 static int tcm_vhost_make_nexus(struct tcm_vhost_tpg *tpg,
1696                                 const char *name)
1697 {
1698         struct se_portal_group *se_tpg;
1699         struct se_session *se_sess;
1700         struct tcm_vhost_nexus *tv_nexus;
1701         struct tcm_vhost_cmd *tv_cmd;
1702         unsigned int i;
1703
1704         mutex_lock(&tpg->tv_tpg_mutex);
1705         if (tpg->tpg_nexus) {
1706                 mutex_unlock(&tpg->tv_tpg_mutex);
1707                 pr_debug("tpg->tpg_nexus already exists\n");
1708                 return -EEXIST;
1709         }
1710         se_tpg = &tpg->se_tpg;
1711
1712         tv_nexus = kzalloc(sizeof(struct tcm_vhost_nexus), GFP_KERNEL);
1713         if (!tv_nexus) {
1714                 mutex_unlock(&tpg->tv_tpg_mutex);
1715                 pr_err("Unable to allocate struct tcm_vhost_nexus\n");
1716                 return -ENOMEM;
1717         }
1718         /*
1719          *  Initialize the struct se_session pointer and setup tagpool
1720          *  for struct tcm_vhost_cmd descriptors
1721          */
1722         tv_nexus->tvn_se_sess = transport_init_session_tags(
1723                                         TCM_VHOST_DEFAULT_TAGS,
1724                                         sizeof(struct tcm_vhost_cmd));
1725         if (IS_ERR(tv_nexus->tvn_se_sess)) {
1726                 mutex_unlock(&tpg->tv_tpg_mutex);
1727                 kfree(tv_nexus);
1728                 return -ENOMEM;
1729         }
1730         se_sess = tv_nexus->tvn_se_sess;
1731         for (i = 0; i < TCM_VHOST_DEFAULT_TAGS; i++) {
1732                 tv_cmd = &((struct tcm_vhost_cmd *)se_sess->sess_cmd_map)[i];
1733
1734                 tv_cmd->tvc_sgl = kzalloc(sizeof(struct scatterlist) *
1735                                         TCM_VHOST_PREALLOC_SGLS, GFP_KERNEL);
1736                 if (!tv_cmd->tvc_sgl) {
1737                         mutex_unlock(&tpg->tv_tpg_mutex);
1738                         pr_err("Unable to allocate tv_cmd->tvc_sgl\n");
1739                         goto out;
1740                 }
1741
1742                 tv_cmd->tvc_upages = kzalloc(sizeof(struct page *) *
1743                                         TCM_VHOST_PREALLOC_PAGES, GFP_KERNEL);
1744                 if (!tv_cmd->tvc_upages) {
1745                         mutex_unlock(&tpg->tv_tpg_mutex);
1746                         pr_err("Unable to allocate tv_cmd->tvc_upages\n");
1747                         goto out;
1748                 }
1749         }
1750         /*
1751          * Since we are running in 'demo mode' this call with generate a
1752          * struct se_node_acl for the tcm_vhost struct se_portal_group with
1753          * the SCSI Initiator port name of the passed configfs group 'name'.
1754          */
1755         tv_nexus->tvn_se_sess->se_node_acl = core_tpg_check_initiator_node_acl(
1756                                 se_tpg, (unsigned char *)name);
1757         if (!tv_nexus->tvn_se_sess->se_node_acl) {
1758                 mutex_unlock(&tpg->tv_tpg_mutex);
1759                 pr_debug("core_tpg_check_initiator_node_acl() failed"
1760                                 " for %s\n", name);
1761                 goto out;
1762         }
1763         /*
1764          * Now register the TCM vhost virtual I_T Nexus as active with the
1765          * call to __transport_register_session()
1766          */
1767         __transport_register_session(se_tpg, tv_nexus->tvn_se_sess->se_node_acl,
1768                         tv_nexus->tvn_se_sess, tv_nexus);
1769         tpg->tpg_nexus = tv_nexus;
1770
1771         mutex_unlock(&tpg->tv_tpg_mutex);
1772         return 0;
1773
1774 out:
1775         tcm_vhost_free_cmd_map_res(tv_nexus, se_sess);
1776         transport_free_session(se_sess);
1777         kfree(tv_nexus);
1778         return -ENOMEM;
1779 }
1780
1781 static int tcm_vhost_drop_nexus(struct tcm_vhost_tpg *tpg)
1782 {
1783         struct se_session *se_sess;
1784         struct tcm_vhost_nexus *tv_nexus;
1785
1786         mutex_lock(&tpg->tv_tpg_mutex);
1787         tv_nexus = tpg->tpg_nexus;
1788         if (!tv_nexus) {
1789                 mutex_unlock(&tpg->tv_tpg_mutex);
1790                 return -ENODEV;
1791         }
1792
1793         se_sess = tv_nexus->tvn_se_sess;
1794         if (!se_sess) {
1795                 mutex_unlock(&tpg->tv_tpg_mutex);
1796                 return -ENODEV;
1797         }
1798
1799         if (tpg->tv_tpg_port_count != 0) {
1800                 mutex_unlock(&tpg->tv_tpg_mutex);
1801                 pr_err("Unable to remove TCM_vhost I_T Nexus with"
1802                         " active TPG port count: %d\n",
1803                         tpg->tv_tpg_port_count);
1804                 return -EBUSY;
1805         }
1806
1807         if (tpg->tv_tpg_vhost_count != 0) {
1808                 mutex_unlock(&tpg->tv_tpg_mutex);
1809                 pr_err("Unable to remove TCM_vhost I_T Nexus with"
1810                         " active TPG vhost count: %d\n",
1811                         tpg->tv_tpg_vhost_count);
1812                 return -EBUSY;
1813         }
1814
1815         pr_debug("TCM_vhost_ConfigFS: Removing I_T Nexus to emulated"
1816                 " %s Initiator Port: %s\n", tcm_vhost_dump_proto_id(tpg->tport),
1817                 tv_nexus->tvn_se_sess->se_node_acl->initiatorname);
1818
1819         tcm_vhost_free_cmd_map_res(tv_nexus, se_sess);
1820         /*
1821          * Release the SCSI I_T Nexus to the emulated vhost Target Port
1822          */
1823         transport_deregister_session(tv_nexus->tvn_se_sess);
1824         tpg->tpg_nexus = NULL;
1825         mutex_unlock(&tpg->tv_tpg_mutex);
1826
1827         kfree(tv_nexus);
1828         return 0;
1829 }
1830
1831 static ssize_t tcm_vhost_tpg_show_nexus(struct se_portal_group *se_tpg,
1832                                         char *page)
1833 {
1834         struct tcm_vhost_tpg *tpg = container_of(se_tpg,
1835                                 struct tcm_vhost_tpg, se_tpg);
1836         struct tcm_vhost_nexus *tv_nexus;
1837         ssize_t ret;
1838
1839         mutex_lock(&tpg->tv_tpg_mutex);
1840         tv_nexus = tpg->tpg_nexus;
1841         if (!tv_nexus) {
1842                 mutex_unlock(&tpg->tv_tpg_mutex);
1843                 return -ENODEV;
1844         }
1845         ret = snprintf(page, PAGE_SIZE, "%s\n",
1846                         tv_nexus->tvn_se_sess->se_node_acl->initiatorname);
1847         mutex_unlock(&tpg->tv_tpg_mutex);
1848
1849         return ret;
1850 }
1851
1852 static ssize_t tcm_vhost_tpg_store_nexus(struct se_portal_group *se_tpg,
1853                                          const char *page,
1854                                          size_t count)
1855 {
1856         struct tcm_vhost_tpg *tpg = container_of(se_tpg,
1857                                 struct tcm_vhost_tpg, se_tpg);
1858         struct tcm_vhost_tport *tport_wwn = tpg->tport;
1859         unsigned char i_port[TCM_VHOST_NAMELEN], *ptr, *port_ptr;
1860         int ret;
1861         /*
1862          * Shutdown the active I_T nexus if 'NULL' is passed..
1863          */
1864         if (!strncmp(page, "NULL", 4)) {
1865                 ret = tcm_vhost_drop_nexus(tpg);
1866                 return (!ret) ? count : ret;
1867         }
1868         /*
1869          * Otherwise make sure the passed virtual Initiator port WWN matches
1870          * the fabric protocol_id set in tcm_vhost_make_tport(), and call
1871          * tcm_vhost_make_nexus().
1872          */
1873         if (strlen(page) >= TCM_VHOST_NAMELEN) {
1874                 pr_err("Emulated NAA Sas Address: %s, exceeds"
1875                                 " max: %d\n", page, TCM_VHOST_NAMELEN);
1876                 return -EINVAL;
1877         }
1878         snprintf(&i_port[0], TCM_VHOST_NAMELEN, "%s", page);
1879
1880         ptr = strstr(i_port, "naa.");
1881         if (ptr) {
1882                 if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_SAS) {
1883                         pr_err("Passed SAS Initiator Port %s does not"
1884                                 " match target port protoid: %s\n", i_port,
1885                                 tcm_vhost_dump_proto_id(tport_wwn));
1886                         return -EINVAL;
1887                 }
1888                 port_ptr = &i_port[0];
1889                 goto check_newline;
1890         }
1891         ptr = strstr(i_port, "fc.");
1892         if (ptr) {
1893                 if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_FCP) {
1894                         pr_err("Passed FCP Initiator Port %s does not"
1895                                 " match target port protoid: %s\n", i_port,
1896                                 tcm_vhost_dump_proto_id(tport_wwn));
1897                         return -EINVAL;
1898                 }
1899                 port_ptr = &i_port[3]; /* Skip over "fc." */
1900                 goto check_newline;
1901         }
1902         ptr = strstr(i_port, "iqn.");
1903         if (ptr) {
1904                 if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_ISCSI) {
1905                         pr_err("Passed iSCSI Initiator Port %s does not"
1906                                 " match target port protoid: %s\n", i_port,
1907                                 tcm_vhost_dump_proto_id(tport_wwn));
1908                         return -EINVAL;
1909                 }
1910                 port_ptr = &i_port[0];
1911                 goto check_newline;
1912         }
1913         pr_err("Unable to locate prefix for emulated Initiator Port:"
1914                         " %s\n", i_port);
1915         return -EINVAL;
1916         /*
1917          * Clear any trailing newline for the NAA WWN
1918          */
1919 check_newline:
1920         if (i_port[strlen(i_port)-1] == '\n')
1921                 i_port[strlen(i_port)-1] = '\0';
1922
1923         ret = tcm_vhost_make_nexus(tpg, port_ptr);
1924         if (ret < 0)
1925                 return ret;
1926
1927         return count;
1928 }
1929
1930 TF_TPG_BASE_ATTR(tcm_vhost, nexus, S_IRUGO | S_IWUSR);
1931
1932 static struct configfs_attribute *tcm_vhost_tpg_attrs[] = {
1933         &tcm_vhost_tpg_nexus.attr,
1934         NULL,
1935 };
1936
1937 static struct se_portal_group *
1938 tcm_vhost_make_tpg(struct se_wwn *wwn,
1939                    struct config_group *group,
1940                    const char *name)
1941 {
1942         struct tcm_vhost_tport *tport = container_of(wwn,
1943                         struct tcm_vhost_tport, tport_wwn);
1944
1945         struct tcm_vhost_tpg *tpg;
1946         unsigned long tpgt;
1947         int ret;
1948
1949         if (strstr(name, "tpgt_") != name)
1950                 return ERR_PTR(-EINVAL);
1951         if (kstrtoul(name + 5, 10, &tpgt) || tpgt > UINT_MAX)
1952                 return ERR_PTR(-EINVAL);
1953
1954         tpg = kzalloc(sizeof(struct tcm_vhost_tpg), GFP_KERNEL);
1955         if (!tpg) {
1956                 pr_err("Unable to allocate struct tcm_vhost_tpg");
1957                 return ERR_PTR(-ENOMEM);
1958         }
1959         mutex_init(&tpg->tv_tpg_mutex);
1960         INIT_LIST_HEAD(&tpg->tv_tpg_list);
1961         tpg->tport = tport;
1962         tpg->tport_tpgt = tpgt;
1963
1964         ret = core_tpg_register(&tcm_vhost_fabric_configfs->tf_ops, wwn,
1965                                 &tpg->se_tpg, tpg, TRANSPORT_TPG_TYPE_NORMAL);
1966         if (ret < 0) {
1967                 kfree(tpg);
1968                 return NULL;
1969         }
1970         mutex_lock(&tcm_vhost_mutex);
1971         list_add_tail(&tpg->tv_tpg_list, &tcm_vhost_list);
1972         mutex_unlock(&tcm_vhost_mutex);
1973
1974         return &tpg->se_tpg;
1975 }
1976
1977 static void tcm_vhost_drop_tpg(struct se_portal_group *se_tpg)
1978 {
1979         struct tcm_vhost_tpg *tpg = container_of(se_tpg,
1980                                 struct tcm_vhost_tpg, se_tpg);
1981
1982         mutex_lock(&tcm_vhost_mutex);
1983         list_del(&tpg->tv_tpg_list);
1984         mutex_unlock(&tcm_vhost_mutex);
1985         /*
1986          * Release the virtual I_T Nexus for this vhost TPG
1987          */
1988         tcm_vhost_drop_nexus(tpg);
1989         /*
1990          * Deregister the se_tpg from TCM..
1991          */
1992         core_tpg_deregister(se_tpg);
1993         kfree(tpg);
1994 }
1995
1996 static struct se_wwn *
1997 tcm_vhost_make_tport(struct target_fabric_configfs *tf,
1998                      struct config_group *group,
1999                      const char *name)
2000 {
2001         struct tcm_vhost_tport *tport;
2002         char *ptr;
2003         u64 wwpn = 0;
2004         int off = 0;
2005
2006         /* if (tcm_vhost_parse_wwn(name, &wwpn, 1) < 0)
2007                 return ERR_PTR(-EINVAL); */
2008
2009         tport = kzalloc(sizeof(struct tcm_vhost_tport), GFP_KERNEL);
2010         if (!tport) {
2011                 pr_err("Unable to allocate struct tcm_vhost_tport");
2012                 return ERR_PTR(-ENOMEM);
2013         }
2014         tport->tport_wwpn = wwpn;
2015         /*
2016          * Determine the emulated Protocol Identifier and Target Port Name
2017          * based on the incoming configfs directory name.
2018          */
2019         ptr = strstr(name, "naa.");
2020         if (ptr) {
2021                 tport->tport_proto_id = SCSI_PROTOCOL_SAS;
2022                 goto check_len;
2023         }
2024         ptr = strstr(name, "fc.");
2025         if (ptr) {
2026                 tport->tport_proto_id = SCSI_PROTOCOL_FCP;
2027                 off = 3; /* Skip over "fc." */
2028                 goto check_len;
2029         }
2030         ptr = strstr(name, "iqn.");
2031         if (ptr) {
2032                 tport->tport_proto_id = SCSI_PROTOCOL_ISCSI;
2033                 goto check_len;
2034         }
2035
2036         pr_err("Unable to locate prefix for emulated Target Port:"
2037                         " %s\n", name);
2038         kfree(tport);
2039         return ERR_PTR(-EINVAL);
2040
2041 check_len:
2042         if (strlen(name) >= TCM_VHOST_NAMELEN) {
2043                 pr_err("Emulated %s Address: %s, exceeds"
2044                         " max: %d\n", name, tcm_vhost_dump_proto_id(tport),
2045                         TCM_VHOST_NAMELEN);
2046                 kfree(tport);
2047                 return ERR_PTR(-EINVAL);
2048         }
2049         snprintf(&tport->tport_name[0], TCM_VHOST_NAMELEN, "%s", &name[off]);
2050
2051         pr_debug("TCM_VHost_ConfigFS: Allocated emulated Target"
2052                 " %s Address: %s\n", tcm_vhost_dump_proto_id(tport), name);
2053
2054         return &tport->tport_wwn;
2055 }
2056
2057 static void tcm_vhost_drop_tport(struct se_wwn *wwn)
2058 {
2059         struct tcm_vhost_tport *tport = container_of(wwn,
2060                                 struct tcm_vhost_tport, tport_wwn);
2061
2062         pr_debug("TCM_VHost_ConfigFS: Deallocating emulated Target"
2063                 " %s Address: %s\n", tcm_vhost_dump_proto_id(tport),
2064                 tport->tport_name);
2065
2066         kfree(tport);
2067 }
2068
2069 static ssize_t
2070 tcm_vhost_wwn_show_attr_version(struct target_fabric_configfs *tf,
2071                                 char *page)
2072 {
2073         return sprintf(page, "TCM_VHOST fabric module %s on %s/%s"
2074                 "on "UTS_RELEASE"\n", TCM_VHOST_VERSION, utsname()->sysname,
2075                 utsname()->machine);
2076 }
2077
2078 TF_WWN_ATTR_RO(tcm_vhost, version);
2079
2080 static struct configfs_attribute *tcm_vhost_wwn_attrs[] = {
2081         &tcm_vhost_wwn_version.attr,
2082         NULL,
2083 };
2084
2085 static struct target_core_fabric_ops tcm_vhost_ops = {
2086         .get_fabric_name                = tcm_vhost_get_fabric_name,
2087         .get_fabric_proto_ident         = tcm_vhost_get_fabric_proto_ident,
2088         .tpg_get_wwn                    = tcm_vhost_get_fabric_wwn,
2089         .tpg_get_tag                    = tcm_vhost_get_tag,
2090         .tpg_get_default_depth          = tcm_vhost_get_default_depth,
2091         .tpg_get_pr_transport_id        = tcm_vhost_get_pr_transport_id,
2092         .tpg_get_pr_transport_id_len    = tcm_vhost_get_pr_transport_id_len,
2093         .tpg_parse_pr_out_transport_id  = tcm_vhost_parse_pr_out_transport_id,
2094         .tpg_check_demo_mode            = tcm_vhost_check_true,
2095         .tpg_check_demo_mode_cache      = tcm_vhost_check_true,
2096         .tpg_check_demo_mode_write_protect = tcm_vhost_check_false,
2097         .tpg_check_prod_mode_write_protect = tcm_vhost_check_false,
2098         .tpg_alloc_fabric_acl           = tcm_vhost_alloc_fabric_acl,
2099         .tpg_release_fabric_acl         = tcm_vhost_release_fabric_acl,
2100         .tpg_get_inst_index             = tcm_vhost_tpg_get_inst_index,
2101         .release_cmd                    = tcm_vhost_release_cmd,
2102         .check_stop_free                = vhost_scsi_check_stop_free,
2103         .shutdown_session               = tcm_vhost_shutdown_session,
2104         .close_session                  = tcm_vhost_close_session,
2105         .sess_get_index                 = tcm_vhost_sess_get_index,
2106         .sess_get_initiator_sid         = NULL,
2107         .write_pending                  = tcm_vhost_write_pending,
2108         .write_pending_status           = tcm_vhost_write_pending_status,
2109         .set_default_node_attributes    = tcm_vhost_set_default_node_attrs,
2110         .get_task_tag                   = tcm_vhost_get_task_tag,
2111         .get_cmd_state                  = tcm_vhost_get_cmd_state,
2112         .queue_data_in                  = tcm_vhost_queue_data_in,
2113         .queue_status                   = tcm_vhost_queue_status,
2114         .queue_tm_rsp                   = tcm_vhost_queue_tm_rsp,
2115         /*
2116          * Setup callers for generic logic in target_core_fabric_configfs.c
2117          */
2118         .fabric_make_wwn                = tcm_vhost_make_tport,
2119         .fabric_drop_wwn                = tcm_vhost_drop_tport,
2120         .fabric_make_tpg                = tcm_vhost_make_tpg,
2121         .fabric_drop_tpg                = tcm_vhost_drop_tpg,
2122         .fabric_post_link               = tcm_vhost_port_link,
2123         .fabric_pre_unlink              = tcm_vhost_port_unlink,
2124         .fabric_make_np                 = NULL,
2125         .fabric_drop_np                 = NULL,
2126         .fabric_make_nodeacl            = tcm_vhost_make_nodeacl,
2127         .fabric_drop_nodeacl            = tcm_vhost_drop_nodeacl,
2128 };
2129
2130 static int tcm_vhost_register_configfs(void)
2131 {
2132         struct target_fabric_configfs *fabric;
2133         int ret;
2134
2135         pr_debug("TCM_VHOST fabric module %s on %s/%s"
2136                 " on "UTS_RELEASE"\n", TCM_VHOST_VERSION, utsname()->sysname,
2137                 utsname()->machine);
2138         /*
2139          * Register the top level struct config_item_type with TCM core
2140          */
2141         fabric = target_fabric_configfs_init(THIS_MODULE, "vhost");
2142         if (IS_ERR(fabric)) {
2143                 pr_err("target_fabric_configfs_init() failed\n");
2144                 return PTR_ERR(fabric);
2145         }
2146         /*
2147          * Setup fabric->tf_ops from our local tcm_vhost_ops
2148          */
2149         fabric->tf_ops = tcm_vhost_ops;
2150         /*
2151          * Setup default attribute lists for various fabric->tf_cit_tmpl
2152          */
2153         TF_CIT_TMPL(fabric)->tfc_wwn_cit.ct_attrs = tcm_vhost_wwn_attrs;
2154         TF_CIT_TMPL(fabric)->tfc_tpg_base_cit.ct_attrs = tcm_vhost_tpg_attrs;
2155         TF_CIT_TMPL(fabric)->tfc_tpg_attrib_cit.ct_attrs = NULL;
2156         TF_CIT_TMPL(fabric)->tfc_tpg_param_cit.ct_attrs = NULL;
2157         TF_CIT_TMPL(fabric)->tfc_tpg_np_base_cit.ct_attrs = NULL;
2158         TF_CIT_TMPL(fabric)->tfc_tpg_nacl_base_cit.ct_attrs = NULL;
2159         TF_CIT_TMPL(fabric)->tfc_tpg_nacl_attrib_cit.ct_attrs = NULL;
2160         TF_CIT_TMPL(fabric)->tfc_tpg_nacl_auth_cit.ct_attrs = NULL;
2161         TF_CIT_TMPL(fabric)->tfc_tpg_nacl_param_cit.ct_attrs = NULL;
2162         /*
2163          * Register the fabric for use within TCM
2164          */
2165         ret = target_fabric_configfs_register(fabric);
2166         if (ret < 0) {
2167                 pr_err("target_fabric_configfs_register() failed"
2168                                 " for TCM_VHOST\n");
2169                 return ret;
2170         }
2171         /*
2172          * Setup our local pointer to *fabric
2173          */
2174         tcm_vhost_fabric_configfs = fabric;
2175         pr_debug("TCM_VHOST[0] - Set fabric -> tcm_vhost_fabric_configfs\n");
2176         return 0;
2177 };
2178
2179 static void tcm_vhost_deregister_configfs(void)
2180 {
2181         if (!tcm_vhost_fabric_configfs)
2182                 return;
2183
2184         target_fabric_configfs_deregister(tcm_vhost_fabric_configfs);
2185         tcm_vhost_fabric_configfs = NULL;
2186         pr_debug("TCM_VHOST[0] - Cleared tcm_vhost_fabric_configfs\n");
2187 };
2188
2189 static int __init tcm_vhost_init(void)
2190 {
2191         int ret = -ENOMEM;
2192         /*
2193          * Use our own dedicated workqueue for submitting I/O into
2194          * target core to avoid contention within system_wq.
2195          */
2196         tcm_vhost_workqueue = alloc_workqueue("tcm_vhost", 0, 0);
2197         if (!tcm_vhost_workqueue)
2198                 goto out;
2199
2200         ret = vhost_scsi_register();
2201         if (ret < 0)
2202                 goto out_destroy_workqueue;
2203
2204         ret = tcm_vhost_register_configfs();
2205         if (ret < 0)
2206                 goto out_vhost_scsi_deregister;
2207
2208         return 0;
2209
2210 out_vhost_scsi_deregister:
2211         vhost_scsi_deregister();
2212 out_destroy_workqueue:
2213         destroy_workqueue(tcm_vhost_workqueue);
2214 out:
2215         return ret;
2216 };
2217
2218 static void tcm_vhost_exit(void)
2219 {
2220         tcm_vhost_deregister_configfs();
2221         vhost_scsi_deregister();
2222         destroy_workqueue(tcm_vhost_workqueue);
2223 };
2224
2225 MODULE_DESCRIPTION("VHOST_SCSI series fabric driver");
2226 MODULE_ALIAS("tcm_vhost");
2227 MODULE_LICENSE("GPL");
2228 module_init(tcm_vhost_init);
2229 module_exit(tcm_vhost_exit);