drivers/target/iscsi/iscsi_target.c

   1 /*******************************************************************************
   2  * This file contains main functions related to the iSCSI Target Core Driver.
   3  *
   4  * \u00a9 Copyright 2007-2011 RisingTide Systems LLC.
   5  *
   6  * Licensed to the Linux Foundation under the General Public License (GPL) version 2.
   7  *
   8  * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License as published by
  12  * the Free Software Foundation; either version 2 of the License, or
  13  * (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  ******************************************************************************/
  20
  21 #include <linux/string.h>
  22 #include <linux/kthread.h>
  23 #include <linux/crypto.h>
  24 #include <linux/completion.h>
  25 #include <linux/module.h>
  26 #include <linux/idr.h>
  27 #include <asm/unaligned.h>
  28 #include <scsi/scsi_device.h>
  29 #include <scsi/iscsi_proto.h>
  30 #include <target/target_core_base.h>
  31 #include <target/target_core_fabric.h>
  32
  33 #include "iscsi_target_core.h"
  34 #include "iscsi_target_parameters.h"
  35 #include "iscsi_target_seq_pdu_list.h"
  36 #include "iscsi_target_tq.h"
  37 #include "iscsi_target_configfs.h"
  38 #include "iscsi_target_datain_values.h"
  39 #include "iscsi_target_erl0.h"
  40 #include "iscsi_target_erl1.h"
  41 #include "iscsi_target_erl2.h"
  42 #include "iscsi_target_login.h"
  43 #include "iscsi_target_tmr.h"
  44 #include "iscsi_target_tpg.h"
  45 #include "iscsi_target_util.h"
  46 #include "iscsi_target.h"
  47 #include "iscsi_target_device.h"
  48 #include "iscsi_target_stat.h"
  49
  50 static LIST_HEAD(g_tiqn_list);
  51 static LIST_HEAD(g_np_list);
  52 static DEFINE_SPINLOCK(tiqn_lock);
  53 static DEFINE_SPINLOCK(np_lock);
  54
  55 static struct idr tiqn_idr;
  56 struct idr sess_idr;
  57 struct mutex auth_id_lock;
  58 spinlock_t sess_idr_lock;
  59
  60 struct iscsit_global *iscsit_global;
  61
  62 struct kmem_cache *lio_cmd_cache;
  63 struct kmem_cache *lio_qr_cache;
  64 struct kmem_cache *lio_dr_cache;
  65 struct kmem_cache *lio_ooo_cache;
  66 struct kmem_cache *lio_r2t_cache;
  67
  68 static int iscsit_handle_immediate_data(struct iscsi_cmd *,
  69                         unsigned char *buf, u32);
  70 static int iscsit_logout_post_handler(struct iscsi_cmd *, struct iscsi_conn *);
  71
  72 struct iscsi_tiqn *iscsit_get_tiqn_for_login(unsigned char *buf)
  73 {
  74         struct iscsi_tiqn *tiqn = NULL;
  75
  76         spin_lock(&tiqn_lock);
  77         list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) {
  78                 if (!strcmp(tiqn->tiqn, buf)) {
  79
  80                         spin_lock(&tiqn->tiqn_state_lock);
  81                         if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) {
  82                                 tiqn->tiqn_access_count++;
  83                                 spin_unlock(&tiqn->tiqn_state_lock);
  84                                 spin_unlock(&tiqn_lock);
  85                                 return tiqn;
  86                         }
  87                         spin_unlock(&tiqn->tiqn_state_lock);
  88                 }
  89         }
  90         spin_unlock(&tiqn_lock);
  91
  92         return NULL;
  93 }
  94
  95 static int iscsit_set_tiqn_shutdown(struct iscsi_tiqn *tiqn)
  96 {
  97         spin_lock(&tiqn->tiqn_state_lock);
  98         if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) {
  99                 tiqn->tiqn_state = TIQN_STATE_SHUTDOWN;
 100                 spin_unlock(&tiqn->tiqn_state_lock);
 101                 return 0;
 102         }
 103         spin_unlock(&tiqn->tiqn_state_lock);
 104
 105         return -1;
 106 }
 107
 108 void iscsit_put_tiqn_for_login(struct iscsi_tiqn *tiqn)
 109 {
 110         spin_lock(&tiqn->tiqn_state_lock);
 111         tiqn->tiqn_access_count--;
 112         spin_unlock(&tiqn->tiqn_state_lock);
 113 }
 114
 115 /*
 116  * Note that IQN formatting is expected to be done in userspace, and
 117  * no explict IQN format checks are done here.
 118  */
 119 struct iscsi_tiqn *iscsit_add_tiqn(unsigned char *buf)
 120 {
 121         struct iscsi_tiqn *tiqn = NULL;
 122         int ret;
 123
 124         if (strlen(buf) >= ISCSI_IQN_LEN) {
 125                 pr_err("Target IQN exceeds %d bytes\n",
 126                                 ISCSI_IQN_LEN);
 127                 return ERR_PTR(-EINVAL);
 128         }
 129
 130         tiqn = kzalloc(sizeof(struct iscsi_tiqn), GFP_KERNEL);
 131         if (!tiqn) {
 132                 pr_err("Unable to allocate struct iscsi_tiqn\n");
 133                 return ERR_PTR(-ENOMEM);
 134         }
 135
 136         sprintf(tiqn->tiqn, "%s", buf);
 137         INIT_LIST_HEAD(&tiqn->tiqn_list);
 138         INIT_LIST_HEAD(&tiqn->tiqn_tpg_list);
 139         spin_lock_init(&tiqn->tiqn_state_lock);
 140         spin_lock_init(&tiqn->tiqn_tpg_lock);
 141         spin_lock_init(&tiqn->sess_err_stats.lock);
 142         spin_lock_init(&tiqn->login_stats.lock);
 143         spin_lock_init(&tiqn->logout_stats.lock);
 144
 145         if (!idr_pre_get(&tiqn_idr, GFP_KERNEL)) {
 146                 pr_err("idr_pre_get() for tiqn_idr failed\n");
 147                 kfree(tiqn);
 148                 return ERR_PTR(-ENOMEM);
 149         }
 150         tiqn->tiqn_state = TIQN_STATE_ACTIVE;
 151
 152         spin_lock(&tiqn_lock);
 153         ret = idr_get_new(&tiqn_idr, NULL, &tiqn->tiqn_index);
 154         if (ret < 0) {
 155                 pr_err("idr_get_new() failed for tiqn->tiqn_index\n");
 156                 spin_unlock(&tiqn_lock);
 157                 kfree(tiqn);
 158                 return ERR_PTR(ret);
 159         }
 160         list_add_tail(&tiqn->tiqn_list, &g_tiqn_list);
 161         spin_unlock(&tiqn_lock);
 162
 163         pr_debug("CORE[0] - Added iSCSI Target IQN: %s\n", tiqn->tiqn);
 164
 165         return tiqn;
 166
 167 }
 168
 169 static void iscsit_wait_for_tiqn(struct iscsi_tiqn *tiqn)
 170 {
 171         /*
 172          * Wait for accesses to said struct iscsi_tiqn to end.
 173          */
 174         spin_lock(&tiqn->tiqn_state_lock);
 175         while (tiqn->tiqn_access_count != 0) {
 176                 spin_unlock(&tiqn->tiqn_state_lock);
 177                 msleep(10);
 178                 spin_lock(&tiqn->tiqn_state_lock);
 179         }
 180         spin_unlock(&tiqn->tiqn_state_lock);
 181 }
 182
 183 void iscsit_del_tiqn(struct iscsi_tiqn *tiqn)
 184 {
 185         /*
 186          * iscsit_set_tiqn_shutdown sets tiqn->tiqn_state = TIQN_STATE_SHUTDOWN
 187          * while holding tiqn->tiqn_state_lock.  This means that all subsequent
 188          * attempts to access this struct iscsi_tiqn will fail from both transport
 189          * fabric and control code paths.
 190          */
 191         if (iscsit_set_tiqn_shutdown(tiqn) < 0) {
 192                 pr_err("iscsit_set_tiqn_shutdown() failed\n");
 193                 return;
 194         }
 195
 196         iscsit_wait_for_tiqn(tiqn);
 197
 198         spin_lock(&tiqn_lock);
 199         list_del(&tiqn->tiqn_list);
 200         idr_remove(&tiqn_idr, tiqn->tiqn_index);
 201         spin_unlock(&tiqn_lock);
 202
 203         pr_debug("CORE[0] - Deleted iSCSI Target IQN: %s\n",
 204                         tiqn->tiqn);
 205         kfree(tiqn);
 206 }
 207
 208 int iscsit_access_np(struct iscsi_np *np, struct iscsi_portal_group *tpg)
 209 {
 210         int ret;
 211         /*
 212          * Determine if the network portal is accepting storage traffic.
 213          */
 214         spin_lock_bh(&np->np_thread_lock);
 215         if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) {
 216                 spin_unlock_bh(&np->np_thread_lock);
 217                 return -1;
 218         }
 219         if (np->np_login_tpg) {
 220                 pr_err("np->np_login_tpg() is not NULL!\n");
 221                 spin_unlock_bh(&np->np_thread_lock);
 222                 return -1;
 223         }
 224         spin_unlock_bh(&np->np_thread_lock);
 225         /*
 226          * Determine if the portal group is accepting storage traffic.
 227          */
 228         spin_lock_bh(&tpg->tpg_state_lock);
 229         if (tpg->tpg_state != TPG_STATE_ACTIVE) {
 230                 spin_unlock_bh(&tpg->tpg_state_lock);
 231                 return -1;
 232         }
 233         spin_unlock_bh(&tpg->tpg_state_lock);
 234
 235         /*
 236          * Here we serialize access across the TIQN+TPG Tuple.
 237          */
 238         ret = mutex_lock_interruptible(&tpg->np_login_lock);
 239         if ((ret != 0) || signal_pending(current))
 240                 return -1;
 241
 242         spin_lock_bh(&np->np_thread_lock);
 243         np->np_login_tpg = tpg;
 244         spin_unlock_bh(&np->np_thread_lock);
 245
 246         return 0;
 247 }
 248
 249 int iscsit_deaccess_np(struct iscsi_np *np, struct iscsi_portal_group *tpg)
 250 {
 251         struct iscsi_tiqn *tiqn = tpg->tpg_tiqn;
 252
 253         spin_lock_bh(&np->np_thread_lock);
 254         np->np_login_tpg = NULL;
 255         spin_unlock_bh(&np->np_thread_lock);
 256
 257         mutex_unlock(&tpg->np_login_lock);
 258
 259         if (tiqn)
 260                 iscsit_put_tiqn_for_login(tiqn);
 261
 262         return 0;
 263 }
 264
 265 static struct iscsi_np *iscsit_get_np(
 266         struct __kernel_sockaddr_storage *sockaddr,
 267         int network_transport)
 268 {
 269         struct sockaddr_in *sock_in, *sock_in_e;
 270         struct sockaddr_in6 *sock_in6, *sock_in6_e;
 271         struct iscsi_np *np;
 272         int ip_match = 0;
 273         u16 port;
 274
 275         spin_lock_bh(&np_lock);
 276         list_for_each_entry(np, &g_np_list, np_list) {
 277                 spin_lock(&np->np_thread_lock);
 278                 if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) {
 279                         spin_unlock(&np->np_thread_lock);
 280                         continue;
 281                 }
 282
 283                 if (sockaddr->ss_family == AF_INET6) {
 284                         sock_in6 = (struct sockaddr_in6 *)sockaddr;
 285                         sock_in6_e = (struct sockaddr_in6 *)&np->np_sockaddr;
 286
 287                         if (!memcmp(&sock_in6->sin6_addr.in6_u,
 288                                     &sock_in6_e->sin6_addr.in6_u,
 289                                     sizeof(struct in6_addr)))
 290                                 ip_match = 1;
 291
 292                         port = ntohs(sock_in6->sin6_port);
 293                 } else {
 294                         sock_in = (struct sockaddr_in *)sockaddr;
 295                         sock_in_e = (struct sockaddr_in *)&np->np_sockaddr;
 296
 297                         if (sock_in->sin_addr.s_addr ==
 298                             sock_in_e->sin_addr.s_addr)
 299                                 ip_match = 1;
 300
 301                         port = ntohs(sock_in->sin_port);
 302                 }
 303
 304                 if ((ip_match == 1) && (np->np_port == port) &&
 305                     (np->np_network_transport == network_transport)) {
 306                         /*
 307                          * Increment the np_exports reference count now to
 308                          * prevent iscsit_del_np() below from being called
 309                          * while iscsi_tpg_add_network_portal() is called.
 310                          */
 311                         np->np_exports++;
 312                         spin_unlock(&np->np_thread_lock);
 313                         spin_unlock_bh(&np_lock);
 314                         return np;
 315                 }
 316                 spin_unlock(&np->np_thread_lock);
 317         }
 318         spin_unlock_bh(&np_lock);
 319
 320         return NULL;
 321 }
 322
 323 struct iscsi_np *iscsit_add_np(
 324         struct __kernel_sockaddr_storage *sockaddr,
 325         char *ip_str,
 326         int network_transport)
 327 {
 328         struct sockaddr_in *sock_in;
 329         struct sockaddr_in6 *sock_in6;
 330         struct iscsi_np *np;
 331         int ret;
 332         /*
 333          * Locate the existing struct iscsi_np if already active..
 334          */
 335         np = iscsit_get_np(sockaddr, network_transport);
 336         if (np)
 337                 return np;
 338
 339         np = kzalloc(sizeof(struct iscsi_np), GFP_KERNEL);
 340         if (!np) {
 341                 pr_err("Unable to allocate memory for struct iscsi_np\n");
 342                 return ERR_PTR(-ENOMEM);
 343         }
 344
 345         np->np_flags |= NPF_IP_NETWORK;
 346         if (sockaddr->ss_family == AF_INET6) {
 347                 sock_in6 = (struct sockaddr_in6 *)sockaddr;
 348                 snprintf(np->np_ip, IPV6_ADDRESS_SPACE, "%s", ip_str);
 349                 np->np_port = ntohs(sock_in6->sin6_port);
 350         } else {
 351                 sock_in = (struct sockaddr_in *)sockaddr;
 352                 sprintf(np->np_ip, "%s", ip_str);
 353                 np->np_port = ntohs(sock_in->sin_port);
 354         }
 355
 356         np->np_network_transport = network_transport;
 357         spin_lock_init(&np->np_thread_lock);
 358         init_completion(&np->np_restart_comp);
 359         INIT_LIST_HEAD(&np->np_list);
 360
 361         ret = iscsi_target_setup_login_socket(np, sockaddr);
 362         if (ret != 0) {
 363                 kfree(np);
 364                 return ERR_PTR(ret);
 365         }
 366
 367         np->np_thread = kthread_run(iscsi_target_login_thread, np, "iscsi_np");
 368         if (IS_ERR(np->np_thread)) {
 369                 pr_err("Unable to create kthread: iscsi_np\n");
 370                 ret = PTR_ERR(np->np_thread);
 371                 kfree(np);
 372                 return ERR_PTR(ret);
 373         }
 374         /*
 375          * Increment the np_exports reference count now to prevent
 376          * iscsit_del_np() below from being run while a new call to
 377          * iscsi_tpg_add_network_portal() for a matching iscsi_np is
 378          * active.  We don't need to hold np->np_thread_lock at this
 379          * point because iscsi_np has not been added to g_np_list yet.
 380          */
 381         np->np_exports = 1;
 382
 383         spin_lock_bh(&np_lock);
 384         list_add_tail(&np->np_list, &g_np_list);
 385         spin_unlock_bh(&np_lock);
 386
 387         pr_debug("CORE[0] - Added Network Portal: %s:%hu on %s\n",
 388                 np->np_ip, np->np_port, (np->np_network_transport == ISCSI_TCP) ?
 389                 "TCP" : "SCTP");
 390
 391         return np;
 392 }
 393
 394 int iscsit_reset_np_thread(
 395         struct iscsi_np *np,
 396         struct iscsi_tpg_np *tpg_np,
 397         struct iscsi_portal_group *tpg)
 398 {
 399         spin_lock_bh(&np->np_thread_lock);
 400         if (tpg && tpg_np) {
 401                 /*
 402                  * The reset operation need only be performed when the
 403                  * passed struct iscsi_portal_group has a login in progress
 404                  * to one of the network portals.
 405                  */
 406                 if (tpg_np->tpg_np->np_login_tpg != tpg) {
 407                         spin_unlock_bh(&np->np_thread_lock);
 408                         return 0;
 409                 }
 410         }
 411         if (np->np_thread_state == ISCSI_NP_THREAD_INACTIVE) {
 412                 spin_unlock_bh(&np->np_thread_lock);
 413                 return 0;
 414         }
 415         np->np_thread_state = ISCSI_NP_THREAD_RESET;
 416
 417         if (np->np_thread) {
 418                 spin_unlock_bh(&np->np_thread_lock);
 419                 send_sig(SIGINT, np->np_thread, 1);
 420                 wait_for_completion(&np->np_restart_comp);
 421                 spin_lock_bh(&np->np_thread_lock);
 422         }
 423         spin_unlock_bh(&np->np_thread_lock);
 424
 425         return 0;
 426 }
 427
 428 int iscsit_del_np_comm(struct iscsi_np *np)
 429 {
 430         if (!np->np_socket)
 431                 return 0;
 432
 433         /*
 434          * Some network transports allocate their own struct sock->file,
 435          * see  if we need to free any additional allocated resources.
 436          */
 437         if (np->np_flags & NPF_SCTP_STRUCT_FILE) {
 438                 kfree(np->np_socket->file);
 439                 np->np_socket->file = NULL;
 440         }
 441
 442         sock_release(np->np_socket);
 443         return 0;
 444 }
 445
 446 int iscsit_del_np(struct iscsi_np *np)
 447 {
 448         spin_lock_bh(&np->np_thread_lock);
 449         np->np_exports--;
 450         if (np->np_exports) {
 451                 spin_unlock_bh(&np->np_thread_lock);
 452                 return 0;
 453         }
 454         np->np_thread_state = ISCSI_NP_THREAD_SHUTDOWN;
 455         spin_unlock_bh(&np->np_thread_lock);
 456
 457         if (np->np_thread) {
 458                 /*
 459                  * We need to send the signal to wakeup Linux/Net
 460                  * which may be sleeping in sock_accept()..
 461                  */
 462                 send_sig(SIGINT, np->np_thread, 1);
 463                 kthread_stop(np->np_thread);
 464         }
 465         iscsit_del_np_comm(np);
 466
 467         spin_lock_bh(&np_lock);
 468         list_del(&np->np_list);
 469         spin_unlock_bh(&np_lock);
 470
 471         pr_debug("CORE[0] - Removed Network Portal: %s:%hu on %s\n",
 472                 np->np_ip, np->np_port, (np->np_network_transport == ISCSI_TCP) ?
 473                 "TCP" : "SCTP");
 474
 475         kfree(np);
 476         return 0;
 477 }
 478
 479 static int __init iscsi_target_init_module(void)
 480 {
 481         int ret = 0;
 482
 483         pr_debug("iSCSI-Target "ISCSIT_VERSION"\n");
 484
 485         iscsit_global = kzalloc(sizeof(struct iscsit_global), GFP_KERNEL);
 486         if (!iscsit_global) {
 487                 pr_err("Unable to allocate memory for iscsit_global\n");
 488                 return -1;
 489         }
 490         mutex_init(&auth_id_lock);
 491         spin_lock_init(&sess_idr_lock);
 492         idr_init(&tiqn_idr);
 493         idr_init(&sess_idr);
 494
 495         ret = iscsi_target_register_configfs();
 496         if (ret < 0)
 497                 goto out;
 498
 499         ret = iscsi_thread_set_init();
 500         if (ret < 0)
 501                 goto configfs_out;
 502
 503         if (iscsi_allocate_thread_sets(TARGET_THREAD_SET_COUNT) !=
 504                         TARGET_THREAD_SET_COUNT) {
 505                 pr_err("iscsi_allocate_thread_sets() returned"
 506                         " unexpected value!\n");
 507                 goto ts_out1;
 508         }
 509
 510         lio_cmd_cache = kmem_cache_create("lio_cmd_cache",
 511                         sizeof(struct iscsi_cmd), __alignof__(struct iscsi_cmd),
 512                         0, NULL);
 513         if (!lio_cmd_cache) {
 514                 pr_err("Unable to kmem_cache_create() for"
 515                                 " lio_cmd_cache\n");
 516                 goto ts_out2;
 517         }
 518
 519         lio_qr_cache = kmem_cache_create("lio_qr_cache",
 520                         sizeof(struct iscsi_queue_req),
 521                         __alignof__(struct iscsi_queue_req), 0, NULL);
 522         if (!lio_qr_cache) {
 523                 pr_err("nable to kmem_cache_create() for"
 524                                 " lio_qr_cache\n");
 525                 goto cmd_out;
 526         }
 527
 528         lio_dr_cache = kmem_cache_create("lio_dr_cache",
 529                         sizeof(struct iscsi_datain_req),
 530                         __alignof__(struct iscsi_datain_req), 0, NULL);
 531         if (!lio_dr_cache) {
 532                 pr_err("Unable to kmem_cache_create() for"
 533                                 " lio_dr_cache\n");
 534                 goto qr_out;
 535         }
 536
 537         lio_ooo_cache = kmem_cache_create("lio_ooo_cache",
 538                         sizeof(struct iscsi_ooo_cmdsn),
 539                         __alignof__(struct iscsi_ooo_cmdsn), 0, NULL);
 540         if (!lio_ooo_cache) {
 541                 pr_err("Unable to kmem_cache_create() for"
 542                                 " lio_ooo_cache\n");
 543                 goto dr_out;
 544         }
 545
 546         lio_r2t_cache = kmem_cache_create("lio_r2t_cache",
 547                         sizeof(struct iscsi_r2t), __alignof__(struct iscsi_r2t),
 548                         0, NULL);
 549         if (!lio_r2t_cache) {
 550                 pr_err("Unable to kmem_cache_create() for"
 551                                 " lio_r2t_cache\n");
 552                 goto ooo_out;
 553         }
 554
 555         if (iscsit_load_discovery_tpg() < 0)
 556                 goto r2t_out;
 557
 558         return ret;
 559 r2t_out:
 560         kmem_cache_destroy(lio_r2t_cache);
 561 ooo_out:
 562         kmem_cache_destroy(lio_ooo_cache);
 563 dr_out:
 564         kmem_cache_destroy(lio_dr_cache);
 565 qr_out:
 566         kmem_cache_destroy(lio_qr_cache);
 567 cmd_out:
 568         kmem_cache_destroy(lio_cmd_cache);
 569 ts_out2:
 570         iscsi_deallocate_thread_sets();
 571 ts_out1:
 572         iscsi_thread_set_free();
 573 configfs_out:
 574         iscsi_target_deregister_configfs();
 575 out:
 576         kfree(iscsit_global);
 577         return -ENOMEM;
 578 }
 579
 580 static void __exit iscsi_target_cleanup_module(void)
 581 {
 582         iscsi_deallocate_thread_sets();
 583         iscsi_thread_set_free();
 584         iscsit_release_discovery_tpg();
 585         kmem_cache_destroy(lio_cmd_cache);
 586         kmem_cache_destroy(lio_qr_cache);
 587         kmem_cache_destroy(lio_dr_cache);
 588         kmem_cache_destroy(lio_ooo_cache);
 589         kmem_cache_destroy(lio_r2t_cache);
 590
 591         iscsi_target_deregister_configfs();
 592
 593         kfree(iscsit_global);
 594 }
 595
 596 int iscsit_add_reject(
 597         u8 reason,
 598         int fail_conn,
 599         unsigned char *buf,
 600         struct iscsi_conn *conn)
 601 {
 602         struct iscsi_cmd *cmd;
 603         struct iscsi_reject *hdr;
 604         int ret;
 605
 606         cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
 607         if (!cmd)
 608                 return -1;
 609
 610         cmd->iscsi_opcode = ISCSI_OP_REJECT;
 611         if (fail_conn)
 612                 cmd->cmd_flags |= ICF_REJECT_FAIL_CONN;
 613
 614         hdr     = (struct iscsi_reject *) cmd->pdu;
 615         hdr->reason = reason;
 616
 617         cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL);
 618         if (!cmd->buf_ptr) {
 619                 pr_err("Unable to allocate memory for cmd->buf_ptr\n");
 620                 iscsit_release_cmd(cmd);
 621                 return -1;
 622         }
 623
 624         spin_lock_bh(&conn->cmd_lock);
 625         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
 626         spin_unlock_bh(&conn->cmd_lock);
 627
 628         cmd->i_state = ISTATE_SEND_REJECT;
 629         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
 630
 631         ret = wait_for_completion_interruptible(&cmd->reject_comp);
 632         if (ret != 0)
 633                 return -1;
 634
 635         return (!fail_conn) ? 0 : -1;
 636 }
 637
 638 int iscsit_add_reject_from_cmd(
 639         u8 reason,
 640         int fail_conn,
 641         int add_to_conn,
 642         unsigned char *buf,
 643         struct iscsi_cmd *cmd)
 644 {
 645         struct iscsi_conn *conn;
 646         struct iscsi_reject *hdr;
 647         int ret;
 648
 649         if (!cmd->conn) {
 650                 pr_err("cmd->conn is NULL for ITT: 0x%08x\n",
 651                                 cmd->init_task_tag);
 652                 return -1;
 653         }
 654         conn = cmd->conn;
 655
 656         cmd->iscsi_opcode = ISCSI_OP_REJECT;
 657         if (fail_conn)
 658                 cmd->cmd_flags |= ICF_REJECT_FAIL_CONN;
 659
 660         hdr     = (struct iscsi_reject *) cmd->pdu;
 661         hdr->reason = reason;
 662
 663         cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL);
 664         if (!cmd->buf_ptr) {
 665                 pr_err("Unable to allocate memory for cmd->buf_ptr\n");
 666                 iscsit_release_cmd(cmd);
 667                 return -1;
 668         }
 669
 670         if (add_to_conn) {
 671                 spin_lock_bh(&conn->cmd_lock);
 672                 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
 673                 spin_unlock_bh(&conn->cmd_lock);
 674         }
 675
 676         cmd->i_state = ISTATE_SEND_REJECT;
 677         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
 678
 679         ret = wait_for_completion_interruptible(&cmd->reject_comp);
 680         if (ret != 0)
 681                 return -1;
 682
 683         return (!fail_conn) ? 0 : -1;
 684 }
 685
 686 /*
 687  * Map some portion of the allocated scatterlist to an iovec, suitable for
 688  * kernel sockets to copy data in/out. This handles both pages and slab-allocated
 689  * buffers, since we have been tricky and mapped t_mem_sg to the buffer in
 690  * either case (see iscsit_alloc_buffs)
 691  */
 692 static int iscsit_map_iovec(
 693         struct iscsi_cmd *cmd,
 694         struct kvec *iov,
 695         u32 data_offset,
 696         u32 data_length)
 697 {
 698         u32 i = 0;
 699         struct scatterlist *sg;
 700         unsigned int page_off;
 701
 702         /*
 703          * We have a private mapping of the allocated pages in t_mem_sg.
 704          * At this point, we also know each contains a page.
 705          */
 706         sg = &cmd->t_mem_sg[data_offset / PAGE_SIZE];
 707         page_off = (data_offset % PAGE_SIZE);
 708
 709         cmd->first_data_sg = sg;
 710         cmd->first_data_sg_off = page_off;
 711
 712         while (data_length) {
 713                 u32 cur_len = min_t(u32, data_length, sg->length - page_off);
 714
 715                 iov[i].iov_base = kmap(sg_page(sg)) + sg->offset + page_off;
 716                 iov[i].iov_len = cur_len;
 717
 718                 data_length -= cur_len;
 719                 page_off = 0;
 720                 sg = sg_next(sg);
 721                 i++;
 722         }
 723
 724         cmd->kmapped_nents = i;
 725
 726         return i;
 727 }
 728
 729 static void iscsit_unmap_iovec(struct iscsi_cmd *cmd)
 730 {
 731         u32 i;
 732         struct scatterlist *sg;
 733
 734         sg = cmd->first_data_sg;
 735
 736         for (i = 0; i < cmd->kmapped_nents; i++)
 737                 kunmap(sg_page(&sg[i]));
 738 }
 739
 740 static void iscsit_ack_from_expstatsn(struct iscsi_conn *conn, u32 exp_statsn)
 741 {
 742         struct iscsi_cmd *cmd;
 743
 744         conn->exp_statsn = exp_statsn;
 745
 746         spin_lock_bh(&conn->cmd_lock);
 747         list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
 748                 spin_lock(&cmd->istate_lock);
 749                 if ((cmd->i_state == ISTATE_SENT_STATUS) &&
 750                     (cmd->stat_sn < exp_statsn)) {
 751                         cmd->i_state = ISTATE_REMOVE;
 752                         spin_unlock(&cmd->istate_lock);
 753                         iscsit_add_cmd_to_immediate_queue(cmd, conn,
 754                                                 cmd->i_state);
 755                         continue;
 756                 }
 757                 spin_unlock(&cmd->istate_lock);
 758         }
 759         spin_unlock_bh(&conn->cmd_lock);
 760 }
 761
 762 static int iscsit_allocate_iovecs(struct iscsi_cmd *cmd)
 763 {
 764         u32 iov_count = (cmd->se_cmd.t_data_nents == 0) ? 1 :
 765                                 cmd->se_cmd.t_data_nents;
 766
 767         iov_count += ISCSI_IOV_DATA_BUFFER;
 768
 769         cmd->iov_data = kzalloc(iov_count * sizeof(struct kvec), GFP_KERNEL);
 770         if (!cmd->iov_data) {
 771                 pr_err("Unable to allocate cmd->iov_data\n");
 772                 return -ENOMEM;
 773         }
 774
 775         cmd->orig_iov_data_count = iov_count;
 776         return 0;
 777 }
 778
 779 static int iscsit_alloc_buffs(struct iscsi_cmd *cmd)
 780 {
 781         struct scatterlist *sgl;
 782         u32 length = cmd->se_cmd.data_length;
 783         int nents = DIV_ROUND_UP(length, PAGE_SIZE);
 784         int i = 0, j = 0, ret;
 785         /*
 786          * If no SCSI payload is present, allocate the default iovecs used for
 787          * iSCSI PDU Header
 788          */
 789         if (!length)
 790                 return iscsit_allocate_iovecs(cmd);
 791
 792         sgl = kzalloc(sizeof(*sgl) * nents, GFP_KERNEL);
 793         if (!sgl)
 794                 return -ENOMEM;
 795
 796         sg_init_table(sgl, nents);
 797
 798         while (length) {
 799                 int buf_size = min_t(int, length, PAGE_SIZE);
 800                 struct page *page;
 801
 802                 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 803                 if (!page)
 804                         goto page_alloc_failed;
 805
 806                 sg_set_page(&sgl[i], page, buf_size, 0);
 807
 808                 length -= buf_size;
 809                 i++;
 810         }
 811
 812         cmd->t_mem_sg = sgl;
 813         cmd->t_mem_sg_nents = nents;
 814
 815         /* BIDI ops not supported */
 816
 817         /* Tell the core about our preallocated memory */
 818         transport_generic_map_mem_to_cmd(&cmd->se_cmd, sgl, nents, NULL, 0);
 819         /*
 820          * Allocate iovecs for SCSI payload after transport_generic_map_mem_to_cmd
 821          * so that cmd->se_cmd.t_tasks_se_num has been set.
 822          */
 823         ret = iscsit_allocate_iovecs(cmd);
 824         if (ret < 0)
 825                 return -ENOMEM;
 826
 827         return 0;
 828
 829 page_alloc_failed:
 830         while (j < i)
 831                 __free_page(sg_page(&sgl[j++]));
 832
 833         kfree(sgl);
 834         return -ENOMEM;
 835 }
 836
 837 static int iscsit_handle_scsi_cmd(
 838         struct iscsi_conn *conn,
 839         unsigned char *buf)
 840 {
 841         int     data_direction, cmdsn_ret = 0, immed_ret, ret, transport_ret;
 842         int     dump_immediate_data = 0, send_check_condition = 0, payload_length;
 843         struct iscsi_cmd        *cmd = NULL;
 844         struct iscsi_scsi_req *hdr;
 845
 846         spin_lock_bh(&conn->sess->session_stats_lock);
 847         conn->sess->cmd_pdus++;
 848         if (conn->sess->se_sess->se_node_acl) {
 849                 spin_lock(&conn->sess->se_sess->se_node_acl->stats_lock);
 850                 conn->sess->se_sess->se_node_acl->num_cmds++;
 851                 spin_unlock(&conn->sess->se_sess->se_node_acl->stats_lock);
 852         }
 853         spin_unlock_bh(&conn->sess->session_stats_lock);
 854
 855         hdr                     = (struct iscsi_scsi_req *) buf;
 856         payload_length          = ntoh24(hdr->dlength);
 857         hdr->itt                = be32_to_cpu(hdr->itt);
 858         hdr->data_length        = be32_to_cpu(hdr->data_length);
 859         hdr->cmdsn              = be32_to_cpu(hdr->cmdsn);
 860         hdr->exp_statsn         = be32_to_cpu(hdr->exp_statsn);
 861
 862         /* FIXME; Add checks for AdditionalHeaderSegment */
 863
 864         if (!(hdr->flags & ISCSI_FLAG_CMD_WRITE) &&
 865             !(hdr->flags & ISCSI_FLAG_CMD_FINAL)) {
 866                 pr_err("ISCSI_FLAG_CMD_WRITE & ISCSI_FLAG_CMD_FINAL"
 867                                 " not set. Bad iSCSI Initiator.\n");
 868                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1,
 869                                 buf, conn);
 870         }
 871
 872         if (((hdr->flags & ISCSI_FLAG_CMD_READ) ||
 873              (hdr->flags & ISCSI_FLAG_CMD_WRITE)) && !hdr->data_length) {
 874                 /*
 875                  * Vmware ESX v3.0 uses a modified Cisco Initiator (v3.4.2)
 876                  * that adds support for RESERVE/RELEASE.  There is a bug
 877                  * add with this new functionality that sets R/W bits when
 878                  * neither CDB carries any READ or WRITE datapayloads.
 879                  */
 880                 if ((hdr->cdb[0] == 0x16) || (hdr->cdb[0] == 0x17)) {
 881                         hdr->flags &= ~ISCSI_FLAG_CMD_READ;
 882                         hdr->flags &= ~ISCSI_FLAG_CMD_WRITE;
 883                         goto done;
 884                 }
 885
 886                 pr_err("ISCSI_FLAG_CMD_READ or ISCSI_FLAG_CMD_WRITE"
 887                         " set when Expected Data Transfer Length is 0 for"
 888                         " CDB: 0x%02x. Bad iSCSI Initiator.\n", hdr->cdb[0]);
 889                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1,
 890                                 buf, conn);
 891         }
 892 done:
 893
 894         if (!(hdr->flags & ISCSI_FLAG_CMD_READ) &&
 895             !(hdr->flags & ISCSI_FLAG_CMD_WRITE) && (hdr->data_length != 0)) {
 896                 pr_err("ISCSI_FLAG_CMD_READ and/or ISCSI_FLAG_CMD_WRITE"
 897                         " MUST be set if Expected Data Transfer Length is not 0."
 898                         " Bad iSCSI Initiator\n");
 899                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1,
 900                                 buf, conn);
 901         }
 902
 903         if ((hdr->flags & ISCSI_FLAG_CMD_READ) &&
 904             (hdr->flags & ISCSI_FLAG_CMD_WRITE)) {
 905                 pr_err("Bidirectional operations not supported!\n");
 906                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1,
 907                                 buf, conn);
 908         }
 909
 910         if (hdr->opcode & ISCSI_OP_IMMEDIATE) {
 911                 pr_err("Illegally set Immediate Bit in iSCSI Initiator"
 912                                 " Scsi Command PDU.\n");
 913                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1,
 914                                 buf, conn);
 915         }
 916
 917         if (payload_length && !conn->sess->sess_ops->ImmediateData) {
 918                 pr_err("ImmediateData=No but DataSegmentLength=%u,"
 919                         " protocol error.\n", payload_length);
 920                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
 921                                 buf, conn);
 922         }
 923
 924         if ((hdr->data_length == payload_length) &&
 925             (!(hdr->flags & ISCSI_FLAG_CMD_FINAL))) {
 926                 pr_err("Expected Data Transfer Length and Length of"
 927                         " Immediate Data are the same, but ISCSI_FLAG_CMD_FINAL"
 928                         " bit is not set protocol error\n");
 929                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
 930                                 buf, conn);
 931         }
 932
 933         if (payload_length > hdr->data_length) {
 934                 pr_err("DataSegmentLength: %u is greater than"
 935                         " EDTL: %u, protocol error.\n", payload_length,
 936                                 hdr->data_length);
 937                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
 938                                 buf, conn);
 939         }
 940
 941         if (payload_length > conn->conn_ops->MaxRecvDataSegmentLength) {
 942                 pr_err("DataSegmentLength: %u is greater than"
 943                         " MaxRecvDataSegmentLength: %u, protocol error.\n",
 944                         payload_length, conn->conn_ops->MaxRecvDataSegmentLength);
 945                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
 946                                 buf, conn);
 947         }
 948
 949         if (payload_length > conn->sess->sess_ops->FirstBurstLength) {
 950                 pr_err("DataSegmentLength: %u is greater than"
 951                         " FirstBurstLength: %u, protocol error.\n",
 952                         payload_length, conn->sess->sess_ops->FirstBurstLength);
 953                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1,
 954                                         buf, conn);
 955         }
 956
 957         data_direction = (hdr->flags & ISCSI_FLAG_CMD_WRITE) ? DMA_TO_DEVICE :
 958                          (hdr->flags & ISCSI_FLAG_CMD_READ) ? DMA_FROM_DEVICE :
 959                           DMA_NONE;
 960
 961         cmd = iscsit_allocate_se_cmd(conn, hdr->data_length, data_direction,
 962                                 (hdr->flags & ISCSI_FLAG_CMD_ATTR_MASK));
 963         if (!cmd)
 964                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1,
 965                                         buf, conn);
 966
 967         pr_debug("Got SCSI Command, ITT: 0x%08x, CmdSN: 0x%08x,"
 968                 " ExpXferLen: %u, Length: %u, CID: %hu\n", hdr->itt,
 969                 hdr->cmdsn, hdr->data_length, payload_length, conn->cid);
 970
 971         cmd->iscsi_opcode       = ISCSI_OP_SCSI_CMD;
 972         cmd->i_state            = ISTATE_NEW_CMD;
 973         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
 974         cmd->immediate_data     = (payload_length) ? 1 : 0;
 975         cmd->unsolicited_data   = ((!(hdr->flags & ISCSI_FLAG_CMD_FINAL) &&
 976                                      (hdr->flags & ISCSI_FLAG_CMD_WRITE)) ? 1 : 0);
 977         if (cmd->unsolicited_data)
 978                 cmd->cmd_flags |= ICF_NON_IMMEDIATE_UNSOLICITED_DATA;
 979
 980         conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt;
 981         if (hdr->flags & ISCSI_FLAG_CMD_READ) {
 982                 spin_lock_bh(&conn->sess->ttt_lock);
 983                 cmd->targ_xfer_tag = conn->sess->targ_xfer_tag++;
 984                 if (cmd->targ_xfer_tag == 0xFFFFFFFF)
 985                         cmd->targ_xfer_tag = conn->sess->targ_xfer_tag++;
 986                 spin_unlock_bh(&conn->sess->ttt_lock);
 987         } else if (hdr->flags & ISCSI_FLAG_CMD_WRITE)
 988                 cmd->targ_xfer_tag = 0xFFFFFFFF;
 989         cmd->cmd_sn             = hdr->cmdsn;
 990         cmd->exp_stat_sn        = hdr->exp_statsn;
 991         cmd->first_burst_len    = payload_length;
 992
 993         if (cmd->data_direction == DMA_FROM_DEVICE) {
 994                 struct iscsi_datain_req *dr;
 995
 996                 dr = iscsit_allocate_datain_req();
 997                 if (!dr)
 998                         return iscsit_add_reject_from_cmd(
 999                                         ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1000                                         1, 1, buf, cmd);
1001
1002                 iscsit_attach_datain_req(cmd, dr);
1003         }
1004
1005         /*
1006          * The CDB is going to an se_device_t.
1007          */
1008         ret = transport_lookup_cmd_lun(&cmd->se_cmd,
1009                                        scsilun_to_int(&hdr->lun));
1010         if (ret < 0) {
1011                 if (cmd->se_cmd.scsi_sense_reason == TCM_NON_EXISTENT_LUN) {
1012                         pr_debug("Responding to non-acl'ed,"
1013                                 " non-existent or non-exported iSCSI LUN:"
1014                                 " 0x%016Lx\n", get_unaligned_le64(&hdr->lun));
1015                 }
1016                 send_check_condition = 1;
1017                 goto attach_cmd;
1018         }
1019         /*
1020          * The Initiator Node has access to the LUN (the addressing method
1021          * is handled inside of iscsit_get_lun_for_cmd()).  Now it's time to
1022          * allocate 1->N transport tasks (depending on sector count and
1023          * maximum request size the physical HBA(s) can handle.
1024          */
1025         transport_ret = transport_generic_allocate_tasks(&cmd->se_cmd, hdr->cdb);
1026         if (transport_ret == -ENOMEM) {
1027                 return iscsit_add_reject_from_cmd(
1028                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1029                                 1, 1, buf, cmd);
1030         } else if (transport_ret < 0) {
1031                 /*
1032                  * Unsupported SAM Opcode.  CHECK_CONDITION will be sent
1033                  * in iscsit_execute_cmd() during the CmdSN OOO Execution
1034                  * Mechinism.
1035                  */
1036                 send_check_condition = 1;
1037         } else {
1038                 cmd->data_length = cmd->se_cmd.data_length;
1039
1040                 if (iscsit_decide_list_to_build(cmd, payload_length) < 0)
1041                         return iscsit_add_reject_from_cmd(
1042                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1043                                 1, 1, buf, cmd);
1044         }
1045
1046 attach_cmd:
1047         spin_lock_bh(&conn->cmd_lock);
1048         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
1049         spin_unlock_bh(&conn->cmd_lock);
1050         /*
1051          * Check if we need to delay processing because of ALUA
1052          * Active/NonOptimized primary access state..
1053          */
1054         core_alua_check_nonop_delay(&cmd->se_cmd);
1055         /*
1056          * Allocate and setup SGL used with transport_generic_map_mem_to_cmd().
1057          * also call iscsit_allocate_iovecs()
1058          */
1059         ret = iscsit_alloc_buffs(cmd);
1060         if (ret < 0)
1061                 return iscsit_add_reject_from_cmd(
1062                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1063                                 1, 0, buf, cmd);
1064         /*
1065          * Check the CmdSN against ExpCmdSN/MaxCmdSN here if
1066          * the Immediate Bit is not set, and no Immediate
1067          * Data is attached.
1068          *
1069          * A PDU/CmdSN carrying Immediate Data can only
1070          * be processed after the DataCRC has passed.
1071          * If the DataCRC fails, the CmdSN MUST NOT
1072          * be acknowledged. (See below)
1073          */
1074         if (!cmd->immediate_data) {
1075                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
1076                 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP)
1077                         return 0;
1078                 else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1079                         return iscsit_add_reject_from_cmd(
1080                                 ISCSI_REASON_PROTOCOL_ERROR,
1081                                 1, 0, buf, cmd);
1082         }
1083
1084         iscsit_ack_from_expstatsn(conn, hdr->exp_statsn);
1085
1086         /*
1087          * If no Immediate Data is attached, it's OK to return now.
1088          */
1089         if (!cmd->immediate_data) {
1090                 if (send_check_condition)
1091                         return 0;
1092
1093                 if (cmd->unsolicited_data) {
1094                         iscsit_set_dataout_sequence_values(cmd);
1095
1096                         spin_lock_bh(&cmd->dataout_timeout_lock);
1097                         iscsit_start_dataout_timer(cmd, cmd->conn);
1098                         spin_unlock_bh(&cmd->dataout_timeout_lock);
1099                 }
1100
1101                 return 0;
1102         }
1103
1104         /*
1105          * Early CHECK_CONDITIONs never make it to the transport processing
1106          * thread.  They are processed in CmdSN order by
1107          * iscsit_check_received_cmdsn() below.
1108          */
1109         if (send_check_condition) {
1110                 immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
1111                 dump_immediate_data = 1;
1112                 goto after_immediate_data;
1113         }
1114         /*
1115          * Call directly into transport_generic_new_cmd() to perform
1116          * the backend memory allocation.
1117          */
1118         ret = transport_generic_new_cmd(&cmd->se_cmd);
1119         if (ret < 0) {
1120                 immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
1121                 dump_immediate_data = 1;
1122                 goto after_immediate_data;
1123         }
1124
1125         immed_ret = iscsit_handle_immediate_data(cmd, buf, payload_length);
1126 after_immediate_data:
1127         if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) {
1128                 /*
1129                  * A PDU/CmdSN carrying Immediate Data passed
1130                  * DataCRC, check against ExpCmdSN/MaxCmdSN if
1131                  * Immediate Bit is not set.
1132                  */
1133                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
1134                 /*
1135                  * Special case for Unsupported SAM WRITE Opcodes
1136                  * and ImmediateData=Yes.
1137                  */
1138                 if (dump_immediate_data) {
1139                         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
1140                                 return -1;
1141                 } else if (cmd->unsolicited_data) {
1142                         iscsit_set_dataout_sequence_values(cmd);
1143
1144                         spin_lock_bh(&cmd->dataout_timeout_lock);
1145                         iscsit_start_dataout_timer(cmd, cmd->conn);
1146                         spin_unlock_bh(&cmd->dataout_timeout_lock);
1147                 }
1148
1149                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1150                         return iscsit_add_reject_from_cmd(
1151                                 ISCSI_REASON_PROTOCOL_ERROR,
1152                                 1, 0, buf, cmd);
1153
1154         } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) {
1155                 /*
1156                  * Immediate Data failed DataCRC and ERL>=1,
1157                  * silently drop this PDU and let the initiator
1158                  * plug the CmdSN gap.
1159                  *
1160                  * FIXME: Send Unsolicited NOPIN with reserved
1161                  * TTT here to help the initiator figure out
1162                  * the missing CmdSN, although they should be
1163                  * intelligent enough to determine the missing
1164                  * CmdSN and issue a retry to plug the sequence.
1165                  */
1166                 cmd->i_state = ISTATE_REMOVE;
1167                 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
1168         } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */
1169                 return -1;
1170
1171         return 0;
1172 }
1173
1174 static u32 iscsit_do_crypto_hash_sg(
1175         struct hash_desc *hash,
1176         struct iscsi_cmd *cmd,
1177         u32 data_offset,
1178         u32 data_length,
1179         u32 padding,
1180         u8 *pad_bytes)
1181 {
1182         u32 data_crc;
1183         u32 i;
1184         struct scatterlist *sg;
1185         unsigned int page_off;
1186
1187         crypto_hash_init(hash);
1188
1189         sg = cmd->first_data_sg;
1190         page_off = cmd->first_data_sg_off;
1191
1192         i = 0;
1193         while (data_length) {
1194                 u32 cur_len = min_t(u32, data_length, (sg[i].length - page_off));
1195
1196                 crypto_hash_update(hash, &sg[i], cur_len);
1197
1198                 data_length -= cur_len;
1199                 page_off = 0;
1200                 i++;
1201         }
1202
1203         if (padding) {
1204                 struct scatterlist pad_sg;
1205
1206                 sg_init_one(&pad_sg, pad_bytes, padding);
1207                 crypto_hash_update(hash, &pad_sg, padding);
1208         }
1209         crypto_hash_final(hash, (u8 *) &data_crc);
1210
1211         return data_crc;
1212 }
1213
1214 static void iscsit_do_crypto_hash_buf(
1215         struct hash_desc *hash,
1216         unsigned char *buf,
1217         u32 payload_length,
1218         u32 padding,
1219         u8 *pad_bytes,
1220         u8 *data_crc)
1221 {
1222         struct scatterlist sg;
1223
1224         crypto_hash_init(hash);
1225
1226         sg_init_one(&sg, buf, payload_length);
1227         crypto_hash_update(hash, &sg, payload_length);
1228
1229         if (padding) {
1230                 sg_init_one(&sg, pad_bytes, padding);
1231                 crypto_hash_update(hash, &sg, padding);
1232         }
1233         crypto_hash_final(hash, data_crc);
1234 }
1235
1236 static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf)
1237 {
1238         int iov_ret, ooo_cmdsn = 0, ret;
1239         u8 data_crc_failed = 0;
1240         u32 checksum, iov_count = 0, padding = 0, rx_got = 0;
1241         u32 rx_size = 0, payload_length;
1242         struct iscsi_cmd *cmd = NULL;
1243         struct se_cmd *se_cmd;
1244         struct iscsi_data *hdr;
1245         struct kvec *iov;
1246         unsigned long flags;
1247
1248         hdr                     = (struct iscsi_data *) buf;
1249         payload_length          = ntoh24(hdr->dlength);
1250         hdr->itt                = be32_to_cpu(hdr->itt);
1251         hdr->ttt                = be32_to_cpu(hdr->ttt);
1252         hdr->exp_statsn         = be32_to_cpu(hdr->exp_statsn);
1253         hdr->datasn             = be32_to_cpu(hdr->datasn);
1254         hdr->offset             = be32_to_cpu(hdr->offset);
1255
1256         if (!payload_length) {
1257                 pr_err("DataOUT payload is ZERO, protocol error.\n");
1258                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
1259                                         buf, conn);
1260         }
1261
1262         /* iSCSI write */
1263         spin_lock_bh(&conn->sess->session_stats_lock);
1264         conn->sess->rx_data_octets += payload_length;
1265         if (conn->sess->se_sess->se_node_acl) {
1266                 spin_lock(&conn->sess->se_sess->se_node_acl->stats_lock);
1267                 conn->sess->se_sess->se_node_acl->write_bytes += payload_length;
1268                 spin_unlock(&conn->sess->se_sess->se_node_acl->stats_lock);
1269         }
1270         spin_unlock_bh(&conn->sess->session_stats_lock);
1271
1272         if (payload_length > conn->conn_ops->MaxRecvDataSegmentLength) {
1273                 pr_err("DataSegmentLength: %u is greater than"
1274                         " MaxRecvDataSegmentLength: %u\n", payload_length,
1275                         conn->conn_ops->MaxRecvDataSegmentLength);
1276                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
1277                                         buf, conn);
1278         }
1279
1280         cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt,
1281                         payload_length);
1282         if (!cmd)
1283                 return 0;
1284
1285         pr_debug("Got DataOut ITT: 0x%08x, TTT: 0x%08x,"
1286                 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n",
1287                 hdr->itt, hdr->ttt, hdr->datasn, hdr->offset,
1288                 payload_length, conn->cid);
1289
1290         if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT) {
1291                 pr_err("Command ITT: 0x%08x received DataOUT after"
1292                         " last DataOUT received, dumping payload\n",
1293                         cmd->init_task_tag);
1294                 return iscsit_dump_data_payload(conn, payload_length, 1);
1295         }
1296
1297         if (cmd->data_direction != DMA_TO_DEVICE) {
1298                 pr_err("Command ITT: 0x%08x received DataOUT for a"
1299                         " NON-WRITE command.\n", cmd->init_task_tag);
1300                 return iscsit_add_reject_from_cmd(ISCSI_REASON_PROTOCOL_ERROR,
1301                                 1, 0, buf, cmd);
1302         }
1303         se_cmd = &cmd->se_cmd;
1304         iscsit_mod_dataout_timer(cmd);
1305
1306         if ((hdr->offset + payload_length) > cmd->data_length) {
1307                 pr_err("DataOut Offset: %u, Length %u greater than"
1308                         " iSCSI Command EDTL %u, protocol error.\n",
1309                         hdr->offset, payload_length, cmd->data_length);
1310                 return iscsit_add_reject_from_cmd(ISCSI_REASON_BOOKMARK_INVALID,
1311                                 1, 0, buf, cmd);
1312         }
1313
1314         if (cmd->unsolicited_data) {
1315                 int dump_unsolicited_data = 0;
1316
1317                 if (conn->sess->sess_ops->InitialR2T) {
1318                         pr_err("Received unexpected unsolicited data"
1319                                 " while InitialR2T=Yes, protocol error.\n");
1320                         transport_send_check_condition_and_sense(&cmd->se_cmd,
1321                                         TCM_UNEXPECTED_UNSOLICITED_DATA, 0);
1322                         return -1;
1323                 }
1324                 /*
1325                  * Special case for dealing with Unsolicited DataOUT
1326                  * and Unsupported SAM WRITE Opcodes and SE resource allocation
1327                  * failures;
1328                  */
1329
1330                 /* Something's amiss if we're not in WRITE_PENDING state... */
1331                 spin_lock_irqsave(&se_cmd->t_state_lock, flags);
1332                 WARN_ON(se_cmd->t_state != TRANSPORT_WRITE_PENDING);
1333                 spin_unlock_irqrestore(&se_cmd->t_state_lock, flags);
1334
1335                 spin_lock_irqsave(&se_cmd->t_state_lock, flags);
1336                 if (!(se_cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE) ||
1337                      (se_cmd->se_cmd_flags & SCF_SCSI_CDB_EXCEPTION))
1338                         dump_unsolicited_data = 1;
1339                 spin_unlock_irqrestore(&se_cmd->t_state_lock, flags);
1340
1341                 if (dump_unsolicited_data) {
1342                         /*
1343                          * Check if a delayed TASK_ABORTED status needs to
1344                          * be sent now if the ISCSI_FLAG_CMD_FINAL has been
1345                          * received with the unsolicitied data out.
1346                          */
1347                         if (hdr->flags & ISCSI_FLAG_CMD_FINAL)
1348                                 iscsit_stop_dataout_timer(cmd);
1349
1350                         transport_check_aborted_status(se_cmd,
1351                                         (hdr->flags & ISCSI_FLAG_CMD_FINAL));
1352                         return iscsit_dump_data_payload(conn, payload_length, 1);
1353                 }
1354         } else {
1355                 /*
1356                  * For the normal solicited data path:
1357                  *
1358                  * Check for a delayed TASK_ABORTED status and dump any
1359                  * incoming data out payload if one exists.  Also, when the
1360                  * ISCSI_FLAG_CMD_FINAL is set to denote the end of the current
1361                  * data out sequence, we decrement outstanding_r2ts.  Once
1362                  * outstanding_r2ts reaches zero, go ahead and send the delayed
1363                  * TASK_ABORTED status.
1364                  */
1365                 if (se_cmd->transport_state & CMD_T_ABORTED) {
1366                         if (hdr->flags & ISCSI_FLAG_CMD_FINAL)
1367                                 if (--cmd->outstanding_r2ts < 1) {
1368                                         iscsit_stop_dataout_timer(cmd);
1369                                         transport_check_aborted_status(
1370                                                         se_cmd, 1);
1371                                 }
1372
1373                         return iscsit_dump_data_payload(conn, payload_length, 1);
1374                 }
1375         }
1376         /*
1377          * Preform DataSN, DataSequenceInOrder, DataPDUInOrder, and
1378          * within-command recovery checks before receiving the payload.
1379          */
1380         ret = iscsit_check_pre_dataout(cmd, buf);
1381         if (ret == DATAOUT_WITHIN_COMMAND_RECOVERY)
1382                 return 0;
1383         else if (ret == DATAOUT_CANNOT_RECOVER)
1384                 return -1;
1385
1386         rx_size += payload_length;
1387         iov = &cmd->iov_data[0];
1388
1389         iov_ret = iscsit_map_iovec(cmd, iov, hdr->offset, payload_length);
1390         if (iov_ret < 0)
1391                 return -1;
1392
1393         iov_count += iov_ret;
1394
1395         padding = ((-payload_length) & 3);
1396         if (padding != 0) {
1397                 iov[iov_count].iov_base = cmd->pad_bytes;
1398                 iov[iov_count++].iov_len = padding;
1399                 rx_size += padding;
1400                 pr_debug("Receiving %u padding bytes.\n", padding);
1401         }
1402
1403         if (conn->conn_ops->DataDigest) {
1404                 iov[iov_count].iov_base = &checksum;
1405                 iov[iov_count++].iov_len = ISCSI_CRC_LEN;
1406                 rx_size += ISCSI_CRC_LEN;
1407         }
1408
1409         rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size);
1410
1411         iscsit_unmap_iovec(cmd);
1412
1413         if (rx_got != rx_size)
1414                 return -1;
1415
1416         if (conn->conn_ops->DataDigest) {
1417                 u32 data_crc;
1418
1419                 data_crc = iscsit_do_crypto_hash_sg(&conn->conn_rx_hash, cmd,
1420                                                     hdr->offset, payload_length, padding,
1421                                                     cmd->pad_bytes);
1422
1423                 if (checksum != data_crc) {
1424                         pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
1425                                 " DataSN: 0x%08x, CRC32C DataDigest 0x%08x"
1426                                 " does not match computed 0x%08x\n",
1427                                 hdr->itt, hdr->offset, payload_length,
1428                                 hdr->datasn, checksum, data_crc);
1429                         data_crc_failed = 1;
1430                 } else {
1431                         pr_debug("Got CRC32C DataDigest 0x%08x for"
1432                                 " %u bytes of Data Out\n", checksum,
1433                                 payload_length);
1434                 }
1435         }
1436         /*
1437          * Increment post receive data and CRC values or perform
1438          * within-command recovery.
1439          */
1440         ret = iscsit_check_post_dataout(cmd, buf, data_crc_failed);
1441         if ((ret == DATAOUT_NORMAL) || (ret == DATAOUT_WITHIN_COMMAND_RECOVERY))
1442                 return 0;
1443         else if (ret == DATAOUT_SEND_R2T) {
1444                 iscsit_set_dataout_sequence_values(cmd);
1445                 iscsit_build_r2ts_for_cmd(cmd, conn, 0);
1446         } else if (ret == DATAOUT_SEND_TO_TRANSPORT) {
1447                 /*
1448                  * Handle extra special case for out of order
1449                  * Unsolicited Data Out.
1450                  */
1451                 spin_lock_bh(&cmd->istate_lock);
1452                 ooo_cmdsn = (cmd->cmd_flags & ICF_OOO_CMDSN);
1453                 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
1454                 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
1455                 spin_unlock_bh(&cmd->istate_lock);
1456
1457                 iscsit_stop_dataout_timer(cmd);
1458                 return (!ooo_cmdsn) ? transport_generic_handle_data(
1459                                         &cmd->se_cmd) : 0;
1460         } else /* DATAOUT_CANNOT_RECOVER */
1461                 return -1;
1462
1463         return 0;
1464 }
1465
1466 static int iscsit_handle_nop_out(
1467         struct iscsi_conn *conn,
1468         unsigned char *buf)
1469 {
1470         unsigned char *ping_data = NULL;
1471         int cmdsn_ret, niov = 0, ret = 0, rx_got, rx_size;
1472         u32 checksum, data_crc, padding = 0, payload_length;
1473         struct iscsi_cmd *cmd = NULL;
1474         struct kvec *iov = NULL;
1475         struct iscsi_nopout *hdr;
1476
1477         hdr                     = (struct iscsi_nopout *) buf;
1478         payload_length          = ntoh24(hdr->dlength);
1479         hdr->itt                = be32_to_cpu(hdr->itt);
1480         hdr->ttt                = be32_to_cpu(hdr->ttt);
1481         hdr->cmdsn              = be32_to_cpu(hdr->cmdsn);
1482         hdr->exp_statsn         = be32_to_cpu(hdr->exp_statsn);
1483
1484         if ((hdr->itt == 0xFFFFFFFF) && !(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
1485                 pr_err("NOPOUT ITT is reserved, but Immediate Bit is"
1486                         " not set, protocol error.\n");
1487                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
1488                                         buf, conn);
1489         }
1490
1491         if (payload_length > conn->conn_ops->MaxRecvDataSegmentLength) {
1492                 pr_err("NOPOUT Ping Data DataSegmentLength: %u is"
1493                         " greater than MaxRecvDataSegmentLength: %u, protocol"
1494                         " error.\n", payload_length,
1495                         conn->conn_ops->MaxRecvDataSegmentLength);
1496                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
1497                                         buf, conn);
1498         }
1499
1500         pr_debug("Got NOPOUT Ping %s ITT: 0x%08x, TTT: 0x%09x,"
1501                 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, Length: %u\n",
1502                 (hdr->itt == 0xFFFFFFFF) ? "Response" : "Request",
1503                 hdr->itt, hdr->ttt, hdr->cmdsn, hdr->exp_statsn,
1504                 payload_length);
1505         /*
1506          * This is not a response to a Unsolicited NopIN, which means
1507          * it can either be a NOPOUT ping request (with a valid ITT),
1508          * or a NOPOUT not requesting a NOPIN (with a reserved ITT).
1509          * Either way, make sure we allocate an struct iscsi_cmd, as both
1510          * can contain ping data.
1511          */
1512         if (hdr->ttt == 0xFFFFFFFF) {
1513                 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
1514                 if (!cmd)
1515                         return iscsit_add_reject(
1516                                         ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1517                                         1, buf, conn);
1518
1519                 cmd->iscsi_opcode       = ISCSI_OP_NOOP_OUT;
1520                 cmd->i_state            = ISTATE_SEND_NOPIN;
1521                 cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ?
1522                                                 1 : 0);
1523                 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt;
1524                 cmd->targ_xfer_tag      = 0xFFFFFFFF;
1525                 cmd->cmd_sn             = hdr->cmdsn;
1526                 cmd->exp_stat_sn        = hdr->exp_statsn;
1527                 cmd->data_direction     = DMA_NONE;
1528         }
1529
1530         if (payload_length && (hdr->ttt == 0xFFFFFFFF)) {
1531                 rx_size = payload_length;
1532                 ping_data = kzalloc(payload_length + 1, GFP_KERNEL);
1533                 if (!ping_data) {
1534                         pr_err("Unable to allocate memory for"
1535                                 " NOPOUT ping data.\n");
1536                         ret = -1;
1537                         goto out;
1538                 }
1539
1540                 iov = &cmd->iov_misc[0];
1541                 iov[niov].iov_base      = ping_data;
1542                 iov[niov++].iov_len     = payload_length;
1543
1544                 padding = ((-payload_length) & 3);
1545                 if (padding != 0) {
1546                         pr_debug("Receiving %u additional bytes"
1547                                 " for padding.\n", padding);
1548                         iov[niov].iov_base      = &cmd->pad_bytes;
1549                         iov[niov++].iov_len     = padding;
1550                         rx_size += padding;
1551                 }
1552                 if (conn->conn_ops->DataDigest) {
1553                         iov[niov].iov_base      = &checksum;
1554                         iov[niov++].iov_len     = ISCSI_CRC_LEN;
1555                         rx_size += ISCSI_CRC_LEN;
1556                 }
1557
1558                 rx_got = rx_data(conn, &cmd->iov_misc[0], niov, rx_size);
1559                 if (rx_got != rx_size) {
1560                         ret = -1;
1561                         goto out;
1562                 }
1563
1564                 if (conn->conn_ops->DataDigest) {
1565                         iscsit_do_crypto_hash_buf(&conn->conn_rx_hash,
1566                                         ping_data, payload_length,
1567                                         padding, cmd->pad_bytes,
1568                                         (u8 *)&data_crc);
1569
1570                         if (checksum != data_crc) {
1571                                 pr_err("Ping data CRC32C DataDigest"
1572                                 " 0x%08x does not match computed 0x%08x\n",
1573                                         checksum, data_crc);
1574                                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1575                                         pr_err("Unable to recover from"
1576                                         " NOPOUT Ping DataCRC failure while in"
1577                                                 " ERL=0.\n");
1578                                         ret = -1;
1579                                         goto out;
1580                                 } else {
1581                                         /*
1582                                          * Silently drop this PDU and let the
1583                                          * initiator plug the CmdSN gap.
1584                                          */
1585                                         pr_debug("Dropping NOPOUT"
1586                                         " Command CmdSN: 0x%08x due to"
1587                                         " DataCRC error.\n", hdr->cmdsn);
1588                                         ret = 0;
1589                                         goto out;
1590                                 }
1591                         } else {
1592                                 pr_debug("Got CRC32C DataDigest"
1593                                 " 0x%08x for %u bytes of ping data.\n",
1594                                         checksum, payload_length);
1595                         }
1596                 }
1597
1598                 ping_data[payload_length] = '\0';
1599                 /*
1600                  * Attach ping data to struct iscsi_cmd->buf_ptr.
1601                  */
1602                 cmd->buf_ptr = ping_data;
1603                 cmd->buf_ptr_size = payload_length;
1604
1605                 pr_debug("Got %u bytes of NOPOUT ping"
1606                         " data.\n", payload_length);
1607                 pr_debug("Ping Data: \"%s\"\n", ping_data);
1608         }
1609
1610         if (hdr->itt != 0xFFFFFFFF) {
1611                 if (!cmd) {
1612                         pr_err("Checking CmdSN for NOPOUT,"
1613                                 " but cmd is NULL!\n");
1614                         return -1;
1615                 }
1616                 /*
1617                  * Initiator is expecting a NopIN ping reply,
1618                  */
1619                 spin_lock_bh(&conn->cmd_lock);
1620                 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
1621                 spin_unlock_bh(&conn->cmd_lock);
1622
1623                 iscsit_ack_from_expstatsn(conn, hdr->exp_statsn);
1624
1625                 if (hdr->opcode & ISCSI_OP_IMMEDIATE) {
1626                         iscsit_add_cmd_to_response_queue(cmd, conn,
1627                                         cmd->i_state);
1628                         return 0;
1629                 }
1630
1631                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
1632                 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
1633                         ret = 0;
1634                         goto ping_out;
1635                 }
1636                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1637                         return iscsit_add_reject_from_cmd(
1638                                         ISCSI_REASON_PROTOCOL_ERROR,
1639                                         1, 0, buf, cmd);
1640
1641                 return 0;
1642         }
1643
1644         if (hdr->ttt != 0xFFFFFFFF) {
1645                 /*
1646                  * This was a response to a unsolicited NOPIN ping.
1647                  */
1648                 cmd = iscsit_find_cmd_from_ttt(conn, hdr->ttt);
1649                 if (!cmd)
1650                         return -1;
1651
1652                 iscsit_stop_nopin_response_timer(conn);
1653
1654                 cmd->i_state = ISTATE_REMOVE;
1655                 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
1656                 iscsit_start_nopin_timer(conn);
1657         } else {
1658                 /*
1659                  * Initiator is not expecting a NOPIN is response.
1660                  * Just ignore for now.
1661                  *
1662                  * iSCSI v19-91 10.18
1663                  * "A NOP-OUT may also be used to confirm a changed
1664                  *  ExpStatSN if another PDU will not be available
1665                  *  for a long time."
1666                  */
1667                 ret = 0;
1668                 goto out;
1669         }
1670
1671         return 0;
1672 out:
1673         if (cmd)
1674                 iscsit_release_cmd(cmd);
1675 ping_out:
1676         kfree(ping_data);
1677         return ret;
1678 }
1679
1680 static int iscsit_handle_task_mgt_cmd(
1681         struct iscsi_conn *conn,
1682         unsigned char *buf)
1683 {
1684         struct iscsi_cmd *cmd;
1685         struct se_tmr_req *se_tmr;
1686         struct iscsi_tmr_req *tmr_req;
1687         struct iscsi_tm *hdr;
1688         int out_of_order_cmdsn = 0;
1689         int ret;
1690         u8 function;
1691
1692         hdr                     = (struct iscsi_tm *) buf;
1693         hdr->itt                = be32_to_cpu(hdr->itt);
1694         hdr->rtt                = be32_to_cpu(hdr->rtt);
1695         hdr->cmdsn              = be32_to_cpu(hdr->cmdsn);
1696         hdr->exp_statsn         = be32_to_cpu(hdr->exp_statsn);
1697         hdr->refcmdsn           = be32_to_cpu(hdr->refcmdsn);
1698         hdr->exp_datasn         = be32_to_cpu(hdr->exp_datasn);
1699         hdr->flags &= ~ISCSI_FLAG_CMD_FINAL;
1700         function = hdr->flags;
1701
1702         pr_debug("Got Task Management Request ITT: 0x%08x, CmdSN:"
1703                 " 0x%08x, Function: 0x%02x, RefTaskTag: 0x%08x, RefCmdSN:"
1704                 " 0x%08x, CID: %hu\n", hdr->itt, hdr->cmdsn, function,
1705                 hdr->rtt, hdr->refcmdsn, conn->cid);
1706
1707         if ((function != ISCSI_TM_FUNC_ABORT_TASK) &&
1708             ((function != ISCSI_TM_FUNC_TASK_REASSIGN) &&
1709              (hdr->rtt != ISCSI_RESERVED_TAG))) {
1710                 pr_err("RefTaskTag should be set to 0xFFFFFFFF.\n");
1711                 hdr->rtt = ISCSI_RESERVED_TAG;
1712         }
1713
1714         if ((function == ISCSI_TM_FUNC_TASK_REASSIGN) &&
1715                         !(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
1716                 pr_err("Task Management Request TASK_REASSIGN not"
1717                         " issued as immediate command, bad iSCSI Initiator"
1718                                 "implementation\n");
1719                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
1720                                         buf, conn);
1721         }
1722         if ((function != ISCSI_TM_FUNC_ABORT_TASK) &&
1723             (hdr->refcmdsn != ISCSI_RESERVED_TAG))
1724                 hdr->refcmdsn = ISCSI_RESERVED_TAG;
1725
1726         cmd = iscsit_allocate_se_cmd_for_tmr(conn, function);
1727         if (!cmd)
1728                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1729                                         1, buf, conn);
1730
1731         cmd->iscsi_opcode       = ISCSI_OP_SCSI_TMFUNC;
1732         cmd->i_state            = ISTATE_SEND_TASKMGTRSP;
1733         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
1734         cmd->init_task_tag      = hdr->itt;
1735         cmd->targ_xfer_tag      = 0xFFFFFFFF;
1736         cmd->cmd_sn             = hdr->cmdsn;
1737         cmd->exp_stat_sn        = hdr->exp_statsn;
1738         se_tmr                  = cmd->se_cmd.se_tmr_req;
1739         tmr_req                 = cmd->tmr_req;
1740         /*
1741          * Locate the struct se_lun for all TMRs not related to ERL=2 TASK_REASSIGN
1742          */
1743         if (function != ISCSI_TM_FUNC_TASK_REASSIGN) {
1744                 ret = transport_lookup_tmr_lun(&cmd->se_cmd,
1745                                                scsilun_to_int(&hdr->lun));
1746                 if (ret < 0) {
1747                         cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1748                         se_tmr->response = ISCSI_TMF_RSP_NO_LUN;
1749                         goto attach;
1750                 }
1751         }
1752
1753         switch (function) {
1754         case ISCSI_TM_FUNC_ABORT_TASK:
1755                 se_tmr->response = iscsit_tmr_abort_task(cmd, buf);
1756                 if (se_tmr->response != ISCSI_TMF_RSP_COMPLETE) {
1757                         cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1758                         goto attach;
1759                 }
1760                 break;
1761         case ISCSI_TM_FUNC_ABORT_TASK_SET:
1762         case ISCSI_TM_FUNC_CLEAR_ACA:
1763         case ISCSI_TM_FUNC_CLEAR_TASK_SET:
1764         case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET:
1765                 break;
1766         case ISCSI_TM_FUNC_TARGET_WARM_RESET:
1767                 if (iscsit_tmr_task_warm_reset(conn, tmr_req, buf) < 0) {
1768                         cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1769                         se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED;
1770                         goto attach;
1771                 }
1772                 break;
1773         case ISCSI_TM_FUNC_TARGET_COLD_RESET:
1774                 if (iscsit_tmr_task_cold_reset(conn, tmr_req, buf) < 0) {
1775                         cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1776                         se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED;
1777                         goto attach;
1778                 }
1779                 break;
1780         case ISCSI_TM_FUNC_TASK_REASSIGN:
1781                 se_tmr->response = iscsit_tmr_task_reassign(cmd, buf);
1782                 /*
1783                  * Perform sanity checks on the ExpDataSN only if the
1784                  * TASK_REASSIGN was successful.
1785                  */
1786                 if (se_tmr->response != ISCSI_TMF_RSP_COMPLETE)
1787                         break;
1788
1789                 if (iscsit_check_task_reassign_expdatasn(tmr_req, conn) < 0)
1790                         return iscsit_add_reject_from_cmd(
1791                                         ISCSI_REASON_BOOKMARK_INVALID, 1, 1,
1792                                         buf, cmd);
1793                 break;
1794         default:
1795                 pr_err("Unknown TMR function: 0x%02x, protocol"
1796                         " error.\n", function);
1797                 cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1798                 se_tmr->response = ISCSI_TMF_RSP_NOT_SUPPORTED;
1799                 goto attach;
1800         }
1801
1802         if ((function != ISCSI_TM_FUNC_TASK_REASSIGN) &&
1803             (se_tmr->response == ISCSI_TMF_RSP_COMPLETE))
1804                 se_tmr->call_transport = 1;
1805 attach:
1806         spin_lock_bh(&conn->cmd_lock);
1807         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
1808         spin_unlock_bh(&conn->cmd_lock);
1809
1810         if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
1811                 int cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
1812                 if (cmdsn_ret == CMDSN_HIGHER_THAN_EXP)
1813                         out_of_order_cmdsn = 1;
1814                 else if (cmdsn_ret == CMDSN_LOWER_THAN_EXP)
1815                         return 0;
1816                 else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1817                         return iscsit_add_reject_from_cmd(
1818                                         ISCSI_REASON_PROTOCOL_ERROR,
1819                                         1, 0, buf, cmd);
1820         }
1821         iscsit_ack_from_expstatsn(conn, hdr->exp_statsn);
1822
1823         if (out_of_order_cmdsn || !(hdr->opcode & ISCSI_OP_IMMEDIATE))
1824                 return 0;
1825         /*
1826          * Found the referenced task, send to transport for processing.
1827          */
1828         if (se_tmr->call_transport)
1829                 return transport_generic_handle_tmr(&cmd->se_cmd);
1830
1831         /*
1832          * Could not find the referenced LUN, task, or Task Management
1833          * command not authorized or supported.  Change state and
1834          * let the tx_thread send the response.
1835          *
1836          * For connection recovery, this is also the default action for
1837          * TMR TASK_REASSIGN.
1838          */
1839         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
1840         return 0;
1841 }
1842
1843 /* #warning FIXME: Support Text Command parameters besides SendTargets */
1844 static int iscsit_handle_text_cmd(
1845         struct iscsi_conn *conn,
1846         unsigned char *buf)
1847 {
1848         char *text_ptr, *text_in;
1849         int cmdsn_ret, niov = 0, rx_got, rx_size;
1850         u32 checksum = 0, data_crc = 0, payload_length;
1851         u32 padding = 0, pad_bytes = 0, text_length = 0;
1852         struct iscsi_cmd *cmd;
1853         struct kvec iov[3];
1854         struct iscsi_text *hdr;
1855
1856         hdr                     = (struct iscsi_text *) buf;
1857         payload_length          = ntoh24(hdr->dlength);
1858         hdr->itt                = be32_to_cpu(hdr->itt);
1859         hdr->ttt                = be32_to_cpu(hdr->ttt);
1860         hdr->cmdsn              = be32_to_cpu(hdr->cmdsn);
1861         hdr->exp_statsn         = be32_to_cpu(hdr->exp_statsn);
1862
1863         if (payload_length > conn->conn_ops->MaxRecvDataSegmentLength) {
1864                 pr_err("Unable to accept text parameter length: %u"
1865                         "greater than MaxRecvDataSegmentLength %u.\n",
1866                        payload_length, conn->conn_ops->MaxRecvDataSegmentLength);
1867                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
1868                                         buf, conn);
1869         }
1870
1871         pr_debug("Got Text Request: ITT: 0x%08x, CmdSN: 0x%08x,"
1872                 " ExpStatSN: 0x%08x, Length: %u\n", hdr->itt, hdr->cmdsn,
1873                 hdr->exp_statsn, payload_length);
1874
1875         rx_size = text_length = payload_length;
1876         if (text_length) {
1877                 text_in = kzalloc(text_length, GFP_KERNEL);
1878                 if (!text_in) {
1879                         pr_err("Unable to allocate memory for"
1880                                 " incoming text parameters\n");
1881                         return -1;
1882                 }
1883
1884                 memset(iov, 0, 3 * sizeof(struct kvec));
1885                 iov[niov].iov_base      = text_in;
1886                 iov[niov++].iov_len     = text_length;
1887
1888                 padding = ((-payload_length) & 3);
1889                 if (padding != 0) {
1890                         iov[niov].iov_base = &pad_bytes;
1891                         iov[niov++].iov_len  = padding;
1892                         rx_size += padding;
1893                         pr_debug("Receiving %u additional bytes"
1894                                         " for padding.\n", padding);
1895                 }
1896                 if (conn->conn_ops->DataDigest) {
1897                         iov[niov].iov_base      = &checksum;
1898                         iov[niov++].iov_len     = ISCSI_CRC_LEN;
1899                         rx_size += ISCSI_CRC_LEN;
1900                 }
1901
1902                 rx_got = rx_data(conn, &iov[0], niov, rx_size);
1903                 if (rx_got != rx_size) {
1904                         kfree(text_in);
1905                         return -1;
1906                 }
1907
1908                 if (conn->conn_ops->DataDigest) {
1909                         iscsit_do_crypto_hash_buf(&conn->conn_rx_hash,
1910                                         text_in, text_length,
1911                                         padding, (u8 *)&pad_bytes,
1912                                         (u8 *)&data_crc);
1913
1914                         if (checksum != data_crc) {
1915                                 pr_err("Text data CRC32C DataDigest"
1916                                         " 0x%08x does not match computed"
1917                                         " 0x%08x\n", checksum, data_crc);
1918                                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1919                                         pr_err("Unable to recover from"
1920                                         " Text Data digest failure while in"
1921                                                 " ERL=0.\n");
1922                                         kfree(text_in);
1923                                         return -1;
1924                                 } else {
1925                                         /*
1926                                          * Silently drop this PDU and let the
1927                                          * initiator plug the CmdSN gap.
1928                                          */
1929                                         pr_debug("Dropping Text"
1930                                         " Command CmdSN: 0x%08x due to"
1931                                         " DataCRC error.\n", hdr->cmdsn);
1932                                         kfree(text_in);
1933                                         return 0;
1934                                 }
1935                         } else {
1936                                 pr_debug("Got CRC32C DataDigest"
1937                                         " 0x%08x for %u bytes of text data.\n",
1938                                                 checksum, text_length);
1939                         }
1940                 }
1941                 text_in[text_length - 1] = '\0';
1942                 pr_debug("Successfully read %d bytes of text"
1943                                 " data.\n", text_length);
1944
1945                 if (strncmp("SendTargets", text_in, 11) != 0) {
1946                         pr_err("Received Text Data that is not"
1947                                 " SendTargets, cannot continue.\n");
1948                         kfree(text_in);
1949                         return -1;
1950                 }
1951                 text_ptr = strchr(text_in, '=');
1952                 if (!text_ptr) {
1953                         pr_err("No \"=\" separator found in Text Data,"
1954                                 "  cannot continue.\n");
1955                         kfree(text_in);
1956                         return -1;
1957                 }
1958                 if (strncmp("=All", text_ptr, 4) != 0) {
1959                         pr_err("Unable to locate All value for"
1960                                 " SendTargets key,  cannot continue.\n");
1961                         kfree(text_in);
1962                         return -1;
1963                 }
1964 /*#warning Support SendTargets=(iSCSI Target Name/Nothing) values. */
1965                 kfree(text_in);
1966         }
1967
1968         cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
1969         if (!cmd)
1970                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1971                                         1, buf, conn);
1972
1973         cmd->iscsi_opcode       = ISCSI_OP_TEXT;
1974         cmd->i_state            = ISTATE_SEND_TEXTRSP;
1975         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
1976         conn->sess->init_task_tag = cmd->init_task_tag  = hdr->itt;
1977         cmd->targ_xfer_tag      = 0xFFFFFFFF;
1978         cmd->cmd_sn             = hdr->cmdsn;
1979         cmd->exp_stat_sn        = hdr->exp_statsn;
1980         cmd->data_direction     = DMA_NONE;
1981
1982         spin_lock_bh(&conn->cmd_lock);
1983         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
1984         spin_unlock_bh(&conn->cmd_lock);
1985
1986         iscsit_ack_from_expstatsn(conn, hdr->exp_statsn);
1987
1988         if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
1989                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
1990                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1991                         return iscsit_add_reject_from_cmd(
1992                                         ISCSI_REASON_PROTOCOL_ERROR,
1993                                         1, 0, buf, cmd);
1994
1995                 return 0;
1996         }
1997
1998         return iscsit_execute_cmd(cmd, 0);
1999 }
2000
2001 int iscsit_logout_closesession(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2002 {
2003         struct iscsi_conn *conn_p;
2004         struct iscsi_session *sess = conn->sess;
2005
2006         pr_debug("Received logout request CLOSESESSION on CID: %hu"
2007                 " for SID: %u.\n", conn->cid, conn->sess->sid);
2008
2009         atomic_set(&sess->session_logout, 1);
2010         atomic_set(&conn->conn_logout_remove, 1);
2011         conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_SESSION;
2012
2013         iscsit_inc_conn_usage_count(conn);
2014         iscsit_inc_session_usage_count(sess);
2015
2016         spin_lock_bh(&sess->conn_lock);
2017         list_for_each_entry(conn_p, &sess->sess_conn_list, conn_list) {
2018                 if (conn_p->conn_state != TARG_CONN_STATE_LOGGED_IN)
2019                         continue;
2020
2021                 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
2022                 conn_p->conn_state = TARG_CONN_STATE_IN_LOGOUT;
2023         }
2024         spin_unlock_bh(&sess->conn_lock);
2025
2026         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2027
2028         return 0;
2029 }
2030
2031 int iscsit_logout_closeconnection(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2032 {
2033         struct iscsi_conn *l_conn;
2034         struct iscsi_session *sess = conn->sess;
2035
2036         pr_debug("Received logout request CLOSECONNECTION for CID:"
2037                 " %hu on CID: %hu.\n", cmd->logout_cid, conn->cid);
2038
2039         /*
2040          * A Logout Request with a CLOSECONNECTION reason code for a CID
2041          * can arrive on a connection with a differing CID.
2042          */
2043         if (conn->cid == cmd->logout_cid) {
2044                 spin_lock_bh(&conn->state_lock);
2045                 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
2046                 conn->conn_state = TARG_CONN_STATE_IN_LOGOUT;
2047
2048                 atomic_set(&conn->conn_logout_remove, 1);
2049                 conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_CONNECTION;
2050                 iscsit_inc_conn_usage_count(conn);
2051
2052                 spin_unlock_bh(&conn->state_lock);
2053         } else {
2054                 /*
2055                  * Handle all different cid CLOSECONNECTION requests in
2056                  * iscsit_logout_post_handler_diffcid() as to give enough
2057                  * time for any non immediate command's CmdSN to be
2058                  * acknowledged on the connection in question.
2059                  *
2060                  * Here we simply make sure the CID is still around.
2061                  */
2062                 l_conn = iscsit_get_conn_from_cid(sess,
2063                                 cmd->logout_cid);
2064                 if (!l_conn) {
2065                         cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND;
2066                         iscsit_add_cmd_to_response_queue(cmd, conn,
2067                                         cmd->i_state);
2068                         return 0;
2069                 }
2070
2071                 iscsit_dec_conn_usage_count(l_conn);
2072         }
2073
2074         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2075
2076         return 0;
2077 }
2078
2079 int iscsit_logout_removeconnforrecovery(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2080 {
2081         struct iscsi_session *sess = conn->sess;
2082
2083         pr_debug("Received explicit REMOVECONNFORRECOVERY logout for"
2084                 " CID: %hu on CID: %hu.\n", cmd->logout_cid, conn->cid);
2085
2086         if (sess->sess_ops->ErrorRecoveryLevel != 2) {
2087                 pr_err("Received Logout Request REMOVECONNFORRECOVERY"
2088                         " while ERL!=2.\n");
2089                 cmd->logout_response = ISCSI_LOGOUT_RECOVERY_UNSUPPORTED;
2090                 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2091                 return 0;
2092         }
2093
2094         if (conn->cid == cmd->logout_cid) {
2095                 pr_err("Received Logout Request REMOVECONNFORRECOVERY"
2096                         " with CID: %hu on CID: %hu, implementation error.\n",
2097                                 cmd->logout_cid, conn->cid);
2098                 cmd->logout_response = ISCSI_LOGOUT_CLEANUP_FAILED;
2099                 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2100                 return 0;
2101         }
2102
2103         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2104
2105         return 0;
2106 }
2107
2108 static int iscsit_handle_logout_cmd(
2109         struct iscsi_conn *conn,
2110         unsigned char *buf)
2111 {
2112         int cmdsn_ret, logout_remove = 0;
2113         u8 reason_code = 0;
2114         struct iscsi_cmd *cmd;
2115         struct iscsi_logout *hdr;
2116         struct iscsi_tiqn *tiqn = iscsit_snmp_get_tiqn(conn);
2117
2118         hdr                     = (struct iscsi_logout *) buf;
2119         reason_code             = (hdr->flags & 0x7f);
2120         hdr->itt                = be32_to_cpu(hdr->itt);
2121         hdr->cid                = be16_to_cpu(hdr->cid);
2122         hdr->cmdsn              = be32_to_cpu(hdr->cmdsn);
2123         hdr->exp_statsn = be32_to_cpu(hdr->exp_statsn);
2124
2125         if (tiqn) {
2126                 spin_lock(&tiqn->logout_stats.lock);
2127                 if (reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION)
2128                         tiqn->logout_stats.normal_logouts++;
2129                 else
2130                         tiqn->logout_stats.abnormal_logouts++;
2131                 spin_unlock(&tiqn->logout_stats.lock);
2132         }
2133
2134         pr_debug("Got Logout Request ITT: 0x%08x CmdSN: 0x%08x"
2135                 " ExpStatSN: 0x%08x Reason: 0x%02x CID: %hu on CID: %hu\n",
2136                 hdr->itt, hdr->cmdsn, hdr->exp_statsn, reason_code,
2137                 hdr->cid, conn->cid);
2138
2139         if (conn->conn_state != TARG_CONN_STATE_LOGGED_IN) {
2140                 pr_err("Received logout request on connection that"
2141                         " is not in logged in state, ignoring request.\n");
2142                 return 0;
2143         }
2144
2145         cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
2146         if (!cmd)
2147                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1,
2148                                         buf, conn);
2149
2150         cmd->iscsi_opcode       = ISCSI_OP_LOGOUT;
2151         cmd->i_state            = ISTATE_SEND_LOGOUTRSP;
2152         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
2153         conn->sess->init_task_tag = cmd->init_task_tag  = hdr->itt;
2154         cmd->targ_xfer_tag      = 0xFFFFFFFF;
2155         cmd->cmd_sn             = hdr->cmdsn;
2156         cmd->exp_stat_sn        = hdr->exp_statsn;
2157         cmd->logout_cid         = hdr->cid;
2158         cmd->logout_reason      = reason_code;
2159         cmd->data_direction     = DMA_NONE;
2160
2161         /*
2162          * We need to sleep in these cases (by returning 1) until the Logout
2163          * Response gets sent in the tx thread.
2164          */
2165         if ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION) ||
2166            ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION) &&
2167             (hdr->cid == conn->cid)))
2168                 logout_remove = 1;
2169
2170         spin_lock_bh(&conn->cmd_lock);
2171         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
2172         spin_unlock_bh(&conn->cmd_lock);
2173
2174         if (reason_code != ISCSI_LOGOUT_REASON_RECOVERY)
2175                 iscsit_ack_from_expstatsn(conn, hdr->exp_statsn);
2176
2177         /*
2178          * Immediate commands are executed, well, immediately.
2179          * Non-Immediate Logout Commands are executed in CmdSN order.
2180          */
2181         if (cmd->immediate_cmd) {
2182                 int ret = iscsit_execute_cmd(cmd, 0);
2183
2184                 if (ret < 0)
2185                         return ret;
2186         } else {
2187                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
2188                 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
2189                         logout_remove = 0;
2190                 } else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) {
2191                         return iscsit_add_reject_from_cmd(
2192                                 ISCSI_REASON_PROTOCOL_ERROR,
2193                                 1, 0, buf, cmd);
2194                 }
2195         }
2196
2197         return logout_remove;
2198 }
2199
2200 static int iscsit_handle_snack(
2201         struct iscsi_conn *conn,
2202         unsigned char *buf)
2203 {
2204         struct iscsi_snack *hdr;
2205
2206         hdr                     = (struct iscsi_snack *) buf;
2207         hdr->flags              &= ~ISCSI_FLAG_CMD_FINAL;
2208         hdr->itt                = be32_to_cpu(hdr->itt);
2209         hdr->ttt                = be32_to_cpu(hdr->ttt);
2210         hdr->exp_statsn         = be32_to_cpu(hdr->exp_statsn);
2211         hdr->begrun             = be32_to_cpu(hdr->begrun);
2212         hdr->runlength          = be32_to_cpu(hdr->runlength);
2213
2214         pr_debug("Got ISCSI_INIT_SNACK, ITT: 0x%08x, ExpStatSN:"
2215                 " 0x%08x, Type: 0x%02x, BegRun: 0x%08x, RunLength: 0x%08x,"
2216                 " CID: %hu\n", hdr->itt, hdr->exp_statsn, hdr->flags,
2217                         hdr->begrun, hdr->runlength, conn->cid);
2218
2219         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
2220                 pr_err("Initiator sent SNACK request while in"
2221                         " ErrorRecoveryLevel=0.\n");
2222                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
2223                                         buf, conn);
2224         }
2225         /*
2226          * SNACK_DATA and SNACK_R2T are both 0,  so check which function to
2227          * call from inside iscsi_send_recovery_datain_or_r2t().
2228          */
2229         switch (hdr->flags & ISCSI_FLAG_SNACK_TYPE_MASK) {
2230         case 0:
2231                 return iscsit_handle_recovery_datain_or_r2t(conn, buf,
2232                         hdr->itt, hdr->ttt, hdr->begrun, hdr->runlength);
2233         case ISCSI_FLAG_SNACK_TYPE_STATUS:
2234                 return iscsit_handle_status_snack(conn, hdr->itt, hdr->ttt,
2235                         hdr->begrun, hdr->runlength);
2236         case ISCSI_FLAG_SNACK_TYPE_DATA_ACK:
2237                 return iscsit_handle_data_ack(conn, hdr->ttt, hdr->begrun,
2238                         hdr->runlength);
2239         case ISCSI_FLAG_SNACK_TYPE_RDATA:
2240                 /* FIXME: Support R-Data SNACK */
2241                 pr_err("R-Data SNACK Not Supported.\n");
2242                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
2243                                         buf, conn);
2244         default:
2245                 pr_err("Unknown SNACK type 0x%02x, protocol"
2246                         " error.\n", hdr->flags & 0x0f);
2247                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
2248                                         buf, conn);
2249         }
2250
2251         return 0;
2252 }
2253
2254 static void iscsit_rx_thread_wait_for_tcp(struct iscsi_conn *conn)
2255 {
2256         if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) ||
2257             (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) {
2258                 wait_for_completion_interruptible_timeout(
2259                                         &conn->rx_half_close_comp,
2260                                         ISCSI_RX_THREAD_TCP_TIMEOUT * HZ);
2261         }
2262 }
2263
2264 static int iscsit_handle_immediate_data(
2265         struct iscsi_cmd *cmd,
2266         unsigned char *buf,
2267         u32 length)
2268 {
2269         int iov_ret, rx_got = 0, rx_size = 0;
2270         u32 checksum, iov_count = 0, padding = 0;
2271         struct iscsi_conn *conn = cmd->conn;
2272         struct kvec *iov;
2273
2274         iov_ret = iscsit_map_iovec(cmd, cmd->iov_data, cmd->write_data_done, length);
2275         if (iov_ret < 0)
2276                 return IMMEDIATE_DATA_CANNOT_RECOVER;
2277
2278         rx_size = length;
2279         iov_count = iov_ret;
2280         iov = &cmd->iov_data[0];
2281
2282         padding = ((-length) & 3);
2283         if (padding != 0) {
2284                 iov[iov_count].iov_base = cmd->pad_bytes;
2285                 iov[iov_count++].iov_len = padding;
2286                 rx_size += padding;
2287         }
2288
2289         if (conn->conn_ops->DataDigest) {
2290                 iov[iov_count].iov_base         = &checksum;
2291                 iov[iov_count++].iov_len        = ISCSI_CRC_LEN;
2292                 rx_size += ISCSI_CRC_LEN;
2293         }
2294
2295         rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size);
2296
2297         iscsit_unmap_iovec(cmd);
2298
2299         if (rx_got != rx_size) {
2300                 iscsit_rx_thread_wait_for_tcp(conn);
2301                 return IMMEDIATE_DATA_CANNOT_RECOVER;
2302         }
2303
2304         if (conn->conn_ops->DataDigest) {
2305                 u32 data_crc;
2306
2307                 data_crc = iscsit_do_crypto_hash_sg(&conn->conn_rx_hash, cmd,
2308                                                     cmd->write_data_done, length, padding,
2309                                                     cmd->pad_bytes);
2310
2311                 if (checksum != data_crc) {
2312                         pr_err("ImmediateData CRC32C DataDigest 0x%08x"
2313                                 " does not match computed 0x%08x\n", checksum,
2314                                 data_crc);
2315
2316                         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
2317                                 pr_err("Unable to recover from"
2318                                         " Immediate Data digest failure while"
2319                                         " in ERL=0.\n");
2320                                 iscsit_add_reject_from_cmd(
2321                                                 ISCSI_REASON_DATA_DIGEST_ERROR,
2322                                                 1, 0, buf, cmd);
2323                                 return IMMEDIATE_DATA_CANNOT_RECOVER;
2324                         } else {
2325                                 iscsit_add_reject_from_cmd(
2326                                                 ISCSI_REASON_DATA_DIGEST_ERROR,
2327                                                 0, 0, buf, cmd);
2328                                 return IMMEDIATE_DATA_ERL1_CRC_FAILURE;
2329                         }
2330                 } else {
2331                         pr_debug("Got CRC32C DataDigest 0x%08x for"
2332                                 " %u bytes of Immediate Data\n", checksum,
2333                                 length);
2334                 }
2335         }
2336
2337         cmd->write_data_done += length;
2338
2339         if (cmd->write_data_done == cmd->data_length) {
2340                 spin_lock_bh(&cmd->istate_lock);
2341                 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
2342                 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
2343                 spin_unlock_bh(&cmd->istate_lock);
2344         }
2345
2346         return IMMEDIATE_DATA_NORMAL_OPERATION;
2347 }
2348
2349 /*
2350  *      Called with sess->conn_lock held.
2351  */
2352 /* #warning iscsi_build_conn_drop_async_message() only sends out on connections
2353         with active network interface */
2354 static void iscsit_build_conn_drop_async_message(struct iscsi_conn *conn)
2355 {
2356         struct iscsi_cmd *cmd;
2357         struct iscsi_conn *conn_p;
2358
2359         /*
2360          * Only send a Asynchronous Message on connections whos network
2361          * interface is still functional.
2362          */
2363         list_for_each_entry(conn_p, &conn->sess->sess_conn_list, conn_list) {
2364                 if (conn_p->conn_state == TARG_CONN_STATE_LOGGED_IN) {
2365                         iscsit_inc_conn_usage_count(conn_p);
2366                         break;
2367                 }
2368         }
2369
2370         if (!conn_p)
2371                 return;
2372
2373         cmd = iscsit_allocate_cmd(conn_p, GFP_KERNEL);
2374         if (!cmd) {
2375                 iscsit_dec_conn_usage_count(conn_p);
2376                 return;
2377         }
2378
2379         cmd->logout_cid = conn->cid;
2380         cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT;
2381         cmd->i_state = ISTATE_SEND_ASYNCMSG;
2382
2383         spin_lock_bh(&conn_p->cmd_lock);
2384         list_add_tail(&cmd->i_conn_node, &conn_p->conn_cmd_list);
2385         spin_unlock_bh(&conn_p->cmd_lock);
2386
2387         iscsit_add_cmd_to_response_queue(cmd, conn_p, cmd->i_state);
2388         iscsit_dec_conn_usage_count(conn_p);
2389 }
2390
2391 static int iscsit_send_conn_drop_async_message(
2392         struct iscsi_cmd *cmd,
2393         struct iscsi_conn *conn)
2394 {
2395         struct iscsi_async *hdr;
2396
2397         cmd->tx_size = ISCSI_HDR_LEN;
2398         cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT;
2399
2400         hdr                     = (struct iscsi_async *) cmd->pdu;
2401         hdr->opcode             = ISCSI_OP_ASYNC_EVENT;
2402         hdr->flags              = ISCSI_FLAG_CMD_FINAL;
2403         cmd->init_task_tag      = 0xFFFFFFFF;
2404         cmd->targ_xfer_tag      = 0xFFFFFFFF;
2405         put_unaligned_be64(0xFFFFFFFFFFFFFFFFULL, &hdr->rsvd4[0]);
2406         cmd->stat_sn            = conn->stat_sn++;
2407         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2408         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2409         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
2410         hdr->async_event        = ISCSI_ASYNC_MSG_DROPPING_CONNECTION;
2411         hdr->param1             = cpu_to_be16(cmd->logout_cid);
2412         hdr->param2             = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Wait);
2413         hdr->param3             = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Retain);
2414
2415         if (conn->conn_ops->HeaderDigest) {
2416                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2417
2418                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2419                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
2420                                 0, NULL, (u8 *)header_digest);
2421
2422                 cmd->tx_size += ISCSI_CRC_LEN;
2423                 pr_debug("Attaching CRC32C HeaderDigest to"
2424                         " Async Message 0x%08x\n", *header_digest);
2425         }
2426
2427         cmd->iov_misc[0].iov_base       = cmd->pdu;
2428         cmd->iov_misc[0].iov_len        = cmd->tx_size;
2429         cmd->iov_misc_count             = 1;
2430
2431         pr_debug("Sending Connection Dropped Async Message StatSN:"
2432                 " 0x%08x, for CID: %hu on CID: %hu\n", cmd->stat_sn,
2433                         cmd->logout_cid, conn->cid);
2434         return 0;
2435 }
2436
2437 static int iscsit_send_data_in(
2438         struct iscsi_cmd *cmd,
2439         struct iscsi_conn *conn,
2440         int *eodr)
2441 {
2442         int iov_ret = 0, set_statsn = 0;
2443         u32 iov_count = 0, tx_size = 0;
2444         struct iscsi_datain datain;
2445         struct iscsi_datain_req *dr;
2446         struct iscsi_data_rsp *hdr;
2447         struct kvec *iov;
2448
2449         memset(&datain, 0, sizeof(struct iscsi_datain));
2450         dr = iscsit_get_datain_values(cmd, &datain);
2451         if (!dr) {
2452                 pr_err("iscsit_get_datain_values failed for ITT: 0x%08x\n",
2453                                 cmd->init_task_tag);
2454                 return -1;
2455         }
2456
2457         /*
2458          * Be paranoid and double check the logic for now.
2459          */
2460         if ((datain.offset + datain.length) > cmd->data_length) {
2461                 pr_err("Command ITT: 0x%08x, datain.offset: %u and"
2462                         " datain.length: %u exceeds cmd->data_length: %u\n",
2463                         cmd->init_task_tag, datain.offset, datain.length,
2464                                 cmd->data_length);
2465                 return -1;
2466         }
2467
2468         spin_lock_bh(&conn->sess->session_stats_lock);
2469         conn->sess->tx_data_octets += datain.length;
2470         if (conn->sess->se_sess->se_node_acl) {
2471                 spin_lock(&conn->sess->se_sess->se_node_acl->stats_lock);
2472                 conn->sess->se_sess->se_node_acl->read_bytes += datain.length;
2473                 spin_unlock(&conn->sess->se_sess->se_node_acl->stats_lock);
2474         }
2475         spin_unlock_bh(&conn->sess->session_stats_lock);
2476         /*
2477          * Special case for successfully execution w/ both DATAIN
2478          * and Sense Data.
2479          */
2480         if ((datain.flags & ISCSI_FLAG_DATA_STATUS) &&
2481             (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE))
2482                 datain.flags &= ~ISCSI_FLAG_DATA_STATUS;
2483         else {
2484                 if ((dr->dr_complete == DATAIN_COMPLETE_NORMAL) ||
2485                     (dr->dr_complete == DATAIN_COMPLETE_CONNECTION_RECOVERY)) {
2486                         iscsit_increment_maxcmdsn(cmd, conn->sess);
2487                         cmd->stat_sn = conn->stat_sn++;
2488                         set_statsn = 1;
2489                 } else if (dr->dr_complete ==
2490                                 DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY)
2491                         set_statsn = 1;
2492         }
2493
2494         hdr     = (struct iscsi_data_rsp *) cmd->pdu;
2495         memset(hdr, 0, ISCSI_HDR_LEN);
2496         hdr->opcode             = ISCSI_OP_SCSI_DATA_IN;
2497         hdr->flags              = datain.flags;
2498         if (hdr->flags & ISCSI_FLAG_DATA_STATUS) {
2499                 if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) {
2500                         hdr->flags |= ISCSI_FLAG_DATA_OVERFLOW;
2501                         hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
2502                 } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) {
2503                         hdr->flags |= ISCSI_FLAG_DATA_UNDERFLOW;
2504                         hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
2505                 }
2506         }
2507         hton24(hdr->dlength, datain.length);
2508         if (hdr->flags & ISCSI_FLAG_DATA_ACK)
2509                 int_to_scsilun(cmd->se_cmd.orig_fe_lun,
2510                                 (struct scsi_lun *)&hdr->lun);
2511         else
2512                 put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun);
2513
2514         hdr->itt                = cpu_to_be32(cmd->init_task_tag);
2515         hdr->ttt                = (hdr->flags & ISCSI_FLAG_DATA_ACK) ?
2516                                    cpu_to_be32(cmd->targ_xfer_tag) :
2517                                    0xFFFFFFFF;
2518         hdr->statsn             = (set_statsn) ? cpu_to_be32(cmd->stat_sn) :
2519                                                 0xFFFFFFFF;
2520         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2521         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
2522         hdr->datasn             = cpu_to_be32(datain.data_sn);
2523         hdr->offset             = cpu_to_be32(datain.offset);
2524
2525         iov = &cmd->iov_data[0];
2526         iov[iov_count].iov_base = cmd->pdu;
2527         iov[iov_count++].iov_len        = ISCSI_HDR_LEN;
2528         tx_size += ISCSI_HDR_LEN;
2529
2530         if (conn->conn_ops->HeaderDigest) {
2531                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2532
2533                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2534                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
2535                                 0, NULL, (u8 *)header_digest);
2536
2537                 iov[0].iov_len += ISCSI_CRC_LEN;
2538                 tx_size += ISCSI_CRC_LEN;
2539
2540                 pr_debug("Attaching CRC32 HeaderDigest"
2541                         " for DataIN PDU 0x%08x\n", *header_digest);
2542         }
2543
2544         iov_ret = iscsit_map_iovec(cmd, &cmd->iov_data[1], datain.offset, datain.length);
2545         if (iov_ret < 0)
2546                 return -1;
2547
2548         iov_count += iov_ret;
2549         tx_size += datain.length;
2550
2551         cmd->padding = ((-datain.length) & 3);
2552         if (cmd->padding) {
2553                 iov[iov_count].iov_base         = cmd->pad_bytes;
2554                 iov[iov_count++].iov_len        = cmd->padding;
2555                 tx_size += cmd->padding;
2556
2557                 pr_debug("Attaching %u padding bytes\n",
2558                                 cmd->padding);
2559         }
2560         if (conn->conn_ops->DataDigest) {
2561                 cmd->data_crc = iscsit_do_crypto_hash_sg(&conn->conn_tx_hash, cmd,
2562                          datain.offset, datain.length, cmd->padding, cmd->pad_bytes);
2563
2564                 iov[iov_count].iov_base = &cmd->data_crc;
2565                 iov[iov_count++].iov_len = ISCSI_CRC_LEN;
2566                 tx_size += ISCSI_CRC_LEN;
2567
2568                 pr_debug("Attached CRC32C DataDigest %d bytes, crc"
2569                         " 0x%08x\n", datain.length+cmd->padding, cmd->data_crc);
2570         }
2571
2572         cmd->iov_data_count = iov_count;
2573         cmd->tx_size = tx_size;
2574
2575         pr_debug("Built DataIN ITT: 0x%08x, StatSN: 0x%08x,"
2576                 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n",
2577                 cmd->init_task_tag, ntohl(hdr->statsn), ntohl(hdr->datasn),
2578                 ntohl(hdr->offset), datain.length, conn->cid);
2579
2580         if (dr->dr_complete) {
2581                 *eodr = (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ?
2582                                 2 : 1;
2583                 iscsit_free_datain_req(cmd, dr);
2584         }
2585
2586         return 0;
2587 }
2588
2589 static int iscsit_send_logout_response(
2590         struct iscsi_cmd *cmd,
2591         struct iscsi_conn *conn)
2592 {
2593         int niov = 0, tx_size;
2594         struct iscsi_conn *logout_conn = NULL;
2595         struct iscsi_conn_recovery *cr = NULL;
2596         struct iscsi_session *sess = conn->sess;
2597         struct kvec *iov;
2598         struct iscsi_logout_rsp *hdr;
2599         /*
2600          * The actual shutting down of Sessions and/or Connections
2601          * for CLOSESESSION and CLOSECONNECTION Logout Requests
2602          * is done in scsi_logout_post_handler().
2603          */
2604         switch (cmd->logout_reason) {
2605         case ISCSI_LOGOUT_REASON_CLOSE_SESSION:
2606                 pr_debug("iSCSI session logout successful, setting"
2607                         " logout response to ISCSI_LOGOUT_SUCCESS.\n");
2608                 cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
2609                 break;
2610         case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION:
2611                 if (cmd->logout_response == ISCSI_LOGOUT_CID_NOT_FOUND)
2612                         break;
2613                 /*
2614                  * For CLOSECONNECTION logout requests carrying
2615                  * a matching logout CID -> local CID, the reference
2616                  * for the local CID will have been incremented in
2617                  * iscsi_logout_closeconnection().
2618                  *
2619                  * For CLOSECONNECTION logout requests carrying
2620                  * a different CID than the connection it arrived
2621                  * on, the connection responding to cmd->logout_cid
2622                  * is stopped in iscsit_logout_post_handler_diffcid().
2623                  */
2624
2625                 pr_debug("iSCSI CID: %hu logout on CID: %hu"
2626                         " successful.\n", cmd->logout_cid, conn->cid);
2627                 cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
2628                 break;
2629         case ISCSI_LOGOUT_REASON_RECOVERY:
2630                 if ((cmd->logout_response == ISCSI_LOGOUT_RECOVERY_UNSUPPORTED) ||
2631                     (cmd->logout_response == ISCSI_LOGOUT_CLEANUP_FAILED))
2632                         break;
2633                 /*
2634                  * If the connection is still active from our point of view
2635                  * force connection recovery to occur.
2636                  */
2637                 logout_conn = iscsit_get_conn_from_cid_rcfr(sess,
2638                                 cmd->logout_cid);
2639                 if ((logout_conn)) {
2640                         iscsit_connection_reinstatement_rcfr(logout_conn);
2641                         iscsit_dec_conn_usage_count(logout_conn);
2642                 }
2643
2644                 cr = iscsit_get_inactive_connection_recovery_entry(
2645                                 conn->sess, cmd->logout_cid);
2646                 if (!cr) {
2647                         pr_err("Unable to locate CID: %hu for"
2648                         " REMOVECONNFORRECOVERY Logout Request.\n",
2649                                 cmd->logout_cid);
2650                         cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND;
2651                         break;
2652                 }
2653
2654                 iscsit_discard_cr_cmds_by_expstatsn(cr, cmd->exp_stat_sn);
2655
2656                 pr_debug("iSCSI REMOVECONNFORRECOVERY logout"
2657                         " for recovery for CID: %hu on CID: %hu successful.\n",
2658                                 cmd->logout_cid, conn->cid);
2659                 cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
2660                 break;
2661         default:
2662                 pr_err("Unknown cmd->logout_reason: 0x%02x\n",
2663                                 cmd->logout_reason);
2664                 return -1;
2665         }
2666
2667         tx_size = ISCSI_HDR_LEN;
2668         hdr                     = (struct iscsi_logout_rsp *)cmd->pdu;
2669         memset(hdr, 0, ISCSI_HDR_LEN);
2670         hdr->opcode             = ISCSI_OP_LOGOUT_RSP;
2671         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
2672         hdr->response           = cmd->logout_response;
2673         hdr->itt                = cpu_to_be32(cmd->init_task_tag);
2674         cmd->stat_sn            = conn->stat_sn++;
2675         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2676
2677         iscsit_increment_maxcmdsn(cmd, conn->sess);
2678         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2679         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
2680
2681         iov = &cmd->iov_misc[0];
2682         iov[niov].iov_base      = cmd->pdu;
2683         iov[niov++].iov_len     = ISCSI_HDR_LEN;
2684
2685         if (conn->conn_ops->HeaderDigest) {
2686                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2687
2688                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2689                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
2690                                 0, NULL, (u8 *)header_digest);
2691
2692                 iov[0].iov_len += ISCSI_CRC_LEN;
2693                 tx_size += ISCSI_CRC_LEN;
2694                 pr_debug("Attaching CRC32C HeaderDigest to"
2695                         " Logout Response 0x%08x\n", *header_digest);
2696         }
2697         cmd->iov_misc_count = niov;
2698         cmd->tx_size = tx_size;
2699
2700         pr_debug("Sending Logout Response ITT: 0x%08x StatSN:"
2701                 " 0x%08x Response: 0x%02x CID: %hu on CID: %hu\n",
2702                 cmd->init_task_tag, cmd->stat_sn, hdr->response,
2703                 cmd->logout_cid, conn->cid);
2704
2705         return 0;
2706 }
2707
2708 /*
2709  *      Unsolicited NOPIN, either requesting a response or not.
2710  */
2711 static int iscsit_send_unsolicited_nopin(
2712         struct iscsi_cmd *cmd,
2713         struct iscsi_conn *conn,
2714         int want_response)
2715 {
2716         int tx_size = ISCSI_HDR_LEN;
2717         struct iscsi_nopin *hdr;
2718
2719         hdr                     = (struct iscsi_nopin *) cmd->pdu;
2720         memset(hdr, 0, ISCSI_HDR_LEN);
2721         hdr->opcode             = ISCSI_OP_NOOP_IN;
2722         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
2723         hdr->itt                = cpu_to_be32(cmd->init_task_tag);
2724         hdr->ttt                = cpu_to_be32(cmd->targ_xfer_tag);
2725         cmd->stat_sn            = conn->stat_sn;
2726         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2727         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2728         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
2729
2730         if (conn->conn_ops->HeaderDigest) {
2731                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2732
2733                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2734                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
2735                                 0, NULL, (u8 *)header_digest);
2736
2737                 tx_size += ISCSI_CRC_LEN;
2738                 pr_debug("Attaching CRC32C HeaderDigest to"
2739                         " NopIN 0x%08x\n", *header_digest);
2740         }
2741
2742         cmd->iov_misc[0].iov_base       = cmd->pdu;
2743         cmd->iov_misc[0].iov_len        = tx_size;
2744         cmd->iov_misc_count     = 1;
2745         cmd->tx_size            = tx_size;
2746
2747         pr_debug("Sending Unsolicited NOPIN TTT: 0x%08x StatSN:"
2748                 " 0x%08x CID: %hu\n", hdr->ttt, cmd->stat_sn, conn->cid);
2749
2750         return 0;
2751 }
2752
2753 static int iscsit_send_nopin_response(
2754         struct iscsi_cmd *cmd,
2755         struct iscsi_conn *conn)
2756 {
2757         int niov = 0, tx_size;
2758         u32 padding = 0;
2759         struct kvec *iov;
2760         struct iscsi_nopin *hdr;
2761
2762         tx_size = ISCSI_HDR_LEN;
2763         hdr                     = (struct iscsi_nopin *) cmd->pdu;
2764         memset(hdr, 0, ISCSI_HDR_LEN);
2765         hdr->opcode             = ISCSI_OP_NOOP_IN;
2766         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
2767         hton24(hdr->dlength, cmd->buf_ptr_size);
2768         put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun);
2769         hdr->itt                = cpu_to_be32(cmd->init_task_tag);
2770         hdr->ttt                = cpu_to_be32(cmd->targ_xfer_tag);
2771         cmd->stat_sn            = conn->stat_sn++;
2772         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2773
2774         iscsit_increment_maxcmdsn(cmd, conn->sess);
2775         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2776         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
2777
2778         iov = &cmd->iov_misc[0];
2779         iov[niov].iov_base      = cmd->pdu;
2780         iov[niov++].iov_len     = ISCSI_HDR_LEN;
2781
2782         if (conn->conn_ops->HeaderDigest) {
2783                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2784
2785                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2786                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
2787                                 0, NULL, (u8 *)header_digest);
2788
2789                 iov[0].iov_len += ISCSI_CRC_LEN;
2790                 tx_size += ISCSI_CRC_LEN;
2791                 pr_debug("Attaching CRC32C HeaderDigest"
2792                         " to NopIn 0x%08x\n", *header_digest);
2793         }
2794
2795         /*
2796          * NOPOUT Ping Data is attached to struct iscsi_cmd->buf_ptr.
2797          * NOPOUT DataSegmentLength is at struct iscsi_cmd->buf_ptr_size.
2798          */
2799         if (cmd->buf_ptr_size) {
2800                 iov[niov].iov_base      = cmd->buf_ptr;
2801                 iov[niov++].iov_len     = cmd->buf_ptr_size;
2802                 tx_size += cmd->buf_ptr_size;
2803
2804                 pr_debug("Echoing back %u bytes of ping"
2805                         " data.\n", cmd->buf_ptr_size);
2806
2807                 padding = ((-cmd->buf_ptr_size) & 3);
2808                 if (padding != 0) {
2809                         iov[niov].iov_base = &cmd->pad_bytes;
2810                         iov[niov++].iov_len = padding;
2811                         tx_size += padding;
2812                         pr_debug("Attaching %u additional"
2813                                 " padding bytes.\n", padding);
2814                 }
2815                 if (conn->conn_ops->DataDigest) {
2816                         iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2817                                 cmd->buf_ptr, cmd->buf_ptr_size,
2818                                 padding, (u8 *)&cmd->pad_bytes,
2819                                 (u8 *)&cmd->data_crc);
2820
2821                         iov[niov].iov_base = &cmd->data_crc;
2822                         iov[niov++].iov_len = ISCSI_CRC_LEN;
2823                         tx_size += ISCSI_CRC_LEN;
2824                         pr_debug("Attached DataDigest for %u"
2825                                 " bytes of ping data, CRC 0x%08x\n",
2826                                 cmd->buf_ptr_size, cmd->data_crc);
2827                 }
2828         }
2829
2830         cmd->iov_misc_count = niov;
2831         cmd->tx_size = tx_size;
2832
2833         pr_debug("Sending NOPIN Response ITT: 0x%08x, TTT:"
2834                 " 0x%08x, StatSN: 0x%08x, Length %u\n", cmd->init_task_tag,
2835                 cmd->targ_xfer_tag, cmd->stat_sn, cmd->buf_ptr_size);
2836
2837         return 0;
2838 }
2839
2840 int iscsit_send_r2t(
2841         struct iscsi_cmd *cmd,
2842         struct iscsi_conn *conn)
2843 {
2844         int tx_size = 0;
2845         struct iscsi_r2t *r2t;
2846         struct iscsi_r2t_rsp *hdr;
2847
2848         r2t = iscsit_get_r2t_from_list(cmd);
2849         if (!r2t)
2850                 return -1;
2851
2852         hdr                     = (struct iscsi_r2t_rsp *) cmd->pdu;
2853         memset(hdr, 0, ISCSI_HDR_LEN);
2854         hdr->opcode             = ISCSI_OP_R2T;
2855         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
2856         int_to_scsilun(cmd->se_cmd.orig_fe_lun,
2857                         (struct scsi_lun *)&hdr->lun);
2858         hdr->itt                = cpu_to_be32(cmd->init_task_tag);
2859         spin_lock_bh(&conn->sess->ttt_lock);
2860         r2t->targ_xfer_tag      = conn->sess->targ_xfer_tag++;
2861         if (r2t->targ_xfer_tag == 0xFFFFFFFF)
2862                 r2t->targ_xfer_tag = conn->sess->targ_xfer_tag++;
2863         spin_unlock_bh(&conn->sess->ttt_lock);
2864         hdr->ttt                = cpu_to_be32(r2t->targ_xfer_tag);
2865         hdr->statsn             = cpu_to_be32(conn->stat_sn);
2866         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2867         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
2868         hdr->r2tsn              = cpu_to_be32(r2t->r2t_sn);
2869         hdr->data_offset        = cpu_to_be32(r2t->offset);
2870         hdr->data_length        = cpu_to_be32(r2t->xfer_len);
2871
2872         cmd->iov_misc[0].iov_base       = cmd->pdu;
2873         cmd->iov_misc[0].iov_len        = ISCSI_HDR_LEN;
2874         tx_size += ISCSI_HDR_LEN;
2875
2876         if (conn->conn_ops->HeaderDigest) {
2877                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2878
2879                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2880                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
2881                                 0, NULL, (u8 *)header_digest);
2882
2883                 cmd->iov_misc[0].iov_len += ISCSI_CRC_LEN;
2884                 tx_size += ISCSI_CRC_LEN;
2885                 pr_debug("Attaching CRC32 HeaderDigest for R2T"
2886                         " PDU 0x%08x\n", *header_digest);
2887         }
2888
2889         pr_debug("Built %sR2T, ITT: 0x%08x, TTT: 0x%08x, StatSN:"
2890                 " 0x%08x, R2TSN: 0x%08x, Offset: %u, DDTL: %u, CID: %hu\n",
2891                 (!r2t->recovery_r2t) ? "" : "Recovery ", cmd->init_task_tag,
2892                 r2t->targ_xfer_tag, ntohl(hdr->statsn), r2t->r2t_sn,
2893                         r2t->offset, r2t->xfer_len, conn->cid);
2894
2895         cmd->iov_misc_count = 1;
2896         cmd->tx_size = tx_size;
2897
2898         spin_lock_bh(&cmd->r2t_lock);
2899         r2t->sent_r2t = 1;
2900         spin_unlock_bh(&cmd->r2t_lock);
2901
2902         return 0;
2903 }
2904
2905 /*
2906  *      type 0: Normal Operation.
2907  *      type 1: Called from Storage Transport.
2908  *      type 2: Called from iscsi_task_reassign_complete_write() for
2909  *              connection recovery.
2910  */
2911 int iscsit_build_r2ts_for_cmd(
2912         struct iscsi_cmd *cmd,
2913         struct iscsi_conn *conn,
2914         int type)
2915 {
2916         int first_r2t = 1;
2917         u32 offset = 0, xfer_len = 0;
2918
2919         spin_lock_bh(&cmd->r2t_lock);
2920         if (cmd->cmd_flags & ICF_SENT_LAST_R2T) {
2921                 spin_unlock_bh(&cmd->r2t_lock);
2922                 return 0;
2923         }
2924
2925         if (conn->sess->sess_ops->DataSequenceInOrder && (type != 2))
2926                 cmd->r2t_offset = max(cmd->r2t_offset, cmd->write_data_done);
2927
2928         while (cmd->outstanding_r2ts < conn->sess->sess_ops->MaxOutstandingR2T) {
2929                 if (conn->sess->sess_ops->DataSequenceInOrder) {
2930                         offset = cmd->r2t_offset;
2931
2932                         if (first_r2t && (type == 2)) {
2933                                 xfer_len = ((offset +
2934                                              (conn->sess->sess_ops->MaxBurstLength -
2935                                              cmd->next_burst_len) >
2936                                              cmd->data_length) ?
2937                                             (cmd->data_length - offset) :
2938                                             (conn->sess->sess_ops->MaxBurstLength -
2939                                              cmd->next_burst_len));
2940                         } else {
2941                                 xfer_len = ((offset +
2942                                              conn->sess->sess_ops->MaxBurstLength) >
2943                                              cmd->data_length) ?
2944                                              (cmd->data_length - offset) :
2945                                              conn->sess->sess_ops->MaxBurstLength;
2946                         }
2947                         cmd->r2t_offset += xfer_len;
2948
2949                         if (cmd->r2t_offset == cmd->data_length)
2950                                 cmd->cmd_flags |= ICF_SENT_LAST_R2T;
2951                 } else {
2952                         struct iscsi_seq *seq;
2953
2954                         seq = iscsit_get_seq_holder_for_r2t(cmd);
2955                         if (!seq) {
2956                                 spin_unlock_bh(&cmd->r2t_lock);
2957                                 return -1;
2958                         }
2959
2960                         offset = seq->offset;
2961                         xfer_len = seq->xfer_len;
2962
2963                         if (cmd->seq_send_order == cmd->seq_count)
2964                                 cmd->cmd_flags |= ICF_SENT_LAST_R2T;
2965                 }
2966                 cmd->outstanding_r2ts++;
2967                 first_r2t = 0;
2968
2969                 if (iscsit_add_r2t_to_list(cmd, offset, xfer_len, 0, 0) < 0) {
2970                         spin_unlock_bh(&cmd->r2t_lock);
2971                         return -1;
2972                 }
2973
2974                 if (cmd->cmd_flags & ICF_SENT_LAST_R2T)
2975                         break;
2976         }
2977         spin_unlock_bh(&cmd->r2t_lock);
2978
2979         return 0;
2980 }
2981
2982 static int iscsit_send_status(
2983         struct iscsi_cmd *cmd,
2984         struct iscsi_conn *conn)
2985 {
2986         u8 iov_count = 0, recovery;
2987         u32 padding = 0, tx_size = 0;
2988         struct iscsi_scsi_rsp *hdr;
2989         struct kvec *iov;
2990
2991         recovery = (cmd->i_state != ISTATE_SEND_STATUS);
2992         if (!recovery)
2993                 cmd->stat_sn = conn->stat_sn++;
2994
2995         spin_lock_bh(&conn->sess->session_stats_lock);
2996         conn->sess->rsp_pdus++;
2997         spin_unlock_bh(&conn->sess->session_stats_lock);
2998
2999         hdr                     = (struct iscsi_scsi_rsp *) cmd->pdu;
3000         memset(hdr, 0, ISCSI_HDR_LEN);
3001         hdr->opcode             = ISCSI_OP_SCSI_CMD_RSP;
3002         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
3003         if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) {
3004                 hdr->flags |= ISCSI_FLAG_CMD_OVERFLOW;
3005                 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
3006         } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) {
3007                 hdr->flags |= ISCSI_FLAG_CMD_UNDERFLOW;
3008                 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
3009         }
3010         hdr->response           = cmd->iscsi_response;
3011         hdr->cmd_status         = cmd->se_cmd.scsi_status;
3012         hdr->itt                = cpu_to_be32(cmd->init_task_tag);
3013         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3014
3015         iscsit_increment_maxcmdsn(cmd, conn->sess);
3016         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3017         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
3018
3019         iov = &cmd->iov_misc[0];
3020         iov[iov_count].iov_base = cmd->pdu;
3021         iov[iov_count++].iov_len = ISCSI_HDR_LEN;
3022         tx_size += ISCSI_HDR_LEN;
3023
3024         /*
3025          * Attach SENSE DATA payload to iSCSI Response PDU
3026          */
3027         if (cmd->se_cmd.sense_buffer &&
3028            ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ||
3029             (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) {
3030                 padding         = -(cmd->se_cmd.scsi_sense_length) & 3;
3031                 hton24(hdr->dlength, cmd->se_cmd.scsi_sense_length);
3032                 iov[iov_count].iov_base = cmd->se_cmd.sense_buffer;
3033                 iov[iov_count++].iov_len =
3034                                 (cmd->se_cmd.scsi_sense_length + padding);
3035                 tx_size += cmd->se_cmd.scsi_sense_length;
3036
3037                 if (padding) {
3038                         memset(cmd->se_cmd.sense_buffer +
3039                                 cmd->se_cmd.scsi_sense_length, 0, padding);
3040                         tx_size += padding;
3041                         pr_debug("Adding %u bytes of padding to"
3042                                 " SENSE.\n", padding);
3043                 }
3044
3045                 if (conn->conn_ops->DataDigest) {
3046                         iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3047                                 cmd->se_cmd.sense_buffer,
3048                                 (cmd->se_cmd.scsi_sense_length + padding),
3049                                 0, NULL, (u8 *)&cmd->data_crc);
3050
3051                         iov[iov_count].iov_base    = &cmd->data_crc;
3052                         iov[iov_count++].iov_len     = ISCSI_CRC_LEN;
3053                         tx_size += ISCSI_CRC_LEN;
3054
3055                         pr_debug("Attaching CRC32 DataDigest for"
3056                                 " SENSE, %u bytes CRC 0x%08x\n",
3057                                 (cmd->se_cmd.scsi_sense_length + padding),
3058                                 cmd->data_crc);
3059                 }
3060
3061                 pr_debug("Attaching SENSE DATA: %u bytes to iSCSI"
3062                                 " Response PDU\n",
3063                                 cmd->se_cmd.scsi_sense_length);
3064         }
3065
3066         if (conn->conn_ops->HeaderDigest) {
3067                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
3068
3069                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3070                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
3071                                 0, NULL, (u8 *)header_digest);
3072
3073                 iov[0].iov_len += ISCSI_CRC_LEN;
3074                 tx_size += ISCSI_CRC_LEN;
3075                 pr_debug("Attaching CRC32 HeaderDigest for Response"
3076                                 " PDU 0x%08x\n", *header_digest);
3077         }
3078
3079         cmd->iov_misc_count = iov_count;
3080         cmd->tx_size = tx_size;
3081
3082         pr_debug("Built %sSCSI Response, ITT: 0x%08x, StatSN: 0x%08x,"
3083                 " Response: 0x%02x, SAM Status: 0x%02x, CID: %hu\n",
3084                 (!recovery) ? "" : "Recovery ", cmd->init_task_tag,
3085                 cmd->stat_sn, 0x00, cmd->se_cmd.scsi_status, conn->cid);
3086
3087         return 0;
3088 }
3089
3090 static u8 iscsit_convert_tcm_tmr_rsp(struct se_tmr_req *se_tmr)
3091 {
3092         switch (se_tmr->response) {
3093         case TMR_FUNCTION_COMPLETE:
3094                 return ISCSI_TMF_RSP_COMPLETE;
3095         case TMR_TASK_DOES_NOT_EXIST:
3096                 return ISCSI_TMF_RSP_NO_TASK;
3097         case TMR_LUN_DOES_NOT_EXIST:
3098                 return ISCSI_TMF_RSP_NO_LUN;
3099         case TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED:
3100                 return ISCSI_TMF_RSP_NOT_SUPPORTED;
3101         case TMR_FUNCTION_AUTHORIZATION_FAILED:
3102                 return ISCSI_TMF_RSP_AUTH_FAILED;
3103         case TMR_FUNCTION_REJECTED:
3104         default:
3105                 return ISCSI_TMF_RSP_REJECTED;
3106         }
3107 }
3108
3109 static int iscsit_send_task_mgt_rsp(
3110         struct iscsi_cmd *cmd,
3111         struct iscsi_conn *conn)
3112 {
3113         struct se_tmr_req *se_tmr = cmd->se_cmd.se_tmr_req;
3114         struct iscsi_tm_rsp *hdr;
3115         u32 tx_size = 0;
3116
3117         hdr                     = (struct iscsi_tm_rsp *) cmd->pdu;
3118         memset(hdr, 0, ISCSI_HDR_LEN);
3119         hdr->opcode             = ISCSI_OP_SCSI_TMFUNC_RSP;
3120         hdr->flags              = ISCSI_FLAG_CMD_FINAL;
3121         hdr->response           = iscsit_convert_tcm_tmr_rsp(se_tmr);
3122         hdr->itt                = cpu_to_be32(cmd->init_task_tag);
3123         cmd->stat_sn            = conn->stat_sn++;
3124         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3125
3126         iscsit_increment_maxcmdsn(cmd, conn->sess);
3127         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3128         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
3129
3130         cmd->iov_misc[0].iov_base       = cmd->pdu;
3131         cmd->iov_misc[0].iov_len        = ISCSI_HDR_LEN;
3132         tx_size += ISCSI_HDR_LEN;
3133
3134         if (conn->conn_ops->HeaderDigest) {
3135                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
3136
3137                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3138                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
3139                                 0, NULL, (u8 *)header_digest);
3140
3141                 cmd->iov_misc[0].iov_len += ISCSI_CRC_LEN;
3142                 tx_size += ISCSI_CRC_LEN;
3143                 pr_debug("Attaching CRC32 HeaderDigest for Task"
3144                         " Mgmt Response PDU 0x%08x\n", *header_digest);
3145         }
3146
3147         cmd->iov_misc_count = 1;
3148         cmd->tx_size = tx_size;
3149
3150         pr_debug("Built Task Management Response ITT: 0x%08x,"
3151                 " StatSN: 0x%08x, Response: 0x%02x, CID: %hu\n",
3152                 cmd->init_task_tag, cmd->stat_sn, hdr->response, conn->cid);
3153
3154         return 0;
3155 }
3156
3157 static bool iscsit_check_inaddr_any(struct iscsi_np *np)
3158 {
3159         bool ret = false;
3160
3161         if (np->np_sockaddr.ss_family == AF_INET6) {
3162                 const struct sockaddr_in6 sin6 = {
3163                         .sin6_addr = IN6ADDR_ANY_INIT };
3164                 struct sockaddr_in6 *sock_in6 =
3165                          (struct sockaddr_in6 *)&np->np_sockaddr;
3166
3167                 if (!memcmp(sock_in6->sin6_addr.s6_addr,
3168                                 sin6.sin6_addr.s6_addr, 16))
3169                         ret = true;
3170         } else {
3171                 struct sockaddr_in * sock_in =
3172                         (struct sockaddr_in *)&np->np_sockaddr;
3173
3174                 if (sock_in->sin_addr.s_addr == INADDR_ANY)
3175                         ret = true;
3176         }
3177
3178         return ret;
3179 }
3180
3181 static int iscsit_build_sendtargets_response(struct iscsi_cmd *cmd)
3182 {
3183         char *payload = NULL;
3184         struct iscsi_conn *conn = cmd->conn;
3185         struct iscsi_portal_group *tpg;
3186         struct iscsi_tiqn *tiqn;
3187         struct iscsi_tpg_np *tpg_np;
3188         int buffer_len, end_of_buf = 0, len = 0, payload_len = 0;
3189         unsigned char buf[256];
3190
3191         buffer_len = (conn->conn_ops->MaxRecvDataSegmentLength > 32768) ?
3192                         32768 : conn->conn_ops->MaxRecvDataSegmentLength;
3193
3194         memset(buf, 0, 256);
3195
3196         payload = kzalloc(buffer_len, GFP_KERNEL);
3197         if (!payload) {
3198                 pr_err("Unable to allocate memory for sendtargets"
3199                                 " response.\n");
3200                 return -ENOMEM;
3201         }
3202
3203         spin_lock(&tiqn_lock);
3204         list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) {
3205                 len = sprintf(buf, "TargetName=%s", tiqn->tiqn);
3206                 len += 1;
3207
3208                 if ((len + payload_len) > buffer_len) {
3209                         spin_unlock(&tiqn->tiqn_tpg_lock);
3210                         end_of_buf = 1;
3211                         goto eob;
3212                 }
3213                 memcpy(payload + payload_len, buf, len);
3214                 payload_len += len;
3215
3216                 spin_lock(&tiqn->tiqn_tpg_lock);
3217                 list_for_each_entry(tpg, &tiqn->tiqn_tpg_list, tpg_list) {
3218
3219                         spin_lock(&tpg->tpg_state_lock);
3220                         if ((tpg->tpg_state == TPG_STATE_FREE) ||
3221                             (tpg->tpg_state == TPG_STATE_INACTIVE)) {
3222                                 spin_unlock(&tpg->tpg_state_lock);
3223                                 continue;
3224                         }
3225                         spin_unlock(&tpg->tpg_state_lock);
3226
3227                         spin_lock(&tpg->tpg_np_lock);
3228                         list_for_each_entry(tpg_np, &tpg->tpg_gnp_list,
3229                                                 tpg_np_list) {
3230                                 struct iscsi_np *np = tpg_np->tpg_np;
3231                                 bool inaddr_any = iscsit_check_inaddr_any(np);
3232
3233                                 len = sprintf(buf, "TargetAddress="
3234                                         "%s%s%s:%hu,%hu",
3235                                         (np->np_sockaddr.ss_family == AF_INET6) ?
3236                                         "[" : "", (inaddr_any == false) ?
3237                                                 np->np_ip : conn->local_ip,
3238                                         (np->np_sockaddr.ss_family == AF_INET6) ?
3239                                         "]" : "", (inaddr_any == false) ?
3240                                                 np->np_port : conn->local_port,
3241                                         tpg->tpgt);
3242                                 len += 1;
3243
3244                                 if ((len + payload_len) > buffer_len) {
3245                                         spin_unlock(&tpg->tpg_np_lock);
3246                                         spin_unlock(&tiqn->tiqn_tpg_lock);
3247                                         end_of_buf = 1;
3248                                         goto eob;
3249                                 }
3250                                 memcpy(payload + payload_len, buf, len);
3251                                 payload_len += len;
3252                         }
3253                         spin_unlock(&tpg->tpg_np_lock);
3254                 }
3255                 spin_unlock(&tiqn->tiqn_tpg_lock);
3256 eob:
3257                 if (end_of_buf)
3258                         break;
3259         }
3260         spin_unlock(&tiqn_lock);
3261
3262         cmd->buf_ptr = payload;
3263
3264         return payload_len;
3265 }
3266
3267 /*
3268  *      FIXME: Add support for F_BIT and C_BIT when the length is longer than
3269  *      MaxRecvDataSegmentLength.
3270  */
3271 static int iscsit_send_text_rsp(
3272         struct iscsi_cmd *cmd,
3273         struct iscsi_conn *conn)
3274 {
3275         struct iscsi_text_rsp *hdr;
3276         struct kvec *iov;
3277         u32 padding = 0, tx_size = 0;
3278         int text_length, iov_count = 0;
3279
3280         text_length = iscsit_build_sendtargets_response(cmd);
3281         if (text_length < 0)
3282                 return text_length;
3283
3284         padding = ((-text_length) & 3);
3285         if (padding != 0) {
3286                 memset(cmd->buf_ptr + text_length, 0, padding);
3287                 pr_debug("Attaching %u additional bytes for"
3288                         " padding.\n", padding);
3289         }
3290
3291         hdr                     = (struct iscsi_text_rsp *) cmd->pdu;
3292         memset(hdr, 0, ISCSI_HDR_LEN);
3293         hdr->opcode             = ISCSI_OP_TEXT_RSP;
3294         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
3295         hton24(hdr->dlength, text_length);
3296         hdr->itt                = cpu_to_be32(cmd->init_task_tag);
3297         hdr->ttt                = cpu_to_be32(cmd->targ_xfer_tag);
3298         cmd->stat_sn            = conn->stat_sn++;
3299         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3300
3301         iscsit_increment_maxcmdsn(cmd, conn->sess);
3302         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3303         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
3304
3305         iov = &cmd->iov_misc[0];
3306
3307         iov[iov_count].iov_base = cmd->pdu;
3308         iov[iov_count++].iov_len = ISCSI_HDR_LEN;
3309         iov[iov_count].iov_base = cmd->buf_ptr;
3310         iov[iov_count++].iov_len = text_length + padding;
3311
3312         tx_size += (ISCSI_HDR_LEN + text_length + padding);
3313
3314         if (conn->conn_ops->HeaderDigest) {
3315                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
3316
3317                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3318                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
3319                                 0, NULL, (u8 *)header_digest);
3320
3321                 iov[0].iov_len += ISCSI_CRC_LEN;
3322                 tx_size += ISCSI_CRC_LEN;
3323                 pr_debug("Attaching CRC32 HeaderDigest for"
3324                         " Text Response PDU 0x%08x\n", *header_digest);
3325         }
3326
3327         if (conn->conn_ops->DataDigest) {
3328                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3329                                 cmd->buf_ptr, (text_length + padding),
3330                                 0, NULL, (u8 *)&cmd->data_crc);
3331
3332                 iov[iov_count].iov_base = &cmd->data_crc;
3333                 iov[iov_count++].iov_len = ISCSI_CRC_LEN;
3334                 tx_size += ISCSI_CRC_LEN;
3335
3336                 pr_debug("Attaching DataDigest for %u bytes of text"
3337                         " data, CRC 0x%08x\n", (text_length + padding),
3338                         cmd->data_crc);
3339         }
3340
3341         cmd->iov_misc_count = iov_count;
3342         cmd->tx_size = tx_size;
3343
3344         pr_debug("Built Text Response: ITT: 0x%08x, StatSN: 0x%08x,"
3345                 " Length: %u, CID: %hu\n", cmd->init_task_tag, cmd->stat_sn,
3346                         text_length, conn->cid);
3347         return 0;
3348 }
3349
3350 static int iscsit_send_reject(
3351         struct iscsi_cmd *cmd,
3352         struct iscsi_conn *conn)
3353 {
3354         u32 iov_count = 0, tx_size = 0;
3355         struct iscsi_reject *hdr;
3356         struct kvec *iov;
3357
3358         hdr                     = (struct iscsi_reject *) cmd->pdu;
3359         hdr->opcode             = ISCSI_OP_REJECT;
3360         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
3361         hton24(hdr->dlength, ISCSI_HDR_LEN);
3362         cmd->stat_sn            = conn->stat_sn++;
3363         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3364         hdr->exp_cmdsn  = cpu_to_be32(conn->sess->exp_cmd_sn);
3365         hdr->max_cmdsn  = cpu_to_be32(conn->sess->max_cmd_sn);
3366
3367         iov = &cmd->iov_misc[0];
3368
3369         iov[iov_count].iov_base = cmd->pdu;
3370         iov[iov_count++].iov_len = ISCSI_HDR_LEN;
3371         iov[iov_count].iov_base = cmd->buf_ptr;
3372         iov[iov_count++].iov_len = ISCSI_HDR_LEN;
3373
3374         tx_size = (ISCSI_HDR_LEN + ISCSI_HDR_LEN);
3375
3376         if (conn->conn_ops->HeaderDigest) {
3377                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
3378
3379                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3380                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
3381                                 0, NULL, (u8 *)header_digest);
3382
3383                 iov[0].iov_len += ISCSI_CRC_LEN;
3384                 tx_size += ISCSI_CRC_LEN;
3385                 pr_debug("Attaching CRC32 HeaderDigest for"
3386                         " REJECT PDU 0x%08x\n", *header_digest);
3387         }
3388
3389         if (conn->conn_ops->DataDigest) {
3390                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3391                                 (unsigned char *)cmd->buf_ptr, ISCSI_HDR_LEN,
3392                                 0, NULL, (u8 *)&cmd->data_crc);
3393
3394                 iov[iov_count].iov_base = &cmd->data_crc;
3395                 iov[iov_count++].iov_len  = ISCSI_CRC_LEN;
3396                 tx_size += ISCSI_CRC_LEN;
3397                 pr_debug("Attaching CRC32 DataDigest for REJECT"
3398                                 " PDU 0x%08x\n", cmd->data_crc);
3399         }
3400
3401         cmd->iov_misc_count = iov_count;
3402         cmd->tx_size = tx_size;
3403
3404         pr_debug("Built Reject PDU StatSN: 0x%08x, Reason: 0x%02x,"
3405                 " CID: %hu\n", ntohl(hdr->statsn), hdr->reason, conn->cid);
3406
3407         return 0;
3408 }
3409
3410 static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *conn)
3411 {
3412         if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) ||
3413             (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) {
3414                 wait_for_completion_interruptible_timeout(
3415                                         &conn->tx_half_close_comp,
3416                                         ISCSI_TX_THREAD_TCP_TIMEOUT * HZ);
3417         }
3418 }
3419
3420 void iscsit_thread_get_cpumask(struct iscsi_conn *conn)
3421 {
3422         struct iscsi_thread_set *ts = conn->thread_set;
3423         int ord, cpu;
3424         /*
3425          * thread_id is assigned from iscsit_global->ts_bitmap from
3426          * within iscsi_thread_set.c:iscsi_allocate_thread_sets()
3427          *
3428          * Here we use thread_id to determine which CPU that this
3429          * iSCSI connection's iscsi_thread_set will be scheduled to
3430          * execute upon.
3431          */
3432         ord = ts->thread_id % cpumask_weight(cpu_online_mask);
3433         for_each_online_cpu(cpu) {
3434                 if (ord-- == 0) {
3435                         cpumask_set_cpu(cpu, conn->conn_cpumask);
3436                         return;
3437                 }
3438         }
3439         /*
3440          * This should never be reached..
3441          */
3442         dump_stack();
3443         cpumask_setall(conn->conn_cpumask);
3444 }
3445
3446 static inline void iscsit_thread_check_cpumask(
3447         struct iscsi_conn *conn,
3448         struct task_struct *p,
3449         int mode)
3450 {
3451         char buf[128];
3452         /*
3453          * mode == 1 signals iscsi_target_tx_thread() usage.
3454          * mode == 0 signals iscsi_target_rx_thread() usage.
3455          */
3456         if (mode == 1) {
3457                 if (!conn->conn_tx_reset_cpumask)
3458                         return;
3459                 conn->conn_tx_reset_cpumask = 0;
3460         } else {
3461                 if (!conn->conn_rx_reset_cpumask)
3462                         return;
3463                 conn->conn_rx_reset_cpumask = 0;
3464         }
3465         /*
3466          * Update the CPU mask for this single kthread so that
3467          * both TX and RX kthreads are scheduled to run on the
3468          * same CPU.
3469          */
3470         memset(buf, 0, 128);
3471         cpumask_scnprintf(buf, 128, conn->conn_cpumask);
3472         set_cpus_allowed_ptr(p, conn->conn_cpumask);
3473 }
3474
3475 int iscsi_target_tx_thread(void *arg)
3476 {
3477         u8 state;
3478         int eodr = 0;
3479         int ret = 0;
3480         int sent_status = 0;
3481         int use_misc = 0;
3482         int map_sg = 0;
3483         struct iscsi_cmd *cmd = NULL;
3484         struct iscsi_conn *conn;
3485         struct iscsi_queue_req *qr = NULL;
3486         struct iscsi_thread_set *ts = arg;
3487         /*
3488          * Allow ourselves to be interrupted by SIGINT so that a
3489          * connection recovery / failure event can be triggered externally.
3490          */
3491         allow_signal(SIGINT);
3492
3493 restart:
3494         conn = iscsi_tx_thread_pre_handler(ts);
3495         if (!conn)
3496                 goto out;
3497
3498         eodr = map_sg = ret = sent_status = use_misc = 0;
3499
3500         while (!kthread_should_stop()) {
3501                 /*
3502                  * Ensure that both TX and RX per connection kthreads
3503                  * are scheduled to run on the same CPU.
3504                  */
3505                 iscsit_thread_check_cpumask(conn, current, 1);
3506
3507                 schedule_timeout_interruptible(MAX_SCHEDULE_TIMEOUT);
3508
3509                 if ((ts->status == ISCSI_THREAD_SET_RESET) ||
3510                      signal_pending(current))
3511                         goto transport_err;
3512
3513                 while ((qr = iscsit_get_cmd_from_immediate_queue(conn))) {
3514                         atomic_set(&conn->check_immediate_queue, 0);
3515                         cmd = qr->cmd;
3516                         state = qr->state;
3517                         kmem_cache_free(lio_qr_cache, qr);
3518
3519                         spin_lock_bh(&cmd->istate_lock);
3520                         switch (state) {
3521                         case ISTATE_SEND_R2T:
3522                                 spin_unlock_bh(&cmd->istate_lock);
3523                                 ret = iscsit_send_r2t(cmd, conn);
3524                                 break;
3525                         case ISTATE_REMOVE:
3526                                 spin_unlock_bh(&cmd->istate_lock);
3527
3528                                 if (cmd->data_direction == DMA_TO_DEVICE)
3529                                         iscsit_stop_dataout_timer(cmd);
3530
3531                                 spin_lock_bh(&conn->cmd_lock);
3532                                 list_del(&cmd->i_conn_node);
3533                                 spin_unlock_bh(&conn->cmd_lock);
3534
3535                                 iscsit_free_cmd(cmd);
3536                                 continue;
3537                         case ISTATE_SEND_NOPIN_WANT_RESPONSE:
3538                                 spin_unlock_bh(&cmd->istate_lock);
3539                                 iscsit_mod_nopin_response_timer(conn);
3540                                 ret = iscsit_send_unsolicited_nopin(cmd,
3541                                                 conn, 1);
3542                                 break;
3543                         case ISTATE_SEND_NOPIN_NO_RESPONSE:
3544                                 spin_unlock_bh(&cmd->istate_lock);
3545                                 ret = iscsit_send_unsolicited_nopin(cmd,
3546                                                 conn, 0);
3547                                 break;
3548                         default:
3549                                 pr_err("Unknown Opcode: 0x%02x ITT:"
3550                                 " 0x%08x, i_state: %d on CID: %hu\n",
3551                                 cmd->iscsi_opcode, cmd->init_task_tag, state,
3552                                 conn->cid);
3553                                 spin_unlock_bh(&cmd->istate_lock);
3554                                 goto transport_err;
3555                         }
3556                         if (ret < 0)
3557                                 goto transport_err;
3558
3559                         if (iscsit_send_tx_data(cmd, conn, 1) < 0) {
3560                                 iscsit_tx_thread_wait_for_tcp(conn);
3561                                 goto transport_err;
3562                         }
3563
3564                         spin_lock_bh(&cmd->istate_lock);
3565                         switch (state) {
3566                         case ISTATE_SEND_R2T:
3567                                 spin_unlock_bh(&cmd->istate_lock);
3568                                 spin_lock_bh(&cmd->dataout_timeout_lock);
3569                                 iscsit_start_dataout_timer(cmd, conn);
3570                                 spin_unlock_bh(&cmd->dataout_timeout_lock);
3571                                 break;
3572                         case ISTATE_SEND_NOPIN_WANT_RESPONSE:
3573                                 cmd->i_state = ISTATE_SENT_NOPIN_WANT_RESPONSE;
3574                                 spin_unlock_bh(&cmd->istate_lock);
3575                                 break;
3576                         case ISTATE_SEND_NOPIN_NO_RESPONSE:
3577                                 cmd->i_state = ISTATE_SENT_STATUS;
3578                                 spin_unlock_bh(&cmd->istate_lock);
3579                                 break;
3580                         default:
3581                                 pr_err("Unknown Opcode: 0x%02x ITT:"
3582                                         " 0x%08x, i_state: %d on CID: %hu\n",
3583                                         cmd->iscsi_opcode, cmd->init_task_tag,
3584                                         state, conn->cid);
3585                                 spin_unlock_bh(&cmd->istate_lock);
3586                                 goto transport_err;
3587                         }
3588                 }
3589
3590                 while ((qr = iscsit_get_cmd_from_response_queue(conn))) {
3591                         cmd = qr->cmd;
3592                         state = qr->state;
3593                         kmem_cache_free(lio_qr_cache, qr);
3594
3595                         spin_lock_bh(&cmd->istate_lock);
3596 check_rsp_state:
3597                         switch (state) {
3598                         case ISTATE_SEND_DATAIN:
3599                                 spin_unlock_bh(&cmd->istate_lock);
3600                                 ret = iscsit_send_data_in(cmd, conn,
3601                                                           &eodr);
3602                                 map_sg = 1;
3603                                 break;
3604                         case ISTATE_SEND_STATUS:
3605                         case ISTATE_SEND_STATUS_RECOVERY:
3606                                 spin_unlock_bh(&cmd->istate_lock);
3607                                 use_misc = 1;
3608                                 ret = iscsit_send_status(cmd, conn);
3609                                 break;
3610                         case ISTATE_SEND_LOGOUTRSP:
3611                                 spin_unlock_bh(&cmd->istate_lock);
3612                                 use_misc = 1;
3613                                 ret = iscsit_send_logout_response(cmd, conn);
3614                                 break;
3615                         case ISTATE_SEND_ASYNCMSG:
3616                                 spin_unlock_bh(&cmd->istate_lock);
3617                                 use_misc = 1;
3618                                 ret = iscsit_send_conn_drop_async_message(
3619                                                 cmd, conn);
3620                                 break;
3621                         case ISTATE_SEND_NOPIN:
3622                                 spin_unlock_bh(&cmd->istate_lock);
3623                                 use_misc = 1;
3624                                 ret = iscsit_send_nopin_response(cmd, conn);
3625                                 break;
3626                         case ISTATE_SEND_REJECT:
3627                                 spin_unlock_bh(&cmd->istate_lock);
3628                                 use_misc = 1;
3629                                 ret = iscsit_send_reject(cmd, conn);
3630                                 break;
3631                         case ISTATE_SEND_TASKMGTRSP:
3632                                 spin_unlock_bh(&cmd->istate_lock);
3633                                 use_misc = 1;
3634                                 ret = iscsit_send_task_mgt_rsp(cmd, conn);
3635                                 if (ret != 0)
3636                                         break;
3637                                 ret = iscsit_tmr_post_handler(cmd, conn);
3638                                 if (ret != 0)
3639                                         iscsit_fall_back_to_erl0(conn->sess);
3640                                 break;
3641                         case ISTATE_SEND_TEXTRSP:
3642                                 spin_unlock_bh(&cmd->istate_lock);
3643                                 use_misc = 1;
3644                                 ret = iscsit_send_text_rsp(cmd, conn);
3645                                 break;
3646                         default:
3647                                 pr_err("Unknown Opcode: 0x%02x ITT:"
3648                                         " 0x%08x, i_state: %d on CID: %hu\n",
3649                                         cmd->iscsi_opcode, cmd->init_task_tag,
3650                                         state, conn->cid);
3651                                 spin_unlock_bh(&cmd->istate_lock);
3652                                 goto transport_err;
3653                         }
3654                         if (ret < 0)
3655                                 goto transport_err;
3656
3657                         if (map_sg && !conn->conn_ops->IFMarker) {
3658                                 if (iscsit_fe_sendpage_sg(cmd, conn) < 0) {
3659                                         iscsit_tx_thread_wait_for_tcp(conn);
3660                                         iscsit_unmap_iovec(cmd);
3661                                         goto transport_err;
3662                                 }
3663                         } else {
3664                                 if (iscsit_send_tx_data(cmd, conn, use_misc) < 0) {
3665                                         iscsit_tx_thread_wait_for_tcp(conn);
3666                                         iscsit_unmap_iovec(cmd);
3667                                         goto transport_err;
3668                                 }
3669                         }
3670                         map_sg = 0;
3671                         iscsit_unmap_iovec(cmd);
3672
3673                         spin_lock_bh(&cmd->istate_lock);
3674                         switch (state) {
3675                         case ISTATE_SEND_DATAIN:
3676                                 if (!eodr)
3677                                         goto check_rsp_state;
3678
3679                                 if (eodr == 1) {
3680                                         cmd->i_state = ISTATE_SENT_LAST_DATAIN;
3681                                         sent_status = 1;
3682                                         eodr = use_misc = 0;
3683                                 } else if (eodr == 2) {
3684                                         cmd->i_state = state =
3685                                                         ISTATE_SEND_STATUS;
3686                                         sent_status = 0;
3687                                         eodr = use_misc = 0;
3688                                         goto check_rsp_state;
3689                                 }
3690                                 break;
3691                         case ISTATE_SEND_STATUS:
3692                                 use_misc = 0;
3693                                 sent_status = 1;
3694                                 break;
3695                         case ISTATE_SEND_ASYNCMSG:
3696                         case ISTATE_SEND_NOPIN:
3697                         case ISTATE_SEND_STATUS_RECOVERY:
3698                         case ISTATE_SEND_TEXTRSP:
3699                                 use_misc = 0;
3700                                 sent_status = 1;
3701                                 break;
3702                         case ISTATE_SEND_REJECT:
3703                                 use_misc = 0;
3704                                 if (cmd->cmd_flags & ICF_REJECT_FAIL_CONN) {
3705                                         cmd->cmd_flags &= ~ICF_REJECT_FAIL_CONN;
3706                                         spin_unlock_bh(&cmd->istate_lock);
3707                                         complete(&cmd->reject_comp);
3708                                         goto transport_err;
3709                                 }
3710                                 complete(&cmd->reject_comp);
3711                                 break;
3712                         case ISTATE_SEND_TASKMGTRSP:
3713                                 use_misc = 0;
3714                                 sent_status = 1;
3715                                 break;
3716                         case ISTATE_SEND_LOGOUTRSP:
3717                                 spin_unlock_bh(&cmd->istate_lock);
3718                                 if (!iscsit_logout_post_handler(cmd, conn))
3719                                         goto restart;
3720                                 spin_lock_bh(&cmd->istate_lock);
3721                                 use_misc = 0;
3722                                 sent_status = 1;
3723                                 break;
3724                         default:
3725                                 pr_err("Unknown Opcode: 0x%02x ITT:"
3726                                         " 0x%08x, i_state: %d on CID: %hu\n",
3727                                         cmd->iscsi_opcode, cmd->init_task_tag,
3728                                         cmd->i_state, conn->cid);
3729                                 spin_unlock_bh(&cmd->istate_lock);
3730                                 goto transport_err;
3731                         }
3732
3733                         if (sent_status) {
3734                                 cmd->i_state = ISTATE_SENT_STATUS;
3735                                 sent_status = 0;
3736                         }
3737                         spin_unlock_bh(&cmd->istate_lock);
3738
3739                         if (atomic_read(&conn->check_immediate_queue))
3740                                 break;
3741                 }
3742         }
3743
3744 transport_err:
3745         iscsit_take_action_for_connection_exit(conn);
3746         goto restart;
3747 out:
3748         return 0;
3749 }
3750
3751 int iscsi_target_rx_thread(void *arg)
3752 {
3753         int ret;
3754         u8 buffer[ISCSI_HDR_LEN], opcode;
3755         u32 checksum = 0, digest = 0;
3756         struct iscsi_conn *conn = NULL;
3757         struct iscsi_thread_set *ts = arg;
3758         struct kvec iov;
3759         /*
3760          * Allow ourselves to be interrupted by SIGINT so that a
3761          * connection recovery / failure event can be triggered externally.
3762          */
3763         allow_signal(SIGINT);
3764
3765 restart:
3766         conn = iscsi_rx_thread_pre_handler(ts);
3767         if (!conn)
3768                 goto out;
3769
3770         while (!kthread_should_stop()) {
3771                 /*
3772                  * Ensure that both TX and RX per connection kthreads
3773                  * are scheduled to run on the same CPU.
3774                  */
3775                 iscsit_thread_check_cpumask(conn, current, 0);
3776
3777                 memset(buffer, 0, ISCSI_HDR_LEN);
3778                 memset(&iov, 0, sizeof(struct kvec));
3779
3780                 iov.iov_base    = buffer;
3781                 iov.iov_len     = ISCSI_HDR_LEN;
3782
3783                 ret = rx_data(conn, &iov, 1, ISCSI_HDR_LEN);
3784                 if (ret != ISCSI_HDR_LEN) {
3785                         iscsit_rx_thread_wait_for_tcp(conn);
3786                         goto transport_err;
3787                 }
3788
3789                 /*
3790                  * Set conn->bad_hdr for use with REJECT PDUs.
3791                  */
3792                 memcpy(&conn->bad_hdr, &buffer, ISCSI_HDR_LEN);
3793
3794                 if (conn->conn_ops->HeaderDigest) {
3795                         iov.iov_base    = &digest;
3796                         iov.iov_len     = ISCSI_CRC_LEN;
3797
3798                         ret = rx_data(conn, &iov, 1, ISCSI_CRC_LEN);
3799                         if (ret != ISCSI_CRC_LEN) {
3800                                 iscsit_rx_thread_wait_for_tcp(conn);
3801                                 goto transport_err;
3802                         }
3803
3804                         iscsit_do_crypto_hash_buf(&conn->conn_rx_hash,
3805                                         buffer, ISCSI_HDR_LEN,
3806                                         0, NULL, (u8 *)&checksum);
3807
3808                         if (digest != checksum) {
3809                                 pr_err("HeaderDigest CRC32C failed,"
3810                                         " received 0x%08x, computed 0x%08x\n",
3811                                         digest, checksum);
3812                                 /*
3813                                  * Set the PDU to 0xff so it will intentionally
3814                                  * hit default in the switch below.
3815                                  */
3816                                 memset(buffer, 0xff, ISCSI_HDR_LEN);
3817                                 spin_lock_bh(&conn->sess->session_stats_lock);
3818                                 conn->sess->conn_digest_errors++;
3819                                 spin_unlock_bh(&conn->sess->session_stats_lock);
3820                         } else {
3821                                 pr_debug("Got HeaderDigest CRC32C"
3822                                                 " 0x%08x\n", checksum);
3823                         }
3824                 }
3825
3826                 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)
3827                         goto transport_err;
3828
3829                 opcode = buffer[0] & ISCSI_OPCODE_MASK;
3830
3831                 if (conn->sess->sess_ops->SessionType &&
3832                    ((!(opcode & ISCSI_OP_TEXT)) ||
3833                     (!(opcode & ISCSI_OP_LOGOUT)))) {
3834                         pr_err("Received illegal iSCSI Opcode: 0x%02x"
3835                         " while in Discovery Session, rejecting.\n", opcode);
3836                         iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
3837                                         buffer, conn);
3838                         goto transport_err;
3839                 }
3840
3841                 switch (opcode) {
3842                 case ISCSI_OP_SCSI_CMD:
3843                         if (iscsit_handle_scsi_cmd(conn, buffer) < 0)
3844                                 goto transport_err;
3845                         break;
3846                 case ISCSI_OP_SCSI_DATA_OUT:
3847                         if (iscsit_handle_data_out(conn, buffer) < 0)
3848                                 goto transport_err;
3849                         break;
3850                 case ISCSI_OP_NOOP_OUT:
3851                         if (iscsit_handle_nop_out(conn, buffer) < 0)
3852                                 goto transport_err;
3853                         break;
3854                 case ISCSI_OP_SCSI_TMFUNC:
3855                         if (iscsit_handle_task_mgt_cmd(conn, buffer) < 0)
3856                                 goto transport_err;
3857                         break;
3858                 case ISCSI_OP_TEXT:
3859                         if (iscsit_handle_text_cmd(conn, buffer) < 0)
3860                                 goto transport_err;
3861                         break;
3862                 case ISCSI_OP_LOGOUT:
3863                         ret = iscsit_handle_logout_cmd(conn, buffer);
3864                         if (ret > 0) {
3865                                 wait_for_completion_timeout(&conn->conn_logout_comp,
3866                                                 SECONDS_FOR_LOGOUT_COMP * HZ);
3867                                 goto transport_err;
3868                         } else if (ret < 0)
3869                                 goto transport_err;
3870                         break;
3871                 case ISCSI_OP_SNACK:
3872                         if (iscsit_handle_snack(conn, buffer) < 0)
3873                                 goto transport_err;
3874                         break;
3875                 default:
3876                         pr_err("Got unknown iSCSI OpCode: 0x%02x\n",
3877                                         opcode);
3878                         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
3879                                 pr_err("Cannot recover from unknown"
3880                                 " opcode while ERL=0, closing iSCSI connection"
3881                                 ".\n");
3882                                 goto transport_err;
3883                         }
3884                         if (!conn->conn_ops->OFMarker) {
3885                                 pr_err("Unable to recover from unknown"
3886                                 " opcode while OFMarker=No, closing iSCSI"
3887                                         " connection.\n");
3888                                 goto transport_err;
3889                         }
3890                         if (iscsit_recover_from_unknown_opcode(conn) < 0) {
3891                                 pr_err("Unable to recover from unknown"
3892                                         " opcode, closing iSCSI connection.\n");
3893                                 goto transport_err;
3894                         }
3895                         break;
3896                 }
3897         }
3898
3899 transport_err:
3900         if (!signal_pending(current))
3901                 atomic_set(&conn->transport_failed, 1);
3902         iscsit_take_action_for_connection_exit(conn);
3903         goto restart;
3904 out:
3905         return 0;
3906 }
3907
3908 static void iscsit_release_commands_from_conn(struct iscsi_conn *conn)
3909 {
3910         struct iscsi_cmd *cmd = NULL, *cmd_tmp = NULL;
3911         struct iscsi_session *sess = conn->sess;
3912         /*
3913          * We expect this function to only ever be called from either RX or TX
3914          * thread context via iscsit_close_connection() once the other context
3915          * has been reset -> returned sleeping pre-handler state.
3916          */
3917         spin_lock_bh(&conn->cmd_lock);
3918         list_for_each_entry_safe(cmd, cmd_tmp, &conn->conn_cmd_list, i_conn_node) {
3919
3920                 list_del(&cmd->i_conn_node);
3921                 spin_unlock_bh(&conn->cmd_lock);
3922
3923                 iscsit_increment_maxcmdsn(cmd, sess);
3924
3925                 iscsit_free_cmd(cmd);
3926
3927                 spin_lock_bh(&conn->cmd_lock);
3928         }
3929         spin_unlock_bh(&conn->cmd_lock);
3930 }
3931
3932 static void iscsit_stop_timers_for_cmds(
3933         struct iscsi_conn *conn)
3934 {
3935         struct iscsi_cmd *cmd;
3936
3937         spin_lock_bh(&conn->cmd_lock);
3938         list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
3939                 if (cmd->data_direction == DMA_TO_DEVICE)
3940                         iscsit_stop_dataout_timer(cmd);
3941         }
3942         spin_unlock_bh(&conn->cmd_lock);
3943 }
3944
3945 int iscsit_close_connection(
3946         struct iscsi_conn *conn)
3947 {
3948         int conn_logout = (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT);
3949         struct iscsi_session    *sess = conn->sess;
3950
3951         pr_debug("Closing iSCSI connection CID %hu on SID:"
3952                 " %u\n", conn->cid, sess->sid);
3953         /*
3954          * Always up conn_logout_comp just in case the RX Thread is sleeping
3955          * and the logout response never got sent because the connection
3956          * failed.
3957          */
3958         complete(&conn->conn_logout_comp);
3959
3960         iscsi_release_thread_set(conn);
3961
3962         iscsit_stop_timers_for_cmds(conn);
3963         iscsit_stop_nopin_response_timer(conn);
3964         iscsit_stop_nopin_timer(conn);
3965         iscsit_free_queue_reqs_for_conn(conn);
3966
3967         /*
3968          * During Connection recovery drop unacknowledged out of order
3969          * commands for this connection, and prepare the other commands
3970          * for realligence.
3971          *
3972          * During normal operation clear the out of order commands (but
3973          * do not free the struct iscsi_ooo_cmdsn's) and release all
3974          * struct iscsi_cmds.
3975          */
3976         if (atomic_read(&conn->connection_recovery)) {
3977                 iscsit_discard_unacknowledged_ooo_cmdsns_for_conn(conn);
3978                 iscsit_prepare_cmds_for_realligance(conn);
3979         } else {
3980                 iscsit_clear_ooo_cmdsns_for_conn(conn);
3981                 iscsit_release_commands_from_conn(conn);
3982         }
3983
3984         /*
3985          * Handle decrementing session or connection usage count if
3986          * a logout response was not able to be sent because the
3987          * connection failed.  Fall back to Session Recovery here.
3988          */
3989         if (atomic_read(&conn->conn_logout_remove)) {
3990                 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_SESSION) {
3991                         iscsit_dec_conn_usage_count(conn);
3992                         iscsit_dec_session_usage_count(sess);
3993                 }
3994                 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION)
3995                         iscsit_dec_conn_usage_count(conn);
3996
3997                 atomic_set(&conn->conn_logout_remove, 0);
3998                 atomic_set(&sess->session_reinstatement, 0);
3999                 atomic_set(&sess->session_fall_back_to_erl0, 1);
4000         }
4001
4002         spin_lock_bh(&sess->conn_lock);
4003         list_del(&conn->conn_list);
4004
4005         /*
4006          * Attempt to let the Initiator know this connection failed by
4007          * sending an Connection Dropped Async Message on another
4008          * active connection.
4009          */
4010         if (atomic_read(&conn->connection_recovery))
4011                 iscsit_build_conn_drop_async_message(conn);
4012
4013         spin_unlock_bh(&sess->conn_lock);
4014
4015         /*
4016          * If connection reinstatement is being performed on this connection,
4017          * up the connection reinstatement semaphore that is being blocked on
4018          * in iscsit_cause_connection_reinstatement().
4019          */
4020         spin_lock_bh(&conn->state_lock);
4021         if (atomic_read(&conn->sleep_on_conn_wait_comp)) {
4022                 spin_unlock_bh(&conn->state_lock);
4023                 complete(&conn->conn_wait_comp);
4024                 wait_for_completion(&conn->conn_post_wait_comp);
4025                 spin_lock_bh(&conn->state_lock);
4026         }
4027
4028         /*
4029          * If connection reinstatement is being performed on this connection
4030          * by receiving a REMOVECONNFORRECOVERY logout request, up the
4031          * connection wait rcfr semaphore that is being blocked on
4032          * an iscsit_connection_reinstatement_rcfr().
4033          */
4034         if (atomic_read(&conn->connection_wait_rcfr)) {
4035                 spin_unlock_bh(&conn->state_lock);
4036                 complete(&conn->conn_wait_rcfr_comp);
4037                 wait_for_completion(&conn->conn_post_wait_comp);
4038                 spin_lock_bh(&conn->state_lock);
4039         }
4040         atomic_set(&conn->connection_reinstatement, 1);
4041         spin_unlock_bh(&conn->state_lock);
4042
4043         /*
4044          * If any other processes are accessing this connection pointer we
4045          * must wait until they have completed.
4046          */
4047         iscsit_check_conn_usage_count(conn);
4048
4049         if (conn->conn_rx_hash.tfm)
4050                 crypto_free_hash(conn->conn_rx_hash.tfm);
4051         if (conn->conn_tx_hash.tfm)
4052                 crypto_free_hash(conn->conn_tx_hash.tfm);
4053
4054         if (conn->conn_cpumask)
4055                 free_cpumask_var(conn->conn_cpumask);
4056
4057         kfree(conn->conn_ops);
4058         conn->conn_ops = NULL;
4059
4060         if (conn->sock) {
4061                 if (conn->conn_flags & CONNFLAG_SCTP_STRUCT_FILE) {
4062                         kfree(conn->sock->file);
4063                         conn->sock->file = NULL;
4064                 }
4065                 sock_release(conn->sock);
4066         }
4067         conn->thread_set = NULL;
4068
4069         pr_debug("Moving to TARG_CONN_STATE_FREE.\n");
4070         conn->conn_state = TARG_CONN_STATE_FREE;
4071         kfree(conn);
4072
4073         spin_lock_bh(&sess->conn_lock);
4074         atomic_dec(&sess->nconn);
4075         pr_debug("Decremented iSCSI connection count to %hu from node:"
4076                 " %s\n", atomic_read(&sess->nconn),
4077                 sess->sess_ops->InitiatorName);
4078         /*
4079          * Make sure that if one connection fails in an non ERL=2 iSCSI
4080          * Session that they all fail.
4081          */
4082         if ((sess->sess_ops->ErrorRecoveryLevel != 2) && !conn_logout &&
4083              !atomic_read(&sess->session_logout))
4084                 atomic_set(&sess->session_fall_back_to_erl0, 1);
4085
4086         /*
4087          * If this was not the last connection in the session, and we are
4088          * performing session reinstatement or falling back to ERL=0, call
4089          * iscsit_stop_session() without sleeping to shutdown the other
4090          * active connections.
4091          */
4092         if (atomic_read(&sess->nconn)) {
4093                 if (!atomic_read(&sess->session_reinstatement) &&
4094                     !atomic_read(&sess->session_fall_back_to_erl0)) {
4095                         spin_unlock_bh(&sess->conn_lock);
4096                         return 0;
4097                 }
4098                 if (!atomic_read(&sess->session_stop_active)) {
4099                         atomic_set(&sess->session_stop_active, 1);
4100                         spin_unlock_bh(&sess->conn_lock);
4101                         iscsit_stop_session(sess, 0, 0);
4102                         return 0;
4103                 }
4104                 spin_unlock_bh(&sess->conn_lock);
4105                 return 0;
4106         }
4107
4108         /*
4109          * If this was the last connection in the session and one of the
4110          * following is occurring:
4111          *
4112          * Session Reinstatement is not being performed, and are falling back
4113          * to ERL=0 call iscsit_close_session().
4114          *
4115          * Session Logout was requested.  iscsit_close_session() will be called
4116          * elsewhere.
4117          *
4118          * Session Continuation is not being performed, start the Time2Retain
4119          * handler and check if sleep_on_sess_wait_sem is active.
4120          */
4121         if (!atomic_read(&sess->session_reinstatement) &&
4122              atomic_read(&sess->session_fall_back_to_erl0)) {
4123                 spin_unlock_bh(&sess->conn_lock);
4124                 target_put_session(sess->se_sess);
4125
4126                 return 0;
4127         } else if (atomic_read(&sess->session_logout)) {
4128                 pr_debug("Moving to TARG_SESS_STATE_FREE.\n");
4129                 sess->session_state = TARG_SESS_STATE_FREE;
4130                 spin_unlock_bh(&sess->conn_lock);
4131
4132                 if (atomic_read(&sess->sleep_on_sess_wait_comp))
4133                         complete(&sess->session_wait_comp);
4134
4135                 return 0;
4136         } else {
4137                 pr_debug("Moving to TARG_SESS_STATE_FAILED.\n");
4138                 sess->session_state = TARG_SESS_STATE_FAILED;
4139
4140                 if (!atomic_read(&sess->session_continuation)) {
4141                         spin_unlock_bh(&sess->conn_lock);
4142                         iscsit_start_time2retain_handler(sess);
4143                 } else
4144                         spin_unlock_bh(&sess->conn_lock);
4145
4146                 if (atomic_read(&sess->sleep_on_sess_wait_comp))
4147                         complete(&sess->session_wait_comp);
4148
4149                 return 0;
4150         }
4151         spin_unlock_bh(&sess->conn_lock);
4152
4153         return 0;
4154 }
4155
4156 int iscsit_close_session(struct iscsi_session *sess)
4157 {
4158         struct iscsi_portal_group *tpg = ISCSI_TPG_S(sess);
4159         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
4160
4161         if (atomic_read(&sess->nconn)) {
4162                 pr_err("%d connection(s) still exist for iSCSI session"
4163                         " to %s\n", atomic_read(&sess->nconn),
4164                         sess->sess_ops->InitiatorName);
4165                 BUG();
4166         }
4167
4168         spin_lock_bh(&se_tpg->session_lock);
4169         atomic_set(&sess->session_logout, 1);
4170         atomic_set(&sess->session_reinstatement, 1);
4171         iscsit_stop_time2retain_timer(sess);
4172         spin_unlock_bh(&se_tpg->session_lock);
4173
4174         /*
4175          * transport_deregister_session_configfs() will clear the
4176          * struct se_node_acl->nacl_sess pointer now as a iscsi_np process context
4177          * can be setting it again with __transport_register_session() in
4178          * iscsi_post_login_handler() again after the iscsit_stop_session()
4179          * completes in iscsi_np context.
4180          */
4181         transport_deregister_session_configfs(sess->se_sess);
4182
4183         /*
4184          * If any other processes are accessing this session pointer we must
4185          * wait until they have completed.  If we are in an interrupt (the
4186          * time2retain handler) and contain and active session usage count we
4187          * restart the timer and exit.
4188          */
4189         if (!in_interrupt()) {
4190                 if (iscsit_check_session_usage_count(sess) == 1)
4191                         iscsit_stop_session(sess, 1, 1);
4192         } else {
4193                 if (iscsit_check_session_usage_count(sess) == 2) {
4194                         atomic_set(&sess->session_logout, 0);
4195                         iscsit_start_time2retain_handler(sess);
4196                         return 0;
4197                 }
4198         }
4199
4200         transport_deregister_session(sess->se_sess);
4201
4202         if (sess->sess_ops->ErrorRecoveryLevel == 2)
4203                 iscsit_free_connection_recovery_entires(sess);
4204
4205         iscsit_free_all_ooo_cmdsns(sess);
4206
4207         spin_lock_bh(&se_tpg->session_lock);
4208         pr_debug("Moving to TARG_SESS_STATE_FREE.\n");
4209         sess->session_state = TARG_SESS_STATE_FREE;
4210         pr_debug("Released iSCSI session from node: %s\n",
4211                         sess->sess_ops->InitiatorName);
4212         tpg->nsessions--;
4213         if (tpg->tpg_tiqn)
4214                 tpg->tpg_tiqn->tiqn_nsessions--;
4215
4216         pr_debug("Decremented number of active iSCSI Sessions on"
4217                 " iSCSI TPG: %hu to %u\n", tpg->tpgt, tpg->nsessions);
4218
4219         spin_lock(&sess_idr_lock);
4220         idr_remove(&sess_idr, sess->session_index);
4221         spin_unlock(&sess_idr_lock);
4222
4223         kfree(sess->sess_ops);
4224         sess->sess_ops = NULL;
4225         spin_unlock_bh(&se_tpg->session_lock);
4226
4227         kfree(sess);
4228         return 0;
4229 }
4230
4231 static void iscsit_logout_post_handler_closesession(
4232         struct iscsi_conn *conn)
4233 {
4234         struct iscsi_session *sess = conn->sess;
4235
4236         iscsi_set_thread_clear(conn, ISCSI_CLEAR_TX_THREAD);
4237         iscsi_set_thread_set_signal(conn, ISCSI_SIGNAL_TX_THREAD);
4238
4239         atomic_set(&conn->conn_logout_remove, 0);
4240         complete(&conn->conn_logout_comp);
4241
4242         iscsit_dec_conn_usage_count(conn);
4243         iscsit_stop_session(sess, 1, 1);
4244         iscsit_dec_session_usage_count(sess);
4245         target_put_session(sess->se_sess);
4246 }
4247
4248 static void iscsit_logout_post_handler_samecid(
4249         struct iscsi_conn *conn)
4250 {
4251         iscsi_set_thread_clear(conn, ISCSI_CLEAR_TX_THREAD);
4252         iscsi_set_thread_set_signal(conn, ISCSI_SIGNAL_TX_THREAD);
4253
4254         atomic_set(&conn->conn_logout_remove, 0);
4255         complete(&conn->conn_logout_comp);
4256
4257         iscsit_cause_connection_reinstatement(conn, 1);
4258         iscsit_dec_conn_usage_count(conn);
4259 }
4260
4261 static void iscsit_logout_post_handler_diffcid(
4262         struct iscsi_conn *conn,
4263         u16 cid)
4264 {
4265         struct iscsi_conn *l_conn;
4266         struct iscsi_session *sess = conn->sess;
4267
4268         if (!sess)
4269                 return;
4270
4271         spin_lock_bh(&sess->conn_lock);
4272         list_for_each_entry(l_conn, &sess->sess_conn_list, conn_list) {
4273                 if (l_conn->cid == cid) {
4274                         iscsit_inc_conn_usage_count(l_conn);
4275                         break;
4276                 }
4277         }
4278         spin_unlock_bh(&sess->conn_lock);
4279
4280         if (!l_conn)
4281                 return;
4282
4283         if (l_conn->sock)
4284                 l_conn->sock->ops->shutdown(l_conn->sock, RCV_SHUTDOWN);
4285
4286         spin_lock_bh(&l_conn->state_lock);
4287         pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
4288         l_conn->conn_state = TARG_CONN_STATE_IN_LOGOUT;
4289         spin_unlock_bh(&l_conn->state_lock);
4290
4291         iscsit_cause_connection_reinstatement(l_conn, 1);
4292         iscsit_dec_conn_usage_count(l_conn);
4293 }
4294
4295 /*
4296  *      Return of 0 causes the TX thread to restart.
4297  */
4298 static int iscsit_logout_post_handler(
4299         struct iscsi_cmd *cmd,
4300         struct iscsi_conn *conn)
4301 {
4302         int ret = 0;
4303
4304         switch (cmd->logout_reason) {
4305         case ISCSI_LOGOUT_REASON_CLOSE_SESSION:
4306                 switch (cmd->logout_response) {
4307                 case ISCSI_LOGOUT_SUCCESS:
4308                 case ISCSI_LOGOUT_CLEANUP_FAILED:
4309                 default:
4310                         iscsit_logout_post_handler_closesession(conn);
4311                         break;
4312                 }
4313                 ret = 0;
4314                 break;
4315         case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION:
4316                 if (conn->cid == cmd->logout_cid) {
4317                         switch (cmd->logout_response) {
4318                         case ISCSI_LOGOUT_SUCCESS:
4319                         case ISCSI_LOGOUT_CLEANUP_FAILED:
4320                         default:
4321                                 iscsit_logout_post_handler_samecid(conn);
4322                                 break;
4323                         }
4324                         ret = 0;
4325                 } else {
4326                         switch (cmd->logout_response) {
4327                         case ISCSI_LOGOUT_SUCCESS:
4328                                 iscsit_logout_post_handler_diffcid(conn,
4329                                         cmd->logout_cid);
4330                                 break;
4331                         case ISCSI_LOGOUT_CID_NOT_FOUND:
4332                         case ISCSI_LOGOUT_CLEANUP_FAILED:
4333                         default:
4334                                 break;
4335                         }
4336                         ret = 1;
4337                 }
4338                 break;
4339         case ISCSI_LOGOUT_REASON_RECOVERY:
4340                 switch (cmd->logout_response) {
4341                 case ISCSI_LOGOUT_SUCCESS:
4342                 case ISCSI_LOGOUT_CID_NOT_FOUND:
4343                 case ISCSI_LOGOUT_RECOVERY_UNSUPPORTED:
4344                 case ISCSI_LOGOUT_CLEANUP_FAILED:
4345                 default:
4346                         break;
4347                 }
4348                 ret = 1;
4349                 break;
4350         default:
4351                 break;
4352
4353         }
4354         return ret;
4355 }
4356
4357 void iscsit_fail_session(struct iscsi_session *sess)
4358 {
4359         struct iscsi_conn *conn;
4360
4361         spin_lock_bh(&sess->conn_lock);
4362         list_for_each_entry(conn, &sess->sess_conn_list, conn_list) {
4363                 pr_debug("Moving to TARG_CONN_STATE_CLEANUP_WAIT.\n");
4364                 conn->conn_state = TARG_CONN_STATE_CLEANUP_WAIT;
4365         }
4366         spin_unlock_bh(&sess->conn_lock);
4367
4368         pr_debug("Moving to TARG_SESS_STATE_FAILED.\n");
4369         sess->session_state = TARG_SESS_STATE_FAILED;
4370 }
4371
4372 int iscsit_free_session(struct iscsi_session *sess)
4373 {
4374         u16 conn_count = atomic_read(&sess->nconn);
4375         struct iscsi_conn *conn, *conn_tmp = NULL;
4376         int is_last;
4377
4378         spin_lock_bh(&sess->conn_lock);
4379         atomic_set(&sess->sleep_on_sess_wait_comp, 1);
4380
4381         list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list,
4382                         conn_list) {
4383                 if (conn_count == 0)
4384                         break;
4385
4386                 if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) {
4387                         is_last = 1;
4388                 } else {
4389                         iscsit_inc_conn_usage_count(conn_tmp);
4390                         is_last = 0;
4391                 }
4392                 iscsit_inc_conn_usage_count(conn);
4393
4394                 spin_unlock_bh(&sess->conn_lock);
4395                 iscsit_cause_connection_reinstatement(conn, 1);
4396                 spin_lock_bh(&sess->conn_lock);
4397
4398                 iscsit_dec_conn_usage_count(conn);
4399                 if (is_last == 0)
4400                         iscsit_dec_conn_usage_count(conn_tmp);
4401
4402                 conn_count--;
4403         }
4404
4405         if (atomic_read(&sess->nconn)) {
4406                 spin_unlock_bh(&sess->conn_lock);
4407                 wait_for_completion(&sess->session_wait_comp);
4408         } else
4409                 spin_unlock_bh(&sess->conn_lock);
4410
4411         target_put_session(sess->se_sess);
4412         return 0;
4413 }
4414
4415 void iscsit_stop_session(
4416         struct iscsi_session *sess,
4417         int session_sleep,
4418         int connection_sleep)
4419 {
4420         u16 conn_count = atomic_read(&sess->nconn);
4421         struct iscsi_conn *conn, *conn_tmp = NULL;
4422         int is_last;
4423
4424         spin_lock_bh(&sess->conn_lock);
4425         if (session_sleep)
4426                 atomic_set(&sess->sleep_on_sess_wait_comp, 1);
4427
4428         if (connection_sleep) {
4429                 list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list,
4430                                 conn_list) {
4431                         if (conn_count == 0)
4432                                 break;
4433
4434                         if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) {
4435                                 is_last = 1;
4436                         } else {
4437                                 iscsit_inc_conn_usage_count(conn_tmp);
4438                                 is_last = 0;
4439                         }
4440                         iscsit_inc_conn_usage_count(conn);
4441
4442                         spin_unlock_bh(&sess->conn_lock);
4443                         iscsit_cause_connection_reinstatement(conn, 1);
4444                         spin_lock_bh(&sess->conn_lock);
4445
4446                         iscsit_dec_conn_usage_count(conn);
4447                         if (is_last == 0)
4448                                 iscsit_dec_conn_usage_count(conn_tmp);
4449                         conn_count--;
4450                 }
4451         } else {
4452                 list_for_each_entry(conn, &sess->sess_conn_list, conn_list)
4453                         iscsit_cause_connection_reinstatement(conn, 0);
4454         }
4455
4456         if (session_sleep && atomic_read(&sess->nconn)) {
4457                 spin_unlock_bh(&sess->conn_lock);
4458                 wait_for_completion(&sess->session_wait_comp);
4459         } else
4460                 spin_unlock_bh(&sess->conn_lock);
4461 }
4462
4463 int iscsit_release_sessions_for_tpg(struct iscsi_portal_group *tpg, int force)
4464 {
4465         struct iscsi_session *sess;
4466         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
4467         struct se_session *se_sess, *se_sess_tmp;
4468         int session_count = 0;
4469
4470         spin_lock_bh(&se_tpg->session_lock);
4471         if (tpg->nsessions && !force) {
4472                 spin_unlock_bh(&se_tpg->session_lock);
4473                 return -1;
4474         }
4475
4476         list_for_each_entry_safe(se_sess, se_sess_tmp, &se_tpg->tpg_sess_list,
4477                         sess_list) {
4478                 sess = (struct iscsi_session *)se_sess->fabric_sess_ptr;
4479
4480                 spin_lock(&sess->conn_lock);
4481                 if (atomic_read(&sess->session_fall_back_to_erl0) ||
4482                     atomic_read(&sess->session_logout) ||
4483                     (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)) {
4484                         spin_unlock(&sess->conn_lock);
4485                         continue;
4486                 }
4487                 atomic_set(&sess->session_reinstatement, 1);
4488                 spin_unlock(&sess->conn_lock);
4489                 spin_unlock_bh(&se_tpg->session_lock);
4490
4491                 iscsit_free_session(sess);
4492                 spin_lock_bh(&se_tpg->session_lock);
4493
4494                 session_count++;
4495         }
4496         spin_unlock_bh(&se_tpg->session_lock);
4497
4498         pr_debug("Released %d iSCSI Session(s) from Target Portal"
4499                         " Group: %hu\n", session_count, tpg->tpgt);
4500         return 0;
4501 }
4502
4503 MODULE_DESCRIPTION("iSCSI-Target Driver for mainline target infrastructure");
4504 MODULE_VERSION("4.1.x");
4505 MODULE_AUTHOR("nab@Linux-iSCSI.org");
4506 MODULE_LICENSE("GPL");
4507
4508 module_init(iscsi_target_init_module);
4509 module_exit(iscsi_target_cleanup_module);