drivers/target/iscsi/iscsi_target_erl0.c

   1 /******************************************************************************
   2  * This file contains error recovery level zero functions used by
   3  * the iSCSI Target driver.
   4  *
   5  * \u00a9 Copyright 2007-2011 RisingTide Systems LLC.
   6  *
   7  * Licensed to the Linux Foundation under the General Public License (GPL) version 2.
   8  *
   9  * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
  10  *
  11  * This program is free software; you can redistribute it and/or modify
  12  * it under the terms of the GNU General Public License as published by
  13  * the Free Software Foundation; either version 2 of the License, or
  14  * (at your option) any later version.
  15  *
  16  * This program is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19  * GNU General Public License for more details.
  20  ******************************************************************************/
  21
  22 #include <scsi/iscsi_proto.h>
  23 #include <target/target_core_base.h>
  24 #include <target/target_core_fabric.h>
  25
  26 #include "iscsi_target_core.h"
  27 #include "iscsi_target_seq_pdu_list.h"
  28 #include "iscsi_target_tq.h"
  29 #include "iscsi_target_erl0.h"
  30 #include "iscsi_target_erl1.h"
  31 #include "iscsi_target_erl2.h"
  32 #include "iscsi_target_util.h"
  33 #include "iscsi_target.h"
  34
  35 /*
  36  *      Used to set values in struct iscsi_cmd that iscsit_dataout_check_sequence()
  37  *      checks against to determine a PDU's Offset+Length is within the current
  38  *      DataOUT Sequence.  Used for DataSequenceInOrder=Yes only.
  39  */
  40 void iscsit_set_dataout_sequence_values(
  41         struct iscsi_cmd *cmd)
  42 {
  43         struct iscsi_conn *conn = cmd->conn;
  44         /*
  45          * Still set seq_start_offset and seq_end_offset for Unsolicited
  46          * DataOUT, even if DataSequenceInOrder=No.
  47          */
  48         if (cmd->unsolicited_data) {
  49                 cmd->seq_start_offset = cmd->write_data_done;
  50                 cmd->seq_end_offset = (cmd->write_data_done +
  51                         ((cmd->se_cmd.data_length >
  52                           conn->sess->sess_ops->FirstBurstLength) ?
  53                          conn->sess->sess_ops->FirstBurstLength : cmd->se_cmd.data_length));
  54                 return;
  55         }
  56
  57         if (!conn->sess->sess_ops->DataSequenceInOrder)
  58                 return;
  59
  60         if (!cmd->seq_start_offset && !cmd->seq_end_offset) {
  61                 cmd->seq_start_offset = cmd->write_data_done;
  62                 cmd->seq_end_offset = (cmd->se_cmd.data_length >
  63                         conn->sess->sess_ops->MaxBurstLength) ?
  64                         (cmd->write_data_done +
  65                         conn->sess->sess_ops->MaxBurstLength) : cmd->se_cmd.data_length;
  66         } else {
  67                 cmd->seq_start_offset = cmd->seq_end_offset;
  68                 cmd->seq_end_offset = ((cmd->seq_end_offset +
  69                         conn->sess->sess_ops->MaxBurstLength) >=
  70                         cmd->se_cmd.data_length) ? cmd->se_cmd.data_length :
  71                         (cmd->seq_end_offset +
  72                          conn->sess->sess_ops->MaxBurstLength);
  73         }
  74 }
  75
  76 static int iscsit_dataout_within_command_recovery_check(
  77         struct iscsi_cmd *cmd,
  78         unsigned char *buf)
  79 {
  80         struct iscsi_conn *conn = cmd->conn;
  81         struct iscsi_data *hdr = (struct iscsi_data *) buf;
  82         u32 payload_length = ntoh24(hdr->dlength);
  83
  84         /*
  85          * We do the within-command recovery checks here as it is
  86          * the first function called in iscsi_check_pre_dataout().
  87          * Basically, if we are in within-command recovery and
  88          * the PDU does not contain the offset the sequence needs,
  89          * dump the payload.
  90          *
  91          * This only applies to DataPDUInOrder=Yes, for
  92          * DataPDUInOrder=No we only re-request the failed PDU
  93          * and check that all PDUs in a sequence are received
  94          * upon end of sequence.
  95          */
  96         if (conn->sess->sess_ops->DataSequenceInOrder) {
  97                 if ((cmd->cmd_flags & ICF_WITHIN_COMMAND_RECOVERY) &&
  98                     cmd->write_data_done != be32_to_cpu(hdr->offset))
  99                         goto dump;
 100
 101                 cmd->cmd_flags &= ~ICF_WITHIN_COMMAND_RECOVERY;
 102         } else {
 103                 struct iscsi_seq *seq;
 104
 105                 seq = iscsit_get_seq_holder(cmd, be32_to_cpu(hdr->offset),
 106                                             payload_length);
 107                 if (!seq)
 108                         return DATAOUT_CANNOT_RECOVER;
 109                 /*
 110                  * Set the struct iscsi_seq pointer to reuse later.
 111                  */
 112                 cmd->seq_ptr = seq;
 113
 114                 if (conn->sess->sess_ops->DataPDUInOrder) {
 115                         if (seq->status ==
 116                             DATAOUT_SEQUENCE_WITHIN_COMMAND_RECOVERY &&
 117                            (seq->offset != be32_to_cpu(hdr->offset) ||
 118                             seq->data_sn != be32_to_cpu(hdr->datasn)))
 119                                 goto dump;
 120                 } else {
 121                         if (seq->status ==
 122                              DATAOUT_SEQUENCE_WITHIN_COMMAND_RECOVERY &&
 123                             seq->data_sn != be32_to_cpu(hdr->datasn))
 124                                 goto dump;
 125                 }
 126
 127                 if (seq->status == DATAOUT_SEQUENCE_COMPLETE)
 128                         goto dump;
 129
 130                 if (seq->status != DATAOUT_SEQUENCE_COMPLETE)
 131                         seq->status = 0;
 132         }
 133
 134         return DATAOUT_NORMAL;
 135
 136 dump:
 137         pr_err("Dumping DataOUT PDU Offset: %u Length: %d DataSN:"
 138                 " 0x%08x\n", hdr->offset, payload_length, hdr->datasn);
 139         return iscsit_dump_data_payload(conn, payload_length, 1);
 140 }
 141
 142 static int iscsit_dataout_check_unsolicited_sequence(
 143         struct iscsi_cmd *cmd,
 144         unsigned char *buf)
 145 {
 146         u32 first_burst_len;
 147         struct iscsi_conn *conn = cmd->conn;
 148         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 149         u32 payload_length = ntoh24(hdr->dlength);
 150
 151
 152         if ((be32_to_cpu(hdr->offset) < cmd->seq_start_offset) ||
 153            ((be32_to_cpu(hdr->offset) + payload_length) > cmd->seq_end_offset)) {
 154                 pr_err("Command ITT: 0x%08x with Offset: %u,"
 155                 " Length: %u outside of Unsolicited Sequence %u:%u while"
 156                 " DataSequenceInOrder=Yes.\n", cmd->init_task_tag,
 157                 be32_to_cpu(hdr->offset), payload_length, cmd->seq_start_offset,
 158                         cmd->seq_end_offset);
 159                 return DATAOUT_CANNOT_RECOVER;
 160         }
 161
 162         first_burst_len = (cmd->first_burst_len + payload_length);
 163
 164         if (first_burst_len > conn->sess->sess_ops->FirstBurstLength) {
 165                 pr_err("Total %u bytes exceeds FirstBurstLength: %u"
 166                         " for this Unsolicited DataOut Burst.\n",
 167                         first_burst_len, conn->sess->sess_ops->FirstBurstLength);
 168                 transport_send_check_condition_and_sense(&cmd->se_cmd,
 169                                 TCM_INCORRECT_AMOUNT_OF_DATA, 0);
 170                 return DATAOUT_CANNOT_RECOVER;
 171         }
 172
 173         /*
 174          * Perform various MaxBurstLength and ISCSI_FLAG_CMD_FINAL sanity
 175          * checks for the current Unsolicited DataOUT Sequence.
 176          */
 177         if (hdr->flags & ISCSI_FLAG_CMD_FINAL) {
 178                 /*
 179                  * Ignore ISCSI_FLAG_CMD_FINAL checks while DataPDUInOrder=No, end of
 180                  * sequence checks are handled in
 181                  * iscsit_dataout_datapduinorder_no_fbit().
 182                  */
 183                 if (!conn->sess->sess_ops->DataPDUInOrder)
 184                         goto out;
 185
 186                 if ((first_burst_len != cmd->se_cmd.data_length) &&
 187                     (first_burst_len != conn->sess->sess_ops->FirstBurstLength)) {
 188                         pr_err("Unsolicited non-immediate data"
 189                         " received %u does not equal FirstBurstLength: %u, and"
 190                         " does not equal ExpXferLen %u.\n", first_burst_len,
 191                                 conn->sess->sess_ops->FirstBurstLength,
 192                                 cmd->se_cmd.data_length);
 193                         transport_send_check_condition_and_sense(&cmd->se_cmd,
 194                                         TCM_INCORRECT_AMOUNT_OF_DATA, 0);
 195                         return DATAOUT_CANNOT_RECOVER;
 196                 }
 197         } else {
 198                 if (first_burst_len == conn->sess->sess_ops->FirstBurstLength) {
 199                         pr_err("Command ITT: 0x%08x reached"
 200                         " FirstBurstLength: %u, but ISCSI_FLAG_CMD_FINAL is not set. protocol"
 201                                 " error.\n", cmd->init_task_tag,
 202                                 conn->sess->sess_ops->FirstBurstLength);
 203                         return DATAOUT_CANNOT_RECOVER;
 204                 }
 205                 if (first_burst_len == cmd->se_cmd.data_length) {
 206                         pr_err("Command ITT: 0x%08x reached"
 207                         " ExpXferLen: %u, but ISCSI_FLAG_CMD_FINAL is not set. protocol"
 208                         " error.\n", cmd->init_task_tag, cmd->se_cmd.data_length);
 209                         return DATAOUT_CANNOT_RECOVER;
 210                 }
 211         }
 212
 213 out:
 214         return DATAOUT_NORMAL;
 215 }
 216
 217 static int iscsit_dataout_check_sequence(
 218         struct iscsi_cmd *cmd,
 219         unsigned char *buf)
 220 {
 221         u32 next_burst_len;
 222         struct iscsi_conn *conn = cmd->conn;
 223         struct iscsi_seq *seq = NULL;
 224         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 225         u32 payload_length = ntoh24(hdr->dlength);
 226
 227         /*
 228          * For DataSequenceInOrder=Yes: Check that the offset and offset+length
 229          * is within range as defined by iscsi_set_dataout_sequence_values().
 230          *
 231          * For DataSequenceInOrder=No: Check that an struct iscsi_seq exists for
 232          * offset+length tuple.
 233          */
 234         if (conn->sess->sess_ops->DataSequenceInOrder) {
 235                 /*
 236                  * Due to possibility of recovery DataOUT sent by the initiator
 237                  * fullfilling an Recovery R2T, it's best to just dump the
 238                  * payload here, instead of erroring out.
 239                  */
 240                 if ((be32_to_cpu(hdr->offset) < cmd->seq_start_offset) ||
 241                    ((be32_to_cpu(hdr->offset) + payload_length) > cmd->seq_end_offset)) {
 242                         pr_err("Command ITT: 0x%08x with Offset: %u,"
 243                         " Length: %u outside of Sequence %u:%u while"
 244                         " DataSequenceInOrder=Yes.\n", cmd->init_task_tag,
 245                         be32_to_cpu(hdr->offset), payload_length, cmd->seq_start_offset,
 246                                 cmd->seq_end_offset);
 247
 248                         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 249                                 return DATAOUT_CANNOT_RECOVER;
 250                         return DATAOUT_WITHIN_COMMAND_RECOVERY;
 251                 }
 252
 253                 next_burst_len = (cmd->next_burst_len + payload_length);
 254         } else {
 255                 seq = iscsit_get_seq_holder(cmd, be32_to_cpu(hdr->offset),
 256                                             payload_length);
 257                 if (!seq)
 258                         return DATAOUT_CANNOT_RECOVER;
 259                 /*
 260                  * Set the struct iscsi_seq pointer to reuse later.
 261                  */
 262                 cmd->seq_ptr = seq;
 263
 264                 if (seq->status == DATAOUT_SEQUENCE_COMPLETE) {
 265                         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 266                                 return DATAOUT_CANNOT_RECOVER;
 267                         return DATAOUT_WITHIN_COMMAND_RECOVERY;
 268                 }
 269
 270                 next_burst_len = (seq->next_burst_len + payload_length);
 271         }
 272
 273         if (next_burst_len > conn->sess->sess_ops->MaxBurstLength) {
 274                 pr_err("Command ITT: 0x%08x, NextBurstLength: %u and"
 275                         " Length: %u exceeds MaxBurstLength: %u. protocol"
 276                         " error.\n", cmd->init_task_tag,
 277                         (next_burst_len - payload_length),
 278                         payload_length, conn->sess->sess_ops->MaxBurstLength);
 279                 return DATAOUT_CANNOT_RECOVER;
 280         }
 281
 282         /*
 283          * Perform various MaxBurstLength and ISCSI_FLAG_CMD_FINAL sanity
 284          * checks for the current DataOUT Sequence.
 285          */
 286         if (hdr->flags & ISCSI_FLAG_CMD_FINAL) {
 287                 /*
 288                  * Ignore ISCSI_FLAG_CMD_FINAL checks while DataPDUInOrder=No, end of
 289                  * sequence checks are handled in
 290                  * iscsit_dataout_datapduinorder_no_fbit().
 291                  */
 292                 if (!conn->sess->sess_ops->DataPDUInOrder)
 293                         goto out;
 294
 295                 if (conn->sess->sess_ops->DataSequenceInOrder) {
 296                         if ((next_burst_len <
 297                              conn->sess->sess_ops->MaxBurstLength) &&
 298                            ((cmd->write_data_done + payload_length) <
 299                              cmd->se_cmd.data_length)) {
 300                                 pr_err("Command ITT: 0x%08x set ISCSI_FLAG_CMD_FINAL"
 301                                 " before end of DataOUT sequence, protocol"
 302                                 " error.\n", cmd->init_task_tag);
 303                                 return DATAOUT_CANNOT_RECOVER;
 304                         }
 305                 } else {
 306                         if (next_burst_len < seq->xfer_len) {
 307                                 pr_err("Command ITT: 0x%08x set ISCSI_FLAG_CMD_FINAL"
 308                                 " before end of DataOUT sequence, protocol"
 309                                 " error.\n", cmd->init_task_tag);
 310                                 return DATAOUT_CANNOT_RECOVER;
 311                         }
 312                 }
 313         } else {
 314                 if (conn->sess->sess_ops->DataSequenceInOrder) {
 315                         if (next_burst_len ==
 316                                         conn->sess->sess_ops->MaxBurstLength) {
 317                                 pr_err("Command ITT: 0x%08x reached"
 318                                 " MaxBurstLength: %u, but ISCSI_FLAG_CMD_FINAL is"
 319                                 " not set, protocol error.", cmd->init_task_tag,
 320                                         conn->sess->sess_ops->MaxBurstLength);
 321                                 return DATAOUT_CANNOT_RECOVER;
 322                         }
 323                         if ((cmd->write_data_done + payload_length) ==
 324                                         cmd->se_cmd.data_length) {
 325                                 pr_err("Command ITT: 0x%08x reached"
 326                                 " last DataOUT PDU in sequence but ISCSI_FLAG_"
 327                                 "CMD_FINAL is not set, protocol error.\n",
 328                                         cmd->init_task_tag);
 329                                 return DATAOUT_CANNOT_RECOVER;
 330                         }
 331                 } else {
 332                         if (next_burst_len == seq->xfer_len) {
 333                                 pr_err("Command ITT: 0x%08x reached"
 334                                 " last DataOUT PDU in sequence but ISCSI_FLAG_"
 335                                 "CMD_FINAL is not set, protocol error.\n",
 336                                         cmd->init_task_tag);
 337                                 return DATAOUT_CANNOT_RECOVER;
 338                         }
 339                 }
 340         }
 341
 342 out:
 343         return DATAOUT_NORMAL;
 344 }
 345
 346 static int iscsit_dataout_check_datasn(
 347         struct iscsi_cmd *cmd,
 348         unsigned char *buf)
 349 {
 350         int dump = 0, recovery = 0;
 351         u32 data_sn = 0;
 352         struct iscsi_conn *conn = cmd->conn;
 353         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 354         u32 payload_length = ntoh24(hdr->dlength);
 355
 356         /*
 357          * Considering the target has no method of re-requesting DataOUT
 358          * by DataSN, if we receieve a greater DataSN than expected we
 359          * assume the functions for DataPDUInOrder=[Yes,No] below will
 360          * handle it.
 361          *
 362          * If the DataSN is less than expected, dump the payload.
 363          */
 364         if (conn->sess->sess_ops->DataSequenceInOrder)
 365                 data_sn = cmd->data_sn;
 366         else {
 367                 struct iscsi_seq *seq = cmd->seq_ptr;
 368                 data_sn = seq->data_sn;
 369         }
 370
 371         if (be32_to_cpu(hdr->datasn) > data_sn) {
 372                 pr_err("Command ITT: 0x%08x, received DataSN: 0x%08x"
 373                         " higher than expected 0x%08x.\n", cmd->init_task_tag,
 374                                 be32_to_cpu(hdr->datasn), data_sn);
 375                 recovery = 1;
 376                 goto recover;
 377         } else if (be32_to_cpu(hdr->datasn) < data_sn) {
 378                 pr_err("Command ITT: 0x%08x, received DataSN: 0x%08x"
 379                         " lower than expected 0x%08x, discarding payload.\n",
 380                         cmd->init_task_tag, be32_to_cpu(hdr->datasn), data_sn);
 381                 dump = 1;
 382                 goto dump;
 383         }
 384
 385         return DATAOUT_NORMAL;
 386
 387 recover:
 388         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 389                 pr_err("Unable to perform within-command recovery"
 390                                 " while ERL=0.\n");
 391                 return DATAOUT_CANNOT_RECOVER;
 392         }
 393 dump:
 394         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 395                 return DATAOUT_CANNOT_RECOVER;
 396
 397         return (recovery || dump) ? DATAOUT_WITHIN_COMMAND_RECOVERY :
 398                                 DATAOUT_NORMAL;
 399 }
 400
 401 static int iscsit_dataout_pre_datapduinorder_yes(
 402         struct iscsi_cmd *cmd,
 403         unsigned char *buf)
 404 {
 405         int dump = 0, recovery = 0;
 406         struct iscsi_conn *conn = cmd->conn;
 407         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 408         u32 payload_length = ntoh24(hdr->dlength);
 409
 410         /*
 411          * For DataSequenceInOrder=Yes: If the offset is greater than the global
 412          * DataPDUInOrder=Yes offset counter in struct iscsi_cmd a protcol error has
 413          * occurred and fail the connection.
 414          *
 415          * For DataSequenceInOrder=No: If the offset is greater than the per
 416          * sequence DataPDUInOrder=Yes offset counter in struct iscsi_seq a protocol
 417          * error has occurred and fail the connection.
 418          */
 419         if (conn->sess->sess_ops->DataSequenceInOrder) {
 420                 if (be32_to_cpu(hdr->offset) != cmd->write_data_done) {
 421                         pr_err("Command ITT: 0x%08x, received offset"
 422                         " %u different than expected %u.\n", cmd->init_task_tag,
 423                                 be32_to_cpu(hdr->offset), cmd->write_data_done);
 424                         recovery = 1;
 425                         goto recover;
 426                 }
 427         } else {
 428                 struct iscsi_seq *seq = cmd->seq_ptr;
 429
 430                 if (be32_to_cpu(hdr->offset) > seq->offset) {
 431                         pr_err("Command ITT: 0x%08x, received offset"
 432                         " %u greater than expected %u.\n", cmd->init_task_tag,
 433                                 be32_to_cpu(hdr->offset), seq->offset);
 434                         recovery = 1;
 435                         goto recover;
 436                 } else if (be32_to_cpu(hdr->offset) < seq->offset) {
 437                         pr_err("Command ITT: 0x%08x, received offset"
 438                         " %u less than expected %u, discarding payload.\n",
 439                                 cmd->init_task_tag, be32_to_cpu(hdr->offset),
 440                                 seq->offset);
 441                         dump = 1;
 442                         goto dump;
 443                 }
 444         }
 445
 446         return DATAOUT_NORMAL;
 447
 448 recover:
 449         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 450                 pr_err("Unable to perform within-command recovery"
 451                                 " while ERL=0.\n");
 452                 return DATAOUT_CANNOT_RECOVER;
 453         }
 454 dump:
 455         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 456                 return DATAOUT_CANNOT_RECOVER;
 457
 458         return (recovery) ? iscsit_recover_dataout_sequence(cmd,
 459                 be32_to_cpu(hdr->offset), payload_length) :
 460                (dump) ? DATAOUT_WITHIN_COMMAND_RECOVERY : DATAOUT_NORMAL;
 461 }
 462
 463 static int iscsit_dataout_pre_datapduinorder_no(
 464         struct iscsi_cmd *cmd,
 465         unsigned char *buf)
 466 {
 467         struct iscsi_pdu *pdu;
 468         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 469         u32 payload_length = ntoh24(hdr->dlength);
 470
 471         pdu = iscsit_get_pdu_holder(cmd, be32_to_cpu(hdr->offset),
 472                                     payload_length);
 473         if (!pdu)
 474                 return DATAOUT_CANNOT_RECOVER;
 475
 476         cmd->pdu_ptr = pdu;
 477
 478         switch (pdu->status) {
 479         case ISCSI_PDU_NOT_RECEIVED:
 480         case ISCSI_PDU_CRC_FAILED:
 481         case ISCSI_PDU_TIMED_OUT:
 482                 break;
 483         case ISCSI_PDU_RECEIVED_OK:
 484                 pr_err("Command ITT: 0x%08x received already gotten"
 485                         " Offset: %u, Length: %u\n", cmd->init_task_tag,
 486                                 be32_to_cpu(hdr->offset), payload_length);
 487                 return iscsit_dump_data_payload(cmd->conn, payload_length, 1);
 488         default:
 489                 return DATAOUT_CANNOT_RECOVER;
 490         }
 491
 492         return DATAOUT_NORMAL;
 493 }
 494
 495 static int iscsit_dataout_update_r2t(struct iscsi_cmd *cmd, u32 offset, u32 length)
 496 {
 497         struct iscsi_r2t *r2t;
 498
 499         if (cmd->unsolicited_data)
 500                 return 0;
 501
 502         r2t = iscsit_get_r2t_for_eos(cmd, offset, length);
 503         if (!r2t)
 504                 return -1;
 505
 506         spin_lock_bh(&cmd->r2t_lock);
 507         r2t->seq_complete = 1;
 508         cmd->outstanding_r2ts--;
 509         spin_unlock_bh(&cmd->r2t_lock);
 510
 511         return 0;
 512 }
 513
 514 static int iscsit_dataout_update_datapduinorder_no(
 515         struct iscsi_cmd *cmd,
 516         u32 data_sn,
 517         int f_bit)
 518 {
 519         int ret = 0;
 520         struct iscsi_pdu *pdu = cmd->pdu_ptr;
 521
 522         pdu->data_sn = data_sn;
 523
 524         switch (pdu->status) {
 525         case ISCSI_PDU_NOT_RECEIVED:
 526                 pdu->status = ISCSI_PDU_RECEIVED_OK;
 527                 break;
 528         case ISCSI_PDU_CRC_FAILED:
 529                 pdu->status = ISCSI_PDU_RECEIVED_OK;
 530                 break;
 531         case ISCSI_PDU_TIMED_OUT:
 532                 pdu->status = ISCSI_PDU_RECEIVED_OK;
 533                 break;
 534         default:
 535                 return DATAOUT_CANNOT_RECOVER;
 536         }
 537
 538         if (f_bit) {
 539                 ret = iscsit_dataout_datapduinorder_no_fbit(cmd, pdu);
 540                 if (ret == DATAOUT_CANNOT_RECOVER)
 541                         return ret;
 542         }
 543
 544         return DATAOUT_NORMAL;
 545 }
 546
 547 static int iscsit_dataout_post_crc_passed(
 548         struct iscsi_cmd *cmd,
 549         unsigned char *buf)
 550 {
 551         int ret, send_r2t = 0;
 552         struct iscsi_conn *conn = cmd->conn;
 553         struct iscsi_seq *seq = NULL;
 554         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 555         u32 payload_length = ntoh24(hdr->dlength);
 556
 557         if (cmd->unsolicited_data) {
 558                 if ((cmd->first_burst_len + payload_length) ==
 559                      conn->sess->sess_ops->FirstBurstLength) {
 560                         if (iscsit_dataout_update_r2t(cmd, be32_to_cpu(hdr->offset),
 561                                         payload_length) < 0)
 562                                 return DATAOUT_CANNOT_RECOVER;
 563                         send_r2t = 1;
 564                 }
 565
 566                 if (!conn->sess->sess_ops->DataPDUInOrder) {
 567                         ret = iscsit_dataout_update_datapduinorder_no(cmd,
 568                                 be32_to_cpu(hdr->datasn),
 569                                 (hdr->flags & ISCSI_FLAG_CMD_FINAL));
 570                         if (ret == DATAOUT_CANNOT_RECOVER)
 571                                 return ret;
 572                 }
 573
 574                 cmd->first_burst_len += payload_length;
 575
 576                 if (conn->sess->sess_ops->DataSequenceInOrder)
 577                         cmd->data_sn++;
 578                 else {
 579                         seq = cmd->seq_ptr;
 580                         seq->data_sn++;
 581                         seq->offset += payload_length;
 582                 }
 583
 584                 if (send_r2t) {
 585                         if (seq)
 586                                 seq->status = DATAOUT_SEQUENCE_COMPLETE;
 587                         cmd->first_burst_len = 0;
 588                         cmd->unsolicited_data = 0;
 589                 }
 590         } else {
 591                 if (conn->sess->sess_ops->DataSequenceInOrder) {
 592                         if ((cmd->next_burst_len + payload_length) ==
 593                              conn->sess->sess_ops->MaxBurstLength) {
 594                                 if (iscsit_dataout_update_r2t(cmd,
 595                                                 be32_to_cpu(hdr->offset),
 596                                                 payload_length) < 0)
 597                                         return DATAOUT_CANNOT_RECOVER;
 598                                 send_r2t = 1;
 599                         }
 600
 601                         if (!conn->sess->sess_ops->DataPDUInOrder) {
 602                                 ret = iscsit_dataout_update_datapduinorder_no(
 603                                                 cmd, be32_to_cpu(hdr->datasn),
 604                                                 (hdr->flags & ISCSI_FLAG_CMD_FINAL));
 605                                 if (ret == DATAOUT_CANNOT_RECOVER)
 606                                         return ret;
 607                         }
 608
 609                         cmd->next_burst_len += payload_length;
 610                         cmd->data_sn++;
 611
 612                         if (send_r2t)
 613                                 cmd->next_burst_len = 0;
 614                 } else {
 615                         seq = cmd->seq_ptr;
 616
 617                         if ((seq->next_burst_len + payload_length) ==
 618                              seq->xfer_len) {
 619                                 if (iscsit_dataout_update_r2t(cmd,
 620                                                 be32_to_cpu(hdr->offset),
 621                                                 payload_length) < 0)
 622                                         return DATAOUT_CANNOT_RECOVER;
 623                                 send_r2t = 1;
 624                         }
 625
 626                         if (!conn->sess->sess_ops->DataPDUInOrder) {
 627                                 ret = iscsit_dataout_update_datapduinorder_no(
 628                                                 cmd, be32_to_cpu(hdr->datasn),
 629                                                 (hdr->flags & ISCSI_FLAG_CMD_FINAL));
 630                                 if (ret == DATAOUT_CANNOT_RECOVER)
 631                                         return ret;
 632                         }
 633
 634                         seq->data_sn++;
 635                         seq->offset += payload_length;
 636                         seq->next_burst_len += payload_length;
 637
 638                         if (send_r2t) {
 639                                 seq->next_burst_len = 0;
 640                                 seq->status = DATAOUT_SEQUENCE_COMPLETE;
 641                         }
 642                 }
 643         }
 644
 645         if (send_r2t && conn->sess->sess_ops->DataSequenceInOrder)
 646                 cmd->data_sn = 0;
 647
 648         cmd->write_data_done += payload_length;
 649
 650         if (cmd->write_data_done == cmd->se_cmd.data_length)
 651                 return DATAOUT_SEND_TO_TRANSPORT;
 652         else if (send_r2t)
 653                 return DATAOUT_SEND_R2T;
 654         else
 655                 return DATAOUT_NORMAL;
 656 }
 657
 658 static int iscsit_dataout_post_crc_failed(
 659         struct iscsi_cmd *cmd,
 660         unsigned char *buf)
 661 {
 662         struct iscsi_conn *conn = cmd->conn;
 663         struct iscsi_pdu *pdu;
 664         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 665         u32 payload_length = ntoh24(hdr->dlength);
 666
 667         if (conn->sess->sess_ops->DataPDUInOrder)
 668                 goto recover;
 669         /*
 670          * The rest of this function is only called when DataPDUInOrder=No.
 671          */
 672         pdu = cmd->pdu_ptr;
 673
 674         switch (pdu->status) {
 675         case ISCSI_PDU_NOT_RECEIVED:
 676                 pdu->status = ISCSI_PDU_CRC_FAILED;
 677                 break;
 678         case ISCSI_PDU_CRC_FAILED:
 679                 break;
 680         case ISCSI_PDU_TIMED_OUT:
 681                 pdu->status = ISCSI_PDU_CRC_FAILED;
 682                 break;
 683         default:
 684                 return DATAOUT_CANNOT_RECOVER;
 685         }
 686
 687 recover:
 688         return iscsit_recover_dataout_sequence(cmd, be32_to_cpu(hdr->offset),
 689                                                 payload_length);
 690 }
 691
 692 /*
 693  *      Called from iscsit_handle_data_out() before DataOUT Payload is received
 694  *      and CRC computed.
 695  */
 696 int iscsit_check_pre_dataout(
 697         struct iscsi_cmd *cmd,
 698         unsigned char *buf)
 699 {
 700         int ret;
 701         struct iscsi_conn *conn = cmd->conn;
 702
 703         ret = iscsit_dataout_within_command_recovery_check(cmd, buf);
 704         if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 705             (ret == DATAOUT_CANNOT_RECOVER))
 706                 return ret;
 707
 708         ret = iscsit_dataout_check_datasn(cmd, buf);
 709         if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 710             (ret == DATAOUT_CANNOT_RECOVER))
 711                 return ret;
 712
 713         if (cmd->unsolicited_data) {
 714                 ret = iscsit_dataout_check_unsolicited_sequence(cmd, buf);
 715                 if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 716                     (ret == DATAOUT_CANNOT_RECOVER))
 717                         return ret;
 718         } else {
 719                 ret = iscsit_dataout_check_sequence(cmd, buf);
 720                 if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 721                     (ret == DATAOUT_CANNOT_RECOVER))
 722                         return ret;
 723         }
 724
 725         return (conn->sess->sess_ops->DataPDUInOrder) ?
 726                 iscsit_dataout_pre_datapduinorder_yes(cmd, buf) :
 727                 iscsit_dataout_pre_datapduinorder_no(cmd, buf);
 728 }
 729
 730 /*
 731  *      Called from iscsit_handle_data_out() after DataOUT Payload is received
 732  *      and CRC computed.
 733  */
 734 int iscsit_check_post_dataout(
 735         struct iscsi_cmd *cmd,
 736         unsigned char *buf,
 737         u8 data_crc_failed)
 738 {
 739         struct iscsi_conn *conn = cmd->conn;
 740
 741         cmd->dataout_timeout_retries = 0;
 742
 743         if (!data_crc_failed)
 744                 return iscsit_dataout_post_crc_passed(cmd, buf);
 745         else {
 746                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 747                         pr_err("Unable to recover from DataOUT CRC"
 748                                 " failure while ERL=0, closing session.\n");
 749                         iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
 750                                           buf);
 751                         return DATAOUT_CANNOT_RECOVER;
 752                 }
 753
 754                 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR, buf);
 755                 return iscsit_dataout_post_crc_failed(cmd, buf);
 756         }
 757 }
 758
 759 static void iscsit_handle_time2retain_timeout(unsigned long data)
 760 {
 761         struct iscsi_session *sess = (struct iscsi_session *) data;
 762         struct iscsi_portal_group *tpg = ISCSI_TPG_S(sess);
 763         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
 764
 765         spin_lock_bh(&se_tpg->session_lock);
 766         if (sess->time2retain_timer_flags & ISCSI_TF_STOP) {
 767                 spin_unlock_bh(&se_tpg->session_lock);
 768                 return;
 769         }
 770         if (atomic_read(&sess->session_reinstatement)) {
 771                 pr_err("Exiting Time2Retain handler because"
 772                                 " session_reinstatement=1\n");
 773                 spin_unlock_bh(&se_tpg->session_lock);
 774                 return;
 775         }
 776         sess->time2retain_timer_flags |= ISCSI_TF_EXPIRED;
 777
 778         pr_err("Time2Retain timer expired for SID: %u, cleaning up"
 779                         " iSCSI session.\n", sess->sid);
 780         {
 781         struct iscsi_tiqn *tiqn = tpg->tpg_tiqn;
 782
 783         if (tiqn) {
 784                 spin_lock(&tiqn->sess_err_stats.lock);
 785                 strcpy(tiqn->sess_err_stats.last_sess_fail_rem_name,
 786                         (void *)sess->sess_ops->InitiatorName);
 787                 tiqn->sess_err_stats.last_sess_failure_type =
 788                                 ISCSI_SESS_ERR_CXN_TIMEOUT;
 789                 tiqn->sess_err_stats.cxn_timeout_errors++;
 790                 sess->conn_timeout_errors++;
 791                 spin_unlock(&tiqn->sess_err_stats.lock);
 792         }
 793         }
 794
 795         spin_unlock_bh(&se_tpg->session_lock);
 796         target_put_session(sess->se_sess);
 797 }
 798
 799 void iscsit_start_time2retain_handler(struct iscsi_session *sess)
 800 {
 801         int tpg_active;
 802         /*
 803          * Only start Time2Retain timer when the associated TPG is still in
 804          * an ACTIVE (eg: not disabled or shutdown) state.
 805          */
 806         spin_lock(&ISCSI_TPG_S(sess)->tpg_state_lock);
 807         tpg_active = (ISCSI_TPG_S(sess)->tpg_state == TPG_STATE_ACTIVE);
 808         spin_unlock(&ISCSI_TPG_S(sess)->tpg_state_lock);
 809
 810         if (!tpg_active)
 811                 return;
 812
 813         if (sess->time2retain_timer_flags & ISCSI_TF_RUNNING)
 814                 return;
 815
 816         pr_debug("Starting Time2Retain timer for %u seconds on"
 817                 " SID: %u\n", sess->sess_ops->DefaultTime2Retain, sess->sid);
 818
 819         init_timer(&sess->time2retain_timer);
 820         sess->time2retain_timer.expires =
 821                 (get_jiffies_64() + sess->sess_ops->DefaultTime2Retain * HZ);
 822         sess->time2retain_timer.data = (unsigned long)sess;
 823         sess->time2retain_timer.function = iscsit_handle_time2retain_timeout;
 824         sess->time2retain_timer_flags &= ~ISCSI_TF_STOP;
 825         sess->time2retain_timer_flags |= ISCSI_TF_RUNNING;
 826         add_timer(&sess->time2retain_timer);
 827 }
 828
 829 /*
 830  *      Called with spin_lock_bh(&struct se_portal_group->session_lock) held
 831  */
 832 int iscsit_stop_time2retain_timer(struct iscsi_session *sess)
 833 {
 834         struct iscsi_portal_group *tpg = ISCSI_TPG_S(sess);
 835         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
 836
 837         if (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)
 838                 return -1;
 839
 840         if (!(sess->time2retain_timer_flags & ISCSI_TF_RUNNING))
 841                 return 0;
 842
 843         sess->time2retain_timer_flags |= ISCSI_TF_STOP;
 844         spin_unlock_bh(&se_tpg->session_lock);
 845
 846         del_timer_sync(&sess->time2retain_timer);
 847
 848         spin_lock_bh(&se_tpg->session_lock);
 849         sess->time2retain_timer_flags &= ~ISCSI_TF_RUNNING;
 850         pr_debug("Stopped Time2Retain Timer for SID: %u\n",
 851                         sess->sid);
 852         return 0;
 853 }
 854
 855 void iscsit_connection_reinstatement_rcfr(struct iscsi_conn *conn)
 856 {
 857         spin_lock_bh(&conn->state_lock);
 858         if (atomic_read(&conn->connection_exit)) {
 859                 spin_unlock_bh(&conn->state_lock);
 860                 goto sleep;
 861         }
 862
 863         if (atomic_read(&conn->transport_failed)) {
 864                 spin_unlock_bh(&conn->state_lock);
 865                 goto sleep;
 866         }
 867         spin_unlock_bh(&conn->state_lock);
 868
 869         iscsi_thread_set_force_reinstatement(conn);
 870
 871 sleep:
 872         wait_for_completion(&conn->conn_wait_rcfr_comp);
 873         complete(&conn->conn_post_wait_comp);
 874 }
 875
 876 void iscsit_cause_connection_reinstatement(struct iscsi_conn *conn, int sleep)
 877 {
 878         spin_lock_bh(&conn->state_lock);
 879         if (atomic_read(&conn->connection_exit)) {
 880                 spin_unlock_bh(&conn->state_lock);
 881                 return;
 882         }
 883
 884         if (atomic_read(&conn->transport_failed)) {
 885                 spin_unlock_bh(&conn->state_lock);
 886                 return;
 887         }
 888
 889         if (atomic_read(&conn->connection_reinstatement)) {
 890                 spin_unlock_bh(&conn->state_lock);
 891                 return;
 892         }
 893
 894         if (iscsi_thread_set_force_reinstatement(conn) < 0) {
 895                 spin_unlock_bh(&conn->state_lock);
 896                 return;
 897         }
 898
 899         atomic_set(&conn->connection_reinstatement, 1);
 900         if (!sleep) {
 901                 spin_unlock_bh(&conn->state_lock);
 902                 return;
 903         }
 904
 905         atomic_set(&conn->sleep_on_conn_wait_comp, 1);
 906         spin_unlock_bh(&conn->state_lock);
 907
 908         wait_for_completion(&conn->conn_wait_comp);
 909         complete(&conn->conn_post_wait_comp);
 910 }
 911 EXPORT_SYMBOL(iscsit_cause_connection_reinstatement);
 912
 913 void iscsit_fall_back_to_erl0(struct iscsi_session *sess)
 914 {
 915         pr_debug("Falling back to ErrorRecoveryLevel=0 for SID:"
 916                         " %u\n", sess->sid);
 917
 918         atomic_set(&sess->session_fall_back_to_erl0, 1);
 919 }
 920
 921 static void iscsit_handle_connection_cleanup(struct iscsi_conn *conn)
 922 {
 923         struct iscsi_session *sess = conn->sess;
 924
 925         if ((sess->sess_ops->ErrorRecoveryLevel == 2) &&
 926             !atomic_read(&sess->session_reinstatement) &&
 927             !atomic_read(&sess->session_fall_back_to_erl0))
 928                 iscsit_connection_recovery_transport_reset(conn);
 929         else {
 930                 pr_debug("Performing cleanup for failed iSCSI"
 931                         " Connection ID: %hu from %s\n", conn->cid,
 932                         sess->sess_ops->InitiatorName);
 933                 iscsit_close_connection(conn);
 934         }
 935 }
 936
 937 void iscsit_take_action_for_connection_exit(struct iscsi_conn *conn)
 938 {
 939         spin_lock_bh(&conn->state_lock);
 940         if (atomic_read(&conn->connection_exit)) {
 941                 spin_unlock_bh(&conn->state_lock);
 942                 return;
 943         }
 944         atomic_set(&conn->connection_exit, 1);
 945
 946         if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT) {
 947                 spin_unlock_bh(&conn->state_lock);
 948                 iscsit_close_connection(conn);
 949                 return;
 950         }
 951
 952         if (conn->conn_state == TARG_CONN_STATE_CLEANUP_WAIT) {
 953                 spin_unlock_bh(&conn->state_lock);
 954                 return;
 955         }
 956
 957         pr_debug("Moving to TARG_CONN_STATE_CLEANUP_WAIT.\n");
 958         conn->conn_state = TARG_CONN_STATE_CLEANUP_WAIT;
 959         spin_unlock_bh(&conn->state_lock);
 960
 961         iscsit_handle_connection_cleanup(conn);
 962 }
 963
 964 /*
 965  *      This is the simple function that makes the magic of
 966  *      sync and steering happen in the follow paradoxical order:
 967  *
 968  *      0) Receive conn->of_marker (bytes left until next OFMarker)
 969  *         bytes into an offload buffer.  When we pass the exact number
 970  *         of bytes in conn->of_marker, iscsit_dump_data_payload() and hence
 971  *         rx_data() will automatically receive the identical u32 marker
 972  *         values and store it in conn->of_marker_offset;
 973  *      1) Now conn->of_marker_offset will contain the offset to the start
 974  *         of the next iSCSI PDU.  Dump these remaining bytes into another
 975  *         offload buffer.
 976  *      2) We are done!
 977  *         Next byte in the TCP stream will contain the next iSCSI PDU!
 978  *         Cool Huh?!
 979  */
 980 int iscsit_recover_from_unknown_opcode(struct iscsi_conn *conn)
 981 {
 982         /*
 983          * Make sure the remaining bytes to next maker is a sane value.
 984          */
 985         if (conn->of_marker > (conn->conn_ops->OFMarkInt * 4)) {
 986                 pr_err("Remaining bytes to OFMarker: %u exceeds"
 987                         " OFMarkInt bytes: %u.\n", conn->of_marker,
 988                                 conn->conn_ops->OFMarkInt * 4);
 989                 return -1;
 990         }
 991
 992         pr_debug("Advancing %u bytes in TCP stream to get to the"
 993                         " next OFMarker.\n", conn->of_marker);
 994
 995         if (iscsit_dump_data_payload(conn, conn->of_marker, 0) < 0)
 996                 return -1;
 997
 998         /*
 999          * Make sure the offset marker we retrived is a valid value.
1000          */
1001         if (conn->of_marker_offset > (ISCSI_HDR_LEN + (ISCSI_CRC_LEN * 2) +
1002             conn->conn_ops->MaxRecvDataSegmentLength)) {
1003                 pr_err("OfMarker offset value: %u exceeds limit.\n",
1004                         conn->of_marker_offset);
1005                 return -1;
1006         }
1007
1008         pr_debug("Discarding %u bytes of TCP stream to get to the"
1009                         " next iSCSI Opcode.\n", conn->of_marker_offset);
1010
1011         if (iscsit_dump_data_payload(conn, conn->of_marker_offset, 0) < 0)
1012                 return -1;
1013
1014         return 0;
1015 }