iscsi-scst/kernel/iscsi.c

   1 /*
   2  *  Copyright (C) 2002 - 2003 Ardis Technolgies <roman@ardistech.com>
   3  *  Copyright (C) 2007 - 2009 Vladislav Bolkhovitin
   4  *  Copyright (C) 2007 - 2009 ID7 Ltd.
   5  *
   6  *  This program is free software; you can redistribute it and/or
   7  *  modify it under the terms of the GNU General Public License
   8  *  as published by the Free Software Foundation, version 2
   9  *  of the License.
  10  *
  11  *  This program is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14  *  GNU General Public License for more details.
  15  */
  16
  17 #include <linux/module.h>
  18 #include <linux/hash.h>
  19 #include <linux/kthread.h>
  20 #include <linux/scatterlist.h>
  21 #include <net/tcp.h>
  22 #include <scsi/scsi.h>
  23
  24 #include "iscsi.h"
  25 #include "digest.h"
  26
  27 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
  28 #warning "Patch put_page_callback-<kernel-version>.patch not applied on your\
  29  kernel or CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION\
  30  config option not set. ISCSI-SCST will be working with not the best\
  31  performance. Refer README file for details."
  32 #endif
  33
  34 #define ISCSI_INIT_WRITE_WAKE           0x1
  35 #define ISCSI_INIT_WRITE_REMOVE_HASH    0x2
  36
  37 static int ctr_major;
  38 static char ctr_name[] = "iscsi-scst-ctl";
  39 static int iscsi_template_registered;
  40
  41 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
  42 unsigned long iscsi_trace_flag = ISCSI_DEFAULT_LOG_FLAGS;
  43 #endif
  44
  45 static struct kmem_cache *iscsi_cmnd_cache;
  46
  47 DEFINE_SPINLOCK(iscsi_rd_lock);
  48 LIST_HEAD(iscsi_rd_list);
  49 DECLARE_WAIT_QUEUE_HEAD(iscsi_rd_waitQ);
  50
  51 DEFINE_SPINLOCK(iscsi_wr_lock);
  52 LIST_HEAD(iscsi_wr_list);
  53 DECLARE_WAIT_QUEUE_HEAD(iscsi_wr_waitQ);
  54
  55 static struct page *dummy_page;
  56 static struct scatterlist dummy_sg;
  57
  58 struct iscsi_thread_t {
  59         struct task_struct *thr;
  60         struct list_head threads_list_entry;
  61 };
  62
  63 static LIST_HEAD(iscsi_threads_list);
  64
  65 static void cmnd_remove_hash(struct iscsi_cmnd *cmnd);
  66 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status);
  67 static void cmnd_prepare_get_rejected_cmd_data(struct iscsi_cmnd *cmnd);
  68 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess);
  69 static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd);
  70 static void req_cmnd_release(struct iscsi_cmnd *req);
  71
  72 static inline u32 cmnd_write_size(struct iscsi_cmnd *cmnd)
  73 {
  74         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
  75
  76         if (hdr->flags & ISCSI_CMD_WRITE)
  77                 return be32_to_cpu(hdr->data_length);
  78         return 0;
  79 }
  80
  81 static inline int cmnd_read_size(struct iscsi_cmnd *cmnd)
  82 {
  83         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
  84
  85         if (hdr->flags & ISCSI_CMD_READ) {
  86                 struct iscsi_ahs_hdr *ahdr;
  87
  88                 if (!(hdr->flags & ISCSI_CMD_WRITE))
  89                         return be32_to_cpu(hdr->data_length);
  90
  91                 ahdr = (struct iscsi_ahs_hdr *)cmnd->pdu.ahs;
  92                 if (ahdr != NULL) {
  93                         uint8_t *p = (uint8_t *)ahdr;
  94                         int size = 0;
  95                         do {
  96                                 int s;
  97
  98                                 ahdr = (struct iscsi_ahs_hdr *)p;
  99
 100                                 if (ahdr->ahstype == ISCSI_AHSTYPE_RLENGTH) {
 101                                         struct iscsi_rlength_ahdr *rh =
 102                                               (struct iscsi_rlength_ahdr *)ahdr;
 103                                         return be32_to_cpu(rh->read_length);
 104                                 }
 105
 106                                 s = 3 + be16_to_cpu(ahdr->ahslength);
 107                                 s = (s + 3) & -4;
 108                                 size += s;
 109                                 p += s;
 110                         } while (size < cmnd->pdu.ahssize);
 111                 }
 112                 return -1;
 113         }
 114         return 0;
 115 }
 116
 117 void iscsi_restart_cmnd(struct iscsi_cmnd *cmnd)
 118 {
 119         EXTRACHECKS_BUG_ON(cmnd->data_waiting);
 120
 121         if (unlikely(test_bit(ISCSI_CONN_REINSTATING,
 122                         &cmnd->conn->conn_aflags))) {
 123                 struct iscsi_target *target = cmnd->conn->session->target;
 124                 bool get_out;
 125
 126                 mutex_lock(&target->target_mutex);
 127
 128                 get_out = test_bit(ISCSI_CONN_REINSTATING,
 129                                 &cmnd->conn->conn_aflags);
 130                 /* Let's don't look dead */
 131                 if (scst_cmd_get_cdb(cmnd->scst_cmd)[0] == TEST_UNIT_READY)
 132                         get_out = false;
 133
 134                 if (!get_out)
 135                         goto unlock_cont;
 136
 137                 TRACE_MGMT_DBG("Pending cmnd %p, because conn %p is "
 138                         "reinstated", cmnd, cmnd->conn);
 139
 140                 cmnd->scst_state = ISCSI_CMD_STATE_REINST_PENDING;
 141                 list_add_tail(&cmnd->reinst_pending_cmd_list_entry,
 142                         &cmnd->conn->reinst_pending_cmd_list);
 143
 144 unlock_cont:
 145                 mutex_unlock(&target->target_mutex);
 146
 147                 if (get_out)
 148                         goto out;
 149         }
 150
 151         cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
 152         scst_restart_cmd(cmnd->scst_cmd, SCST_PREPROCESS_STATUS_SUCCESS,
 153                 SCST_CONTEXT_THREAD);
 154
 155 out:
 156         return;
 157 }
 158
 159 static inline void iscsi_restart_waiting_cmnd(struct iscsi_cmnd *cmnd)
 160 {
 161         /*
 162          * There is no race with conn_abort(), since all functions
 163          * called from single read thread
 164          */
 165         iscsi_extracheck_is_rd_thread(cmnd->conn);
 166         cmnd->data_waiting = 0;
 167
 168         iscsi_restart_cmnd(cmnd);
 169         return;
 170 }
 171
 172 static inline void iscsi_fail_waiting_cmnd(struct iscsi_cmnd *cmnd)
 173 {
 174         TRACE_MGMT_DBG("Failing data waiting cmd %p", cmnd);
 175
 176         /*
 177          * There is no race with conn_abort(), since all functions
 178          * called from single read thread
 179          */
 180         iscsi_extracheck_is_rd_thread(cmnd->conn);
 181         cmnd->data_waiting = 0;
 182
 183         req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
 184 }
 185
 186 struct iscsi_cmnd *cmnd_alloc(struct iscsi_conn *conn,
 187                               struct iscsi_cmnd *parent)
 188 {
 189         struct iscsi_cmnd *cmnd;
 190
 191         /* ToDo: __GFP_NOFAIL?? */
 192 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17)
 193         cmnd = kmem_cache_alloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
 194         memset(cmnd, 0, sizeof(*cmnd));
 195 #else
 196         cmnd = kmem_cache_zalloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
 197 #endif
 198
 199         atomic_set(&cmnd->ref_cnt, 1);
 200         cmnd->scst_state = ISCSI_CMD_STATE_NEW;
 201         cmnd->conn = conn;
 202         cmnd->parent_req = parent;
 203
 204         if (parent == NULL) {
 205                 conn_get(conn);
 206
 207 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
 208                 atomic_set(&cmnd->net_ref_cnt, 0);
 209 #endif
 210                 spin_lock_init(&cmnd->rsp_cmd_lock);
 211                 INIT_LIST_HEAD(&cmnd->rsp_cmd_list);
 212                 INIT_LIST_HEAD(&cmnd->rx_ddigest_cmd_list);
 213
 214                 spin_lock_bh(&conn->cmd_list_lock);
 215                 list_add_tail(&cmnd->cmd_list_entry, &conn->cmd_list);
 216                 spin_unlock_bh(&conn->cmd_list_lock);
 217         }
 218
 219         TRACE_DBG("conn %p, parent %p, cmnd %p", conn, parent, cmnd);
 220         return cmnd;
 221 }
 222
 223 /* Frees a command. Also frees the additional header. */
 224 static void cmnd_free(struct iscsi_cmnd *cmnd)
 225 {
 226         TRACE_DBG("%p", cmnd);
 227
 228         if (unlikely(cmnd->tm_aborted)) {
 229                 TRACE_MGMT_DBG("Free aborted cmd %p (scst cmd %p, state %d, "
 230                         "parent_req %p)", cmnd, cmnd->scst_cmd,
 231                         cmnd->scst_state, cmnd->parent_req);
 232         }
 233
 234         /* Catch users from cmd_list or rsp_cmd_list */
 235         EXTRACHECKS_BUG_ON(atomic_read(&cmnd->ref_cnt) != 0);
 236
 237         kfree(cmnd->pdu.ahs);
 238
 239         if (unlikely(cmnd->on_write_list || cmnd->on_written_list)) {
 240                 struct iscsi_scsi_cmd_hdr *req = cmnd_hdr(cmnd);
 241
 242                 PRINT_CRIT_ERROR("cmnd %p still on some list?, %x, %x, %x, "
 243                         "%x, %x, %x, %x", cmnd, req->opcode, req->scb[0],
 244                         req->flags, req->itt, be32_to_cpu(req->data_length),
 245                         req->cmd_sn, be32_to_cpu(cmnd->pdu.datasize));
 246
 247                 if (unlikely(cmnd->parent_req)) {
 248                         struct iscsi_scsi_cmd_hdr *preq =
 249                                         cmnd_hdr(cmnd->parent_req);
 250                         PRINT_CRIT_ERROR("%p %x %u", preq, preq->opcode,
 251                                 preq->scb[0]);
 252                 }
 253                 sBUG();
 254         }
 255
 256         kmem_cache_free(iscsi_cmnd_cache, cmnd);
 257         return;
 258 }
 259
 260 /* Might be called unded some lock and on SIRQ */
 261 void cmnd_done(struct iscsi_cmnd *cmnd)
 262 {
 263         TRACE_DBG("%p", cmnd);
 264
 265         if (unlikely(cmnd->tm_aborted)) {
 266                 TRACE_MGMT_DBG("Done aborted cmd %p (scst cmd %p, state %d, "
 267                         "parent_req %p)", cmnd, cmnd->scst_cmd,
 268                         cmnd->scst_state, cmnd->parent_req);
 269         }
 270
 271         EXTRACHECKS_BUG_ON(cmnd->on_rx_digest_list);
 272
 273         if (cmnd->on_written_list) {
 274                 struct iscsi_conn *conn = cmnd->conn;
 275                 TRACE_DBG("Deleting cmd %p from conn %p written_list", cmnd,
 276                         conn);
 277                 spin_lock_bh(&conn->write_list_lock);
 278                 list_del(&cmnd->written_list_entry);
 279                 cmnd->on_written_list = 0;
 280                 spin_unlock_bh(&conn->write_list_lock);
 281         }
 282
 283         if (cmnd->parent_req == NULL) {
 284                 struct iscsi_conn *conn = cmnd->conn;
 285                 TRACE_DBG("Deleting req %p from conn %p", cmnd, conn);
 286
 287                 spin_lock_bh(&conn->cmd_list_lock);
 288                 list_del(&cmnd->cmd_list_entry);
 289                 spin_unlock_bh(&conn->cmd_list_lock);
 290
 291                 conn_put(conn);
 292
 293                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rsp_cmd_list));
 294                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rx_ddigest_cmd_list));
 295
 296                 /* Order between above and below code is important! */
 297
 298                 if ((cmnd->scst_cmd != NULL) || (cmnd->scst_aen != NULL)) {
 299                         switch (cmnd->scst_state) {
 300                         case ISCSI_CMD_STATE_PROCESSED:
 301                                 TRACE_DBG("cmd %p PROCESSED", cmnd);
 302                                 scst_tgt_cmd_done(cmnd->scst_cmd,
 303                                         SCST_CONTEXT_DIRECT);
 304                                 break;
 305
 306                         case ISCSI_CMD_STATE_AFTER_PREPROC:
 307                         {
 308                                 struct scst_cmd *scst_cmd = cmnd->scst_cmd;
 309                                 TRACE_DBG("cmd %p AFTER_PREPROC", cmnd);
 310                                 cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
 311                                 cmnd->scst_cmd = NULL;
 312                                 scst_restart_cmd(scst_cmd,
 313                                         SCST_PREPROCESS_STATUS_ERROR_FATAL,
 314                                         SCST_CONTEXT_THREAD);
 315                                 break;
 316                         }
 317
 318                         case ISCSI_CMD_STATE_AEN:
 319                                 TRACE_DBG("cmd %p AEN PROCESSED", cmnd);
 320                                 scst_aen_done(cmnd->scst_aen);
 321                                 break;
 322
 323                         default:
 324                                 PRINT_CRIT_ERROR("Unexpected cmnd scst state "
 325                                         "%d", cmnd->scst_state);
 326                                 sBUG();
 327                                 break;
 328                         }
 329                 }
 330         } else {
 331                 TRACE_DBG("Deleting rsp %p from parent %p", cmnd,
 332                         cmnd->parent_req);
 333
 334                 spin_lock_bh(&cmnd->parent_req->rsp_cmd_lock);
 335                 list_del(&cmnd->rsp_cmd_list_entry);
 336                 spin_unlock_bh(&cmnd->parent_req->rsp_cmd_lock);
 337
 338                 cmnd_put(cmnd->parent_req);
 339         }
 340
 341         /* Order between above and below code is important! */
 342
 343         if (cmnd->own_sg) {
 344                 TRACE_DBG("%s", "own_sg");
 345                 if ((cmnd->sg != &dummy_sg) && (cmnd->sg != cmnd->rsp_sg))
 346                         scst_free(cmnd->sg, cmnd->sg_cnt);
 347 #ifdef CONFIG_SCST_DEBUG
 348                 cmnd->own_sg = 0;
 349                 cmnd->sg = NULL;
 350                 cmnd->sg_cnt = -1;
 351 #endif
 352         }
 353
 354         if (cmnd->dec_active_cmnds) {
 355                 struct iscsi_session *sess = cmnd->conn->session;
 356                 TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
 357                         "new value %d)", cmnd, sess,
 358                         atomic_read(&sess->active_cmds)-1);
 359                 atomic_dec(&sess->active_cmds);
 360 #ifdef CONFIG_SCST_EXTRACHECKS
 361                 if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
 362                         PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
 363                                 atomic_read(&sess->active_cmds));
 364                         sBUG();
 365                 }
 366 #endif
 367         }
 368
 369         cmnd_free(cmnd);
 370         return;
 371 }
 372
 373 /*
 374  * Corresponding conn may also gets destroyed atfer this function, except only
 375  * if it's called from the read thread!
 376  *
 377  * It can't be called in parallel with iscsi_cmnds_init_write()!
 378  */
 379 void req_cmnd_release_force(struct iscsi_cmnd *req, int flags)
 380 {
 381         struct iscsi_cmnd *rsp, *t;
 382         struct iscsi_conn *conn = req->conn;
 383         LIST_HEAD(cmds_list);
 384
 385         TRACE_ENTRY();
 386
 387         TRACE_MGMT_DBG("%p", req);
 388
 389         sBUG_ON(req == conn->read_cmnd);
 390
 391         if (flags & ISCSI_FORCE_RELEASE_WRITE) {
 392                 spin_lock_bh(&conn->write_list_lock);
 393                 list_for_each_entry_safe(rsp, t, &conn->write_list,
 394                                                 write_list_entry) {
 395                         if (rsp->parent_req != req)
 396                                 continue;
 397
 398                         cmd_del_from_write_list(rsp);
 399
 400                         list_add_tail(&rsp->write_list_entry, &cmds_list);
 401                 }
 402                 spin_unlock_bh(&conn->write_list_lock);
 403
 404                 list_for_each_entry_safe(rsp, t, &cmds_list,
 405                                                 write_list_entry) {
 406                         TRACE_MGMT_DBG("Putting write rsp %p", rsp);
 407                         list_del(&rsp->write_list_entry);
 408                         cmnd_put(rsp);
 409                 }
 410         }
 411
 412 again_rsp:
 413         spin_lock_bh(&req->rsp_cmd_lock);
 414         list_for_each_entry_reverse(rsp, &req->rsp_cmd_list,
 415                         rsp_cmd_list_entry) {
 416                 bool r;
 417
 418                 if (rsp->force_cleanup_done)
 419                         continue;
 420
 421                 rsp->force_cleanup_done = 1;
 422
 423                 if (cmnd_get_check(rsp))
 424                         continue;
 425
 426                 spin_unlock_bh(&req->rsp_cmd_lock);
 427
 428                 spin_lock_bh(&conn->write_list_lock);
 429                 r = rsp->on_write_list || rsp->write_processing_started;
 430                 spin_unlock_bh(&conn->write_list_lock);
 431
 432                 cmnd_put(rsp);
 433
 434                 if (r)
 435                         goto again_rsp;
 436
 437                 /*
 438                  * If both on_write_list and write_processing_started not set,
 439                  * we can safely put() rsp.
 440                  */
 441                 TRACE_MGMT_DBG("Putting rsp %p", rsp);
 442                 cmnd_put(rsp);
 443                 goto again_rsp;
 444         }
 445         spin_unlock_bh(&req->rsp_cmd_lock);
 446
 447         req_cmnd_release(req);
 448
 449         TRACE_EXIT();
 450         return;
 451 }
 452
 453 /*
 454  * Corresponding conn may also gets destroyed atfer this function, except only
 455  * if it's called from the read thread!
 456  */
 457 static void req_cmnd_release(struct iscsi_cmnd *req)
 458 {
 459         struct iscsi_cmnd *c, *t;
 460
 461         TRACE_ENTRY();
 462
 463         TRACE_DBG("%p", req);
 464
 465 #ifdef CONFIG_SCST_EXTRACHECKS
 466         sBUG_ON(req->release_called);
 467         req->release_called = 1;
 468 #endif
 469
 470         if (unlikely(req->tm_aborted)) {
 471                 TRACE_MGMT_DBG("Release aborted req cmd %p (scst cmd %p, "
 472                         "state %d)", req, req->scst_cmd, req->scst_state);
 473         }
 474
 475         sBUG_ON(req->parent_req != NULL);
 476
 477         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
 478                                 rx_ddigest_cmd_list_entry) {
 479                 cmd_del_from_rx_ddigest_list(c);
 480                 cmnd_put(c);
 481         }
 482
 483         if (req->hashed)
 484                 cmnd_remove_hash(req);
 485
 486         if (req->dec_active_cmnds) {
 487                 struct iscsi_session *sess = req->conn->session;
 488                 TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
 489                         "new value %d)", req, sess,
 490                         atomic_read(&sess->active_cmds)-1);
 491                 atomic_dec(&sess->active_cmds);
 492                 req->dec_active_cmnds = 0;
 493 #ifdef CONFIG_SCST_EXTRACHECKS
 494                 if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
 495                         PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
 496                                 atomic_read(&sess->active_cmds));
 497                         sBUG();
 498                 }
 499 #endif
 500         }
 501
 502         cmnd_put(req);
 503
 504         TRACE_EXIT();
 505         return;
 506 }
 507
 508 /*
 509  * Corresponding conn may also gets destroyed atfer this function, except only
 510  * if it's called from the read thread!
 511  */
 512 void rsp_cmnd_release(struct iscsi_cmnd *cmnd)
 513 {
 514         TRACE_DBG("%p", cmnd);
 515
 516 #ifdef CONFIG_SCST_EXTRACHECKS
 517         sBUG_ON(cmnd->release_called);
 518         cmnd->release_called = 1;
 519 #endif
 520
 521         sBUG_ON(cmnd->hashed);
 522         sBUG_ON(cmnd->parent_req == NULL);
 523
 524         cmnd_put(cmnd);
 525         return;
 526 }
 527
 528 /**
 529  * create a new command used as response.
 530  *
 531  * iscsi_cmnd_create_rsp_cmnd -
 532  * @cmnd: ptr to request command
 533  *
 534  * @return    ptr to response command or NULL
 535  */
 536 static struct iscsi_cmnd *iscsi_cmnd_create_rsp_cmnd(struct iscsi_cmnd *parent)
 537 {
 538         struct iscsi_cmnd *rsp;
 539
 540         rsp = cmnd_alloc(parent->conn, parent);
 541
 542         spin_lock_bh(&parent->rsp_cmd_lock);
 543         TRACE_DBG("Adding rsp %p to parent %p", rsp, parent);
 544         list_add_tail(&rsp->rsp_cmd_list_entry, &parent->rsp_cmd_list);
 545         spin_unlock_bh(&parent->rsp_cmd_lock);
 546         cmnd_get(parent);
 547         return rsp;
 548 }
 549
 550 static inline struct iscsi_cmnd *get_rsp_cmnd(struct iscsi_cmnd *req)
 551 {
 552         struct iscsi_cmnd *res = NULL;
 553
 554         /* Currently this lock isn't needed, but just in case.. */
 555         spin_lock_bh(&req->rsp_cmd_lock);
 556         if (!list_empty(&req->rsp_cmd_list)) {
 557                 res = list_entry(req->rsp_cmd_list.prev, struct iscsi_cmnd,
 558                         rsp_cmd_list_entry);
 559         }
 560         spin_unlock_bh(&req->rsp_cmd_lock);
 561
 562         return res;
 563 }
 564
 565 static void iscsi_cmnds_init_write(struct list_head *send, int flags)
 566 {
 567         struct iscsi_cmnd *rsp = list_entry(send->next, struct iscsi_cmnd,
 568                                                 write_list_entry);
 569         struct iscsi_conn *conn = rsp->conn;
 570         struct list_head *pos, *next;
 571
 572         sBUG_ON(list_empty(send));
 573
 574         /*
 575          * If we don't remove hashed req cmd from the hash list here, before
 576          * submitting it for transmittion, we will have a race, when for
 577          * some reason cmd's release is delayed after transmittion and
 578          * initiator sends cmd with the same ITT => this command will be
 579          * erroneously rejected as a duplicate.
 580          */
 581         if ((flags & ISCSI_INIT_WRITE_REMOVE_HASH) &&
 582             rsp->parent_req->hashed &&
 583             (rsp->parent_req->r2t_length == 0) &&
 584             (rsp->parent_req->outstanding_r2t == 0))
 585                 cmnd_remove_hash(rsp->parent_req);
 586
 587         if (!(conn->ddigest_type & DIGEST_NONE)) {
 588                 list_for_each(pos, send) {
 589                         rsp = list_entry(pos, struct iscsi_cmnd,
 590                                                 write_list_entry);
 591
 592                         if (rsp->pdu.datasize != 0) {
 593                                 TRACE_DBG("Doing data digest (%p:%x)", rsp,
 594                                         cmnd_opcode(rsp));
 595                                 digest_tx_data(rsp);
 596                         }
 597                 }
 598         }
 599
 600         spin_lock_bh(&conn->write_list_lock);
 601         list_for_each_safe(pos, next, send) {
 602                 rsp = list_entry(pos, struct iscsi_cmnd, write_list_entry);
 603
 604                 TRACE_DBG("%p:%x", rsp, cmnd_opcode(rsp));
 605
 606                 sBUG_ON(conn != rsp->conn);
 607
 608                 list_del(&rsp->write_list_entry);
 609                 cmd_add_on_write_list(conn, rsp);
 610         }
 611         spin_unlock_bh(&conn->write_list_lock);
 612
 613         if (flags & ISCSI_INIT_WRITE_WAKE)
 614                 iscsi_make_conn_wr_active(conn);
 615
 616         return;
 617 }
 618
 619 static void iscsi_cmnd_init_write(struct iscsi_cmnd *rsp, int flags)
 620 {
 621         LIST_HEAD(head);
 622
 623         if (unlikely(rsp->on_write_list)) {
 624                 PRINT_CRIT_ERROR("cmd already on write list (%x %x %x %x %u "
 625                         "%u %u %u %u %u %u %d %d",
 626                         cmnd_itt(rsp), cmnd_ttt(rsp), cmnd_opcode(rsp),
 627                         cmnd_scsicode(rsp), rsp->r2t_sn,
 628                         rsp->r2t_length, rsp->is_unsolicited_data,
 629                         rsp->target_task_tag, rsp->outstanding_r2t,
 630                         rsp->hdigest, rsp->ddigest,
 631                         list_empty(&rsp->rsp_cmd_list), rsp->hashed);
 632                 sBUG();
 633         }
 634         list_add_tail(&rsp->write_list_entry, &head);
 635         iscsi_cmnds_init_write(&head, flags);
 636         return;
 637 }
 638
 639 static void send_data_rsp(struct iscsi_cmnd *req, u8 status, int send_status)
 640 {
 641         struct iscsi_cmnd *rsp;
 642         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
 643         struct iscsi_data_in_hdr *rsp_hdr;
 644         u32 pdusize, expsize, size, offset, sn;
 645         LIST_HEAD(send);
 646
 647         TRACE_DBG("req %p", req);
 648
 649         pdusize = req->conn->session->sess_param.max_xmit_data_length;
 650         expsize = req->read_size;
 651         size = min(expsize, (u32)req->bufflen);
 652         offset = 0;
 653         sn = 0;
 654
 655         while (1) {
 656                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
 657                 TRACE_DBG("rsp %p", rsp);
 658                 rsp->sg = req->sg;
 659                 rsp->sg_cnt = req->sg_cnt;
 660                 rsp->bufflen = req->bufflen;
 661                 rsp_hdr = (struct iscsi_data_in_hdr *)&rsp->pdu.bhs;
 662
 663                 rsp_hdr->opcode = ISCSI_OP_SCSI_DATA_IN;
 664                 rsp_hdr->itt = req_hdr->itt;
 665                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
 666                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
 667                 rsp_hdr->data_sn = cpu_to_be32(sn);
 668
 669                 if (size <= pdusize) {
 670                         TRACE_DBG("offset %d, size %d", offset, size);
 671                         rsp->pdu.datasize = size;
 672                         if (send_status) {
 673                                 int scsisize;
 674
 675                                 TRACE_DBG("status %x", status);
 676
 677                                 EXTRACHECKS_BUG_ON((cmnd_hdr(req)->flags & ISCSI_CMD_WRITE) != 0);
 678
 679                                 rsp_hdr->flags = ISCSI_FLG_FINAL | ISCSI_FLG_STATUS;
 680                                 rsp_hdr->cmd_status = status;
 681
 682                                 scsisize = req->bufflen;
 683                                 if (scsisize < expsize) {
 684                                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
 685                                         size = expsize - scsisize;
 686                                 } else if (scsisize > expsize) {
 687                                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
 688                                         size = scsisize - expsize;
 689                                 } else
 690                                         size = 0;
 691                                 rsp_hdr->residual_count = cpu_to_be32(size);
 692                         }
 693                         list_add_tail(&rsp->write_list_entry, &send);
 694                         break;
 695                 }
 696
 697                 TRACE_DBG("pdusize %d, offset %d, size %d", pdusize, offset,
 698                         size);
 699
 700                 rsp->pdu.datasize = pdusize;
 701
 702                 size -= pdusize;
 703                 offset += pdusize;
 704                 sn++;
 705
 706                 list_add_tail(&rsp->write_list_entry, &send);
 707         }
 708         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_REMOVE_HASH);
 709         return;
 710 }
 711
 712 static struct iscsi_cmnd *create_status_rsp(struct iscsi_cmnd *req, int status,
 713         const u8 *sense_buf, int sense_len)
 714 {
 715         struct iscsi_cmnd *rsp;
 716         struct iscsi_scsi_rsp_hdr *rsp_hdr;
 717         struct scatterlist *sg;
 718
 719         rsp = iscsi_cmnd_create_rsp_cmnd(req);
 720         TRACE_DBG("%p", rsp);
 721
 722         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
 723         rsp_hdr->opcode = ISCSI_OP_SCSI_RSP;
 724         rsp_hdr->flags = ISCSI_FLG_FINAL;
 725         rsp_hdr->response = ISCSI_RESPONSE_COMMAND_COMPLETED;
 726         rsp_hdr->cmd_status = status;
 727         rsp_hdr->itt = cmnd_hdr(req)->itt;
 728
 729         if (SCST_SENSE_VALID(sense_buf)) {
 730                 TRACE_DBG("%s", "SENSE VALID");
 731
 732                 sg = rsp->sg = rsp->rsp_sg;
 733                 rsp->sg_cnt = 2;
 734                 rsp->own_sg = 1;
 735
 736                 sg_init_table(sg, 2);
 737                 sg_set_buf(&sg[0], &rsp->sense_hdr, sizeof(rsp->sense_hdr));
 738                 sg_set_buf(&sg[1], sense_buf, sense_len);
 739
 740                 rsp->sense_hdr.length = cpu_to_be16(sense_len);
 741
 742                 rsp->pdu.datasize = sizeof(rsp->sense_hdr) + sense_len;
 743                 rsp->bufflen = rsp->pdu.datasize;
 744         } else {
 745                 rsp->pdu.datasize = 0;
 746                 rsp->bufflen = 0;
 747         }
 748
 749         return rsp;
 750 }
 751
 752 static void iscsi_cmnd_reject(struct iscsi_cmnd *req, int reason)
 753 {
 754         struct iscsi_cmnd *rsp;
 755         struct iscsi_reject_hdr *rsp_hdr;
 756         struct scatterlist *sg;
 757
 758         TRACE_MGMT_DBG("Reject: req %p, reason %x", req, reason);
 759
 760         sBUG_ON(req->rejected);
 761         req->rejected = 1;
 762         req->reject_reason = ISCSI_REJECT_CMD;
 763
 764         rsp = iscsi_cmnd_create_rsp_cmnd(req);
 765         rsp_hdr = (struct iscsi_reject_hdr *)&rsp->pdu.bhs;
 766
 767         rsp_hdr->opcode = ISCSI_OP_REJECT;
 768         rsp_hdr->ffffffff = ISCSI_RESERVED_TAG;
 769         rsp_hdr->reason = reason;
 770
 771         sg = rsp->sg = rsp->rsp_sg;
 772         rsp->sg_cnt = 1;
 773         rsp->own_sg = 1;
 774         sg_init_one(sg, &req->pdu.bhs, sizeof(struct iscsi_hdr));
 775         rsp->bufflen = rsp->pdu.datasize = sizeof(struct iscsi_hdr);
 776
 777         iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH |
 778                                          ISCSI_INIT_WRITE_WAKE);
 779
 780         cmnd_prepare_get_rejected_cmd_data(req);
 781         return;
 782 }
 783
 784 static inline int iscsi_get_allowed_cmds(struct iscsi_session *sess)
 785 {
 786         int res = max(-1, (int)sess->max_queued_cmnds -
 787                                 atomic_read(&sess->active_cmds)-1);
 788         TRACE_DBG("allowed cmds %d (sess %p, active_cmds %d)", res,
 789                 sess, atomic_read(&sess->active_cmds));
 790         return res;
 791 }
 792
 793 static u32 cmnd_set_sn(struct iscsi_cmnd *cmnd, int set_stat_sn)
 794 {
 795         struct iscsi_conn *conn = cmnd->conn;
 796         struct iscsi_session *sess = conn->session;
 797         u32 res;
 798
 799         spin_lock(&sess->sn_lock);
 800
 801         if (set_stat_sn)
 802                 cmnd->pdu.bhs.sn = cpu_to_be32(conn->stat_sn++);
 803         cmnd->pdu.bhs.exp_sn = cpu_to_be32(sess->exp_cmd_sn);
 804         cmnd->pdu.bhs.max_sn = cpu_to_be32(sess->exp_cmd_sn +
 805                                  iscsi_get_allowed_cmds(sess));
 806
 807         res = cpu_to_be32(conn->stat_sn);
 808
 809         spin_unlock(&sess->sn_lock);
 810         return res;
 811 }
 812
 813 /* Called under sn_lock */
 814 static void __update_stat_sn(struct iscsi_cmnd *cmnd)
 815 {
 816         struct iscsi_conn *conn = cmnd->conn;
 817         u32 exp_stat_sn;
 818
 819         cmnd->pdu.bhs.exp_sn = exp_stat_sn = be32_to_cpu(cmnd->pdu.bhs.exp_sn);
 820         TRACE_DBG("%x,%x", cmnd_opcode(cmnd), exp_stat_sn);
 821         if ((int)(exp_stat_sn - conn->exp_stat_sn) > 0 &&
 822             (int)(exp_stat_sn - conn->stat_sn) <= 0) {
 823                 /* free pdu resources */
 824                 cmnd->conn->exp_stat_sn = exp_stat_sn;
 825         }
 826         return;
 827 }
 828
 829 static inline void update_stat_sn(struct iscsi_cmnd *cmnd)
 830 {
 831         spin_lock(&cmnd->conn->session->sn_lock);
 832         __update_stat_sn(cmnd);
 833         spin_unlock(&cmnd->conn->session->sn_lock);
 834         return;
 835 }
 836
 837 /* Called under sn_lock */
 838 static int check_cmd_sn(struct iscsi_cmnd *cmnd)
 839 {
 840         struct iscsi_session *session = cmnd->conn->session;
 841         u32 cmd_sn;
 842
 843         cmnd->pdu.bhs.sn = cmd_sn = be32_to_cpu(cmnd->pdu.bhs.sn);
 844         TRACE_DBG("%d(%d)", cmd_sn, session->exp_cmd_sn);
 845         if (likely((s32)(cmd_sn - session->exp_cmd_sn) >= 0))
 846                 return 0;
 847         PRINT_ERROR("sequence error (%x,%x)", cmd_sn, session->exp_cmd_sn);
 848         return -ISCSI_REASON_PROTOCOL_ERROR;
 849 }
 850
 851 static inline struct iscsi_cmnd *__cmnd_find_hash(
 852         struct iscsi_session *session, u32 itt, u32 ttt)
 853 {
 854         struct list_head *head;
 855         struct iscsi_cmnd *cmnd;
 856
 857         head = &session->cmnd_hash[cmnd_hashfn(itt)];
 858
 859         list_for_each_entry(cmnd, head, hash_list_entry) {
 860                 if (cmnd->pdu.bhs.itt == itt) {
 861                         if (ttt != ISCSI_RESERVED_TAG &&
 862                             ttt != cmnd->target_task_tag)
 863                                 continue;
 864                         return cmnd;
 865                 }
 866         }
 867         return NULL;
 868 }
 869
 870 static struct iscsi_cmnd *cmnd_find_hash(struct iscsi_session *session,
 871         u32 itt, u32 ttt)
 872 {
 873         struct iscsi_cmnd *cmnd;
 874
 875         spin_lock(&session->cmnd_hash_lock);
 876         cmnd = __cmnd_find_hash(session, itt, ttt);
 877         spin_unlock(&session->cmnd_hash_lock);
 878
 879         return cmnd;
 880 }
 881
 882 static struct iscsi_cmnd *cmnd_find_hash_get(struct iscsi_session *session,
 883         u32 itt, u32 ttt)
 884 {
 885         struct iscsi_cmnd *cmnd;
 886
 887         spin_lock(&session->cmnd_hash_lock);
 888         cmnd = __cmnd_find_hash(session, itt, ttt);
 889         if (cmnd != NULL) {
 890                 if (unlikely(cmnd_get_check(cmnd)))
 891                         cmnd = NULL;
 892         }
 893         spin_unlock(&session->cmnd_hash_lock);
 894
 895         return cmnd;
 896 }
 897
 898 static int cmnd_insert_hash(struct iscsi_cmnd *cmnd)
 899 {
 900         struct iscsi_session *session = cmnd->conn->session;
 901         struct iscsi_cmnd *tmp;
 902         struct list_head *head;
 903         int err = 0;
 904         u32 itt = cmnd->pdu.bhs.itt;
 905
 906         TRACE_DBG("%p:%x", cmnd, itt);
 907         if (unlikely(itt == ISCSI_RESERVED_TAG)) {
 908                 PRINT_ERROR("%s", "ITT is RESERVED_TAG");
 909                 PRINT_BUFFER("Incorrect BHS", &cmnd->pdu.bhs,
 910                         sizeof(cmnd->pdu.bhs));
 911                 err = -ISCSI_REASON_PROTOCOL_ERROR;
 912                 goto out;
 913         }
 914
 915         spin_lock(&session->cmnd_hash_lock);
 916
 917         head = &session->cmnd_hash[cmnd_hashfn(cmnd->pdu.bhs.itt)];
 918
 919         tmp = __cmnd_find_hash(session, itt, ISCSI_RESERVED_TAG);
 920         if (likely(!tmp)) {
 921                 list_add_tail(&cmnd->hash_list_entry, head);
 922                 cmnd->hashed = 1;
 923         } else {
 924                 PRINT_ERROR("Task %x in progress, cmnd %p", itt, cmnd);
 925                 err = -ISCSI_REASON_TASK_IN_PROGRESS;
 926         }
 927
 928         spin_unlock(&session->cmnd_hash_lock);
 929
 930         if (likely(!err)) {
 931                 spin_lock(&session->sn_lock);
 932                 __update_stat_sn(cmnd);
 933                 err = check_cmd_sn(cmnd);
 934                 spin_unlock(&session->sn_lock);
 935         }
 936
 937 out:
 938         return err;
 939 }
 940
 941 static void cmnd_remove_hash(struct iscsi_cmnd *cmnd)
 942 {
 943         struct iscsi_session *session = cmnd->conn->session;
 944         struct iscsi_cmnd *tmp;
 945
 946         spin_lock(&session->cmnd_hash_lock);
 947
 948         tmp = __cmnd_find_hash(session, cmnd->pdu.bhs.itt, ISCSI_RESERVED_TAG);
 949
 950         if (likely(tmp && tmp == cmnd)) {
 951                 list_del(&cmnd->hash_list_entry);
 952                 cmnd->hashed = 0;
 953         } else {
 954                 PRINT_ERROR("%p:%x not found", cmnd, cmnd_itt(cmnd));
 955         }
 956
 957         spin_unlock(&session->cmnd_hash_lock);
 958         return;
 959 }
 960
 961 static void cmnd_prepare_get_rejected_cmd_data(struct iscsi_cmnd *cmnd)
 962 {
 963         struct iscsi_conn *conn = cmnd->conn;
 964         struct scatterlist *sg = cmnd->sg;
 965         char __user *addr;
 966         u32 size;
 967         unsigned int i;
 968
 969         TRACE_MGMT_DBG("Skipping (%p, %x %x %x %u, %p, scst state %d)", cmnd,
 970                 cmnd_itt(cmnd), cmnd_opcode(cmnd), cmnd_hdr(cmnd)->scb[0],
 971                 cmnd->pdu.datasize, cmnd->scst_cmd, cmnd->scst_state);
 972
 973         iscsi_extracheck_is_rd_thread(conn);
 974
 975         size = cmnd->pdu.datasize;
 976         if (!size)
 977                 return;
 978
 979         if (sg == NULL) {
 980                 /*
 981                  * There are no problems with the safety from concurrent
 982                  * accesses to dummy_page in dummy_sg, since data only
 983                  * will be read and then discarded.
 984                  */
 985                 sg = cmnd->sg = &dummy_sg;
 986                 cmnd->bufflen = PAGE_SIZE;
 987                 cmnd->own_sg = 1;
 988         }
 989
 990         addr = (char __force __user *)(page_address(sg_page(&sg[0])));
 991         sBUG_ON(addr == NULL);
 992         conn->read_size = size;
 993         for (i = 0; size > PAGE_SIZE; i++, size -= cmnd->bufflen) {
 994                 /* We already checked pdu.datasize in check_segment_length() */
 995                 sBUG_ON(i >= ISCSI_CONN_IOV_MAX);
 996                 conn->read_iov[i].iov_base = addr;
 997                 conn->read_iov[i].iov_len = cmnd->bufflen;
 998         }
 999         conn->read_iov[i].iov_base = addr;
1000         conn->read_iov[i].iov_len = size;
1001         conn->read_msg.msg_iov = conn->read_iov;
1002         conn->read_msg.msg_iovlen = ++i;
1003
1004         return;
1005 }
1006
1007 static void iscsi_set_resid(struct iscsi_cmnd *req, struct iscsi_cmnd *rsp,
1008         bool data_used)
1009 {
1010         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1011         struct iscsi_scsi_rsp_hdr *rsp_hdr;
1012         int resid, resp_len, in_resp_len;
1013
1014         if ((req_hdr->flags & ISCSI_CMD_READ) &&
1015             (req_hdr->flags & ISCSI_CMD_WRITE)) {
1016                 rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
1017
1018                 if (data_used) {
1019                         resp_len = req->bufflen;
1020                         if (req->scst_cmd != NULL)
1021                                 in_resp_len = scst_cmd_get_in_bufflen(req->scst_cmd);
1022                         else
1023                                 in_resp_len = 0;
1024                 } else {
1025                         resp_len = 0;
1026                         in_resp_len = 0;
1027                 }
1028
1029                 resid = be32_to_cpu(req_hdr->data_length) - in_resp_len;
1030                 if (resid > 0) {
1031                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
1032                         rsp_hdr->residual_count = cpu_to_be32(resid);
1033                 } else if (resid < 0) {
1034                         resid = -resid;
1035                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
1036                         rsp_hdr->residual_count = cpu_to_be32(resid);
1037                 }
1038
1039                 resid = req->read_size - resp_len;
1040                 if (resid > 0) {
1041                         rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_UNDERFLOW;
1042                         rsp_hdr->bi_residual_count = cpu_to_be32(resid);
1043                 } else if (resid < 0) {
1044                         resid = -resid;
1045                         rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_OVERFLOW;
1046                         rsp_hdr->bi_residual_count = cpu_to_be32(resid);
1047                 }
1048         } else {
1049                 if (data_used)
1050                         resp_len = req->bufflen;
1051                 else
1052                         resp_len = 0;
1053
1054                 resid = req->read_size - resp_len;
1055                 if (resid > 0) {
1056                         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
1057                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
1058                         rsp_hdr->residual_count = cpu_to_be32(resid);
1059                 } else if (resid < 0) {
1060                         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
1061                         resid = -resid;
1062                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
1063                         rsp_hdr->residual_count = cpu_to_be32(resid);
1064                 }
1065         }
1066         return;
1067 }
1068
1069 static void cmnd_reject_scsi_cmd(struct iscsi_cmnd *req)
1070 {
1071         struct iscsi_cmnd *rsp;
1072
1073         TRACE_DBG("%p", req);
1074
1075         sBUG_ON(req->rejected);
1076         req->rejected = 1;
1077         req->reject_reason = ISCSI_REJECT_SCSI_CMD;
1078
1079         rsp = get_rsp_cmnd(req);
1080         if (rsp == NULL) {
1081                 /* That can be true for aborted commands */
1082                 goto out_reject;
1083         }
1084
1085         sBUG_ON(cmnd_opcode(rsp) != ISCSI_OP_SCSI_RSP);
1086
1087         iscsi_set_resid(req, rsp, false);
1088
1089         iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH |
1090                                          ISCSI_INIT_WRITE_WAKE);
1091
1092 out_reject:
1093         cmnd_prepare_get_rejected_cmd_data(req);
1094         return;
1095 }
1096
1097 static int cmnd_prepare_recv_pdu(struct iscsi_conn *conn,
1098         struct iscsi_cmnd *cmd, u32 offset, u32 size)
1099 {
1100         struct scatterlist *sg = cmd->sg;
1101         unsigned int bufflen = cmd->bufflen;
1102         unsigned int idx, i;
1103         char __user *addr;
1104         int res = 0;
1105
1106         TRACE_DBG("%p %u,%u", cmd->sg, offset, size);
1107
1108         iscsi_extracheck_is_rd_thread(conn);
1109
1110         if (unlikely((offset >= bufflen) ||
1111                      (offset + size > bufflen))) {
1112                 PRINT_ERROR("Wrong ltn (%u %u %u)", offset, size, bufflen);
1113                 mark_conn_closed(conn);
1114                 res = -EIO;
1115                 goto out;
1116         }
1117
1118         offset += sg[0].offset;
1119         idx = offset >> PAGE_SHIFT;
1120         offset &= ~PAGE_MASK;
1121
1122         conn->read_msg.msg_iov = conn->read_iov;
1123         conn->read_size = size;
1124
1125         i = 0;
1126         while (1) {
1127                 addr = (char __force __user *)(page_address(sg_page(&sg[idx])));
1128                 sBUG_ON(addr == NULL);
1129                 conn->read_iov[i].iov_base = addr + offset;
1130                 if (offset + size <= PAGE_SIZE) {
1131                         TRACE_DBG("idx=%d, offset=%u, size=%d, addr=%p",
1132                                 idx, offset, size, addr);
1133                         conn->read_iov[i].iov_len = size;
1134                         conn->read_msg.msg_iovlen = ++i;
1135                         break;
1136                 }
1137                 conn->read_iov[i].iov_len = PAGE_SIZE - offset;
1138                 TRACE_DBG("idx=%d, offset=%u, size=%d, iov_len=%zd, addr=%p",
1139                         idx, offset, size, conn->read_iov[i].iov_len, addr);
1140                 size -= conn->read_iov[i].iov_len;
1141                 if (unlikely(++i >= ISCSI_CONN_IOV_MAX)) {
1142                         PRINT_ERROR("Initiator %s violated negotiated "
1143                                 "parameters by sending too much data (size "
1144                                 "left %d)", conn->session->initiator_name,
1145                                 size);
1146                         mark_conn_closed(conn);
1147                         res = -EINVAL;
1148                         break;
1149                 }
1150                 idx++;
1151                 offset = sg[idx].offset;
1152         }
1153         TRACE_DBG("msg_iov=%p, msg_iovlen=%zd",
1154                 conn->read_msg.msg_iov, conn->read_msg.msg_iovlen);
1155
1156 out:
1157         return res;
1158 }
1159
1160 static void send_r2t(struct iscsi_cmnd *req)
1161 {
1162         struct iscsi_session *session = req->conn->session;
1163         struct iscsi_cmnd *rsp;
1164         struct iscsi_r2t_hdr *rsp_hdr;
1165         u32 offset, burst;
1166         LIST_HEAD(send);
1167
1168         if (unlikely(req->tm_aborted)) {
1169                 TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted on R2T "
1170                         "(r2t_length %d, outstanding_r2t %d)", req,
1171                         req->scst_cmd, req->r2t_length, req->outstanding_r2t);
1172                 if (req->outstanding_r2t == 0)
1173                         iscsi_fail_waiting_cmnd(req);
1174                 goto out;
1175         }
1176
1177         /*
1178          * There is no race with data_out_start() and conn_abort(), since
1179          * all functions called from single read thread
1180          */
1181         iscsi_extracheck_is_rd_thread(req->conn);
1182
1183         burst = session->sess_param.max_burst_length;
1184         offset = be32_to_cpu(cmnd_hdr(req)->data_length) - req->r2t_length;
1185
1186         do {
1187                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
1188                 rsp->pdu.bhs.ttt = req->target_task_tag;
1189                 rsp_hdr = (struct iscsi_r2t_hdr *)&rsp->pdu.bhs;
1190                 rsp_hdr->opcode = ISCSI_OP_R2T;
1191                 rsp_hdr->flags = ISCSI_FLG_FINAL;
1192                 rsp_hdr->lun = cmnd_hdr(req)->lun;
1193                 rsp_hdr->itt = cmnd_hdr(req)->itt;
1194                 rsp_hdr->r2t_sn = cpu_to_be32(req->r2t_sn++);
1195                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
1196                 if (req->r2t_length > burst) {
1197                         rsp_hdr->data_length = cpu_to_be32(burst);
1198                         req->r2t_length -= burst;
1199                         offset += burst;
1200                 } else {
1201                         rsp_hdr->data_length = cpu_to_be32(req->r2t_length);
1202                         req->r2t_length = 0;
1203                 }
1204
1205                 TRACE_WRITE("%x %u %u %u %u", cmnd_itt(req),
1206                         be32_to_cpu(rsp_hdr->data_length),
1207                         be32_to_cpu(rsp_hdr->buffer_offset),
1208                         be32_to_cpu(rsp_hdr->r2t_sn), req->outstanding_r2t);
1209
1210                 list_add_tail(&rsp->write_list_entry, &send);
1211
1212                 if (++req->outstanding_r2t >= session->sess_param.max_outstanding_r2t)
1213                         break;
1214
1215         } while (req->r2t_length != 0);
1216
1217         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_WAKE);
1218
1219 out:
1220         return;
1221 }
1222
1223 static int iscsi_pre_exec(struct scst_cmd *scst_cmd)
1224 {
1225         int res = SCST_PREPROCESS_STATUS_SUCCESS;
1226         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
1227                 scst_cmd_get_tgt_priv(scst_cmd);
1228         struct iscsi_cmnd *c, *t;
1229
1230         TRACE_ENTRY();
1231
1232         EXTRACHECKS_BUG_ON(scst_cmd_atomic(scst_cmd));
1233
1234         /* If data digest isn't used this list will be empty */
1235         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
1236                                 rx_ddigest_cmd_list_entry) {
1237                 TRACE_DBG("Checking digest of RX ddigest cmd %p", c);
1238                 if (digest_rx_data(c) != 0) {
1239                         scst_set_cmd_error(scst_cmd,
1240                                 SCST_LOAD_SENSE(iscsi_sense_crc_error));
1241                         res = SCST_PREPROCESS_STATUS_ERROR_SENSE_SET;
1242                         /*
1243                          * The rest of rx_ddigest_cmd_list will be freed
1244                          * in req_cmnd_release()
1245                          */
1246                         goto out;
1247                 }
1248                 cmd_del_from_rx_ddigest_list(c);
1249                 cmnd_put(c);
1250         }
1251
1252 out:
1253         TRACE_EXIT_RES(res);
1254         return res;
1255 }
1256
1257 static int noop_out_start(struct iscsi_cmnd *cmnd)
1258 {
1259         struct iscsi_conn *conn = cmnd->conn;
1260         u32 size, tmp;
1261         int i, err = 0;
1262
1263         TRACE_DBG("%p", cmnd);
1264
1265         iscsi_extracheck_is_rd_thread(conn);
1266
1267         if (unlikely(cmnd_ttt(cmnd) != cpu_to_be32(ISCSI_RESERVED_TAG))) {
1268                 /*
1269                  * We don't request a NOP-Out by sending a NOP-In.
1270                  * See 10.18.2 in the draft 20.
1271                  */
1272                 PRINT_ERROR("Initiator sent command with not RESERVED tag and "
1273                         "TTT %x", cmnd_itt(cmnd));
1274                 err = -ISCSI_REASON_PROTOCOL_ERROR;
1275                 goto out;
1276         }
1277
1278         if (cmnd_itt(cmnd) == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1279                 if (unlikely(!(cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE)))
1280                         PRINT_ERROR("%s", "Initiator sent RESERVED tag for "
1281                                 "non-immediate command");
1282                 spin_lock(&conn->session->sn_lock);
1283                 __update_stat_sn(cmnd);
1284                 err = check_cmd_sn(cmnd);
1285                 spin_unlock(&conn->session->sn_lock);
1286                 if (unlikely(err))
1287                         goto out;
1288         } else {
1289                 err = cmnd_insert_hash(cmnd);
1290                 if (unlikely(err < 0)) {
1291                         PRINT_ERROR("Can't insert in hash: ignore this "
1292                                 "request %x", cmnd_itt(cmnd));
1293                         goto out;
1294                 }
1295         }
1296
1297         size = cmnd->pdu.datasize;
1298
1299         if (size) {
1300                 conn->read_msg.msg_iov = conn->read_iov;
1301                 if (cmnd->pdu.bhs.itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1302                         struct scatterlist *sg;
1303
1304                         cmnd->sg = sg = scst_alloc(size, GFP_KERNEL,
1305                                                 &cmnd->sg_cnt);
1306                         if (sg == NULL) {
1307                                 TRACE(TRACE_OUT_OF_MEM, "Allocating buffer for"
1308                                       " %d NOP-Out payload failed", size);
1309                                 err = -ISCSI_REASON_OUT_OF_RESOURCES;
1310                                 goto out;
1311                         }
1312
1313                         /* We already checked it in check_segment_length() */
1314                         sBUG_ON(cmnd->sg_cnt > (signed)ISCSI_CONN_IOV_MAX);
1315
1316                         cmnd->own_sg = 1;
1317                         cmnd->bufflen = size;
1318
1319                         for (i = 0; i < cmnd->sg_cnt; i++) {
1320                                 conn->read_iov[i].iov_base =
1321                                         (void __force __user *)(page_address(sg_page(&sg[i])));
1322                                 tmp = min_t(u32, size, PAGE_SIZE);
1323                                 conn->read_iov[i].iov_len = tmp;
1324                                 conn->read_size += tmp;
1325                                 size -= tmp;
1326                         }
1327                         sBUG_ON(size != 0);
1328                 } else {
1329                         /*
1330                          * There are no problems with the safety from concurrent
1331                          * accesses to dummy_page, since for ISCSI_RESERVED_TAG
1332                          * the data only read and then discarded.
1333                          */
1334                         for (i = 0; i < (signed)ISCSI_CONN_IOV_MAX; i++) {
1335                                 conn->read_iov[i].iov_base =
1336                                         (void __force __user *)(page_address(dummy_page));
1337                                 tmp = min_t(u32, size, PAGE_SIZE);
1338                                 conn->read_iov[i].iov_len = tmp;
1339                                 conn->read_size += tmp;
1340                                 size -= tmp;
1341                         }
1342
1343                         /* We already checked size in check_segment_length() */
1344                         sBUG_ON(size != 0);
1345                 }
1346
1347                 conn->read_msg.msg_iovlen = i;
1348                 TRACE_DBG("msg_iov=%p, msg_iovlen=%zd", conn->read_msg.msg_iov,
1349                         conn->read_msg.msg_iovlen);
1350         }
1351
1352 out:
1353         return err;
1354 }
1355
1356 static inline u32 get_next_ttt(struct iscsi_conn *conn)
1357 {
1358         u32 ttt;
1359         struct iscsi_session *session = conn->session;
1360
1361         iscsi_extracheck_is_rd_thread(conn);
1362
1363         if (session->next_ttt == ISCSI_RESERVED_TAG)
1364                 session->next_ttt++;
1365         ttt = session->next_ttt++;
1366
1367         return cpu_to_be32(ttt);
1368 }
1369
1370 int cmnd_rx_continue(struct iscsi_cmnd *req)
1371 {
1372         struct iscsi_conn *conn = req->conn;
1373         struct iscsi_session *session = conn->session;
1374         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1375         struct scst_cmd *scst_cmd = req->scst_cmd;
1376         scst_data_direction dir;
1377         int res = 0;
1378
1379         TRACE_ENTRY();
1380
1381         TRACE_DBG("scsi command: %02x", req_hdr->scb[0]);
1382
1383         if (unlikely(req->scst_state != ISCSI_CMD_STATE_AFTER_PREPROC)) {
1384                 TRACE_DBG("req %p is in %x state", req, req->scst_state);
1385                 if (req->scst_state == ISCSI_CMD_STATE_PROCESSED) {
1386                         cmnd_reject_scsi_cmd(req);
1387                         goto out;
1388                 }
1389                 if (unlikely(req->tm_aborted)) {
1390                         TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
1391                                 req->scst_cmd);
1392                         cmnd_prepare_get_rejected_cmd_data(req);
1393                         goto out;
1394                 }
1395                 sBUG();
1396         }
1397
1398         dir = scst_cmd_get_data_direction(scst_cmd);
1399         if (dir & SCST_DATA_WRITE) {
1400                 req->is_unsolicited_data = !(req_hdr->flags & ISCSI_CMD_FINAL);
1401                 req->r2t_length = be32_to_cpu(req_hdr->data_length) -
1402                                         req->pdu.datasize;
1403                 if (req->r2t_length > 0)
1404                         req->data_waiting = 1;
1405         } else {
1406                 if (unlikely(!(req_hdr->flags & ISCSI_CMD_FINAL) ||
1407                              req->pdu.datasize)) {
1408                         PRINT_ERROR("Unexpected unsolicited data (ITT %x "
1409                                 "CDB %x", cmnd_itt(req), req_hdr->scb[0]);
1410                         scst_set_cmd_error(scst_cmd,
1411                                 SCST_LOAD_SENSE(iscsi_sense_unexpected_unsolicited_data));
1412                         if (scst_cmd_get_sense_buffer(scst_cmd) != NULL)
1413                                 create_status_rsp(req, SAM_STAT_CHECK_CONDITION,
1414                                     scst_cmd_get_sense_buffer(scst_cmd),
1415                                     scst_cmd_get_sense_buffer_len(scst_cmd));
1416                         else
1417                                 create_status_rsp(req, SAM_STAT_BUSY, NULL, 0);
1418                         cmnd_reject_scsi_cmd(req);
1419                         goto out;
1420                 }
1421         }
1422
1423         req->target_task_tag = get_next_ttt(conn);
1424         if (dir != SCST_DATA_BIDI) {
1425                 req->sg = scst_cmd_get_sg(scst_cmd);
1426                 req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
1427                 req->bufflen = scst_cmd_get_bufflen(scst_cmd);
1428         } else {
1429                 req->sg = scst_cmd_get_in_sg(scst_cmd);
1430                 req->sg_cnt = scst_cmd_get_in_sg_cnt(scst_cmd);
1431                 req->bufflen = scst_cmd_get_in_bufflen(scst_cmd);
1432         }
1433         if (unlikely(req->r2t_length > req->bufflen)) {
1434                 PRINT_ERROR("req->r2t_length %d > req->bufflen %d",
1435                         req->r2t_length, req->bufflen);
1436                 req->r2t_length = req->bufflen;
1437         }
1438
1439         TRACE_DBG("req=%p, dir=%d, is_unsolicited_data=%d, "
1440                 "r2t_length=%d, bufflen=%d", req, dir,
1441                 req->is_unsolicited_data, req->r2t_length, req->bufflen);
1442
1443         if (unlikely(!session->sess_param.immediate_data &&
1444                      req->pdu.datasize)) {
1445                 PRINT_ERROR("Initiator %s violated negotiated paremeters: "
1446                         "forbidden immediate data sent (ITT %x, op  %x)",
1447                         session->initiator_name, cmnd_itt(req),
1448                         req_hdr->scb[0]);
1449                 res = -EINVAL;
1450                 goto out;
1451         }
1452
1453         if (unlikely(session->sess_param.initial_r2t &&
1454                      !(req_hdr->flags & ISCSI_CMD_FINAL))) {
1455                 PRINT_ERROR("Initiator %s violated negotiated paremeters: "
1456                         "initial R2T is required (ITT %x, op  %x)",
1457                         session->initiator_name, cmnd_itt(req),
1458                         req_hdr->scb[0]);
1459                 res = -EINVAL;
1460                 goto out;
1461         }
1462
1463         if (req->pdu.datasize)
1464                 res = cmnd_prepare_recv_pdu(conn, req, 0, req->pdu.datasize);
1465
1466 out:
1467         /* Aborted commands will be freed in cmnd_rx_end() */
1468         TRACE_EXIT_RES(res);
1469         return res;
1470 }
1471
1472 static int scsi_cmnd_start(struct iscsi_cmnd *req)
1473 {
1474         struct iscsi_conn *conn = req->conn;
1475         struct iscsi_session *session = conn->session;
1476         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1477         struct scst_cmd *scst_cmd;
1478         scst_data_direction dir;
1479         struct iscsi_ahs_hdr *ahdr;
1480         int res = 0;
1481
1482         TRACE_ENTRY();
1483
1484         TRACE_DBG("scsi command: %02x", req_hdr->scb[0]);
1485
1486         TRACE_DBG("Incrementing active_cmds (cmd %p, sess %p, "
1487                 "new value %d)", req, session,
1488                 atomic_read(&session->active_cmds)+1);
1489         atomic_inc(&session->active_cmds);
1490         req->dec_active_cmnds = 1;
1491
1492         scst_cmd = scst_rx_cmd(session->scst_sess,
1493                 (uint8_t *)&req_hdr->lun, sizeof(req_hdr->lun),
1494                 req_hdr->scb, sizeof(req_hdr->scb), SCST_NON_ATOMIC);
1495         if (scst_cmd == NULL) {
1496                 create_status_rsp(req, SAM_STAT_BUSY, NULL, 0);
1497                 cmnd_reject_scsi_cmd(req);
1498                 goto out;
1499         }
1500
1501         req->scst_cmd = scst_cmd;
1502         scst_cmd_set_tag(scst_cmd, req_hdr->itt);
1503         scst_cmd_set_tgt_priv(scst_cmd, req);
1504
1505         if ((req_hdr->flags & ISCSI_CMD_READ) &&
1506             (req_hdr->flags & ISCSI_CMD_WRITE)) {
1507                 int sz = cmnd_read_size(req);
1508                 if (unlikely(sz < 0)) {
1509                         PRINT_ERROR("%s", "BIDI data transfer, but initiator "
1510                                 "not supplied Bidirectional Read Expected Data "
1511                                 "Transfer Length AHS");
1512                         scst_set_cmd_error(scst_cmd,
1513                            SCST_LOAD_SENSE(scst_sense_parameter_value_invalid));
1514                         /*
1515                          * scst_cmd_init_done() will handle commands with
1516                          * set status as preliminary completed
1517                          */
1518                 } else {
1519                         req->read_size = sz;
1520                         dir = SCST_DATA_BIDI;
1521                         scst_cmd_set_expected(scst_cmd, dir, sz);
1522                         scst_cmd_set_expected_in_transfer_len(scst_cmd,
1523                                 be32_to_cpu(req_hdr->data_length));
1524 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
1525                         scst_cmd_set_tgt_need_alloc_data_buf(scst_cmd);
1526 #endif
1527                 }
1528         } else if (req_hdr->flags & ISCSI_CMD_READ) {
1529                 req->read_size = be32_to_cpu(req_hdr->data_length);
1530                 dir = SCST_DATA_READ;
1531                 scst_cmd_set_expected(scst_cmd, dir, req->read_size);
1532 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
1533                 scst_cmd_set_tgt_need_alloc_data_buf(scst_cmd);
1534 #endif
1535         } else if (req_hdr->flags & ISCSI_CMD_WRITE) {
1536                 dir = SCST_DATA_WRITE;
1537                 scst_cmd_set_expected(scst_cmd, dir,
1538                         be32_to_cpu(req_hdr->data_length));
1539         } else {
1540                 dir = SCST_DATA_NONE;
1541                 scst_cmd_set_expected(scst_cmd, dir, 0);
1542         }
1543
1544         switch (req_hdr->flags & ISCSI_CMD_ATTR_MASK) {
1545         case ISCSI_CMD_SIMPLE:
1546                 scst_cmd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1547                 break;
1548         case ISCSI_CMD_HEAD_OF_QUEUE:
1549                 scst_cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1550                 break;
1551         case ISCSI_CMD_ORDERED:
1552                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1553                 break;
1554         case ISCSI_CMD_ACA:
1555                 scst_cmd->queue_type = SCST_CMD_QUEUE_ACA;
1556                 break;
1557         case ISCSI_CMD_UNTAGGED:
1558                 scst_cmd->queue_type = SCST_CMD_QUEUE_UNTAGGED;
1559                 break;
1560         default:
1561                 PRINT_ERROR("Unknown task code %x, use ORDERED instead",
1562                         req_hdr->flags & ISCSI_CMD_ATTR_MASK);
1563                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1564                 break;
1565         }
1566
1567         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
1568         scst_cmd_set_tgt_sn(scst_cmd, req_hdr->cmd_sn);
1569
1570         ahdr = (struct iscsi_ahs_hdr *)req->pdu.ahs;
1571         if (ahdr != NULL) {
1572                 uint8_t *p = (uint8_t *)ahdr;
1573                 int size = 0;
1574                 do {
1575                         int s;
1576
1577                         ahdr = (struct iscsi_ahs_hdr *)p;
1578
1579                         if (ahdr->ahstype == ISCSI_AHSTYPE_CDB) {
1580                                 struct iscsi_cdb_ahdr *eca =
1581                                         (struct iscsi_cdb_ahdr *)ahdr;
1582                                 scst_cmd_set_ext_cdb(scst_cmd, eca->cdb,
1583                                         be16_to_cpu(ahdr->ahslength) - 1);
1584                                 break;
1585                         }
1586                         s = 3 + be16_to_cpu(ahdr->ahslength);
1587                         s = (s + 3) & -4;
1588                         size += s;
1589                         p += s;
1590                 } while (size < req->pdu.ahssize);
1591         }
1592
1593         TRACE_DBG("START Command (tag %d, queue_type %d)",
1594                 req_hdr->itt, scst_cmd->queue_type);
1595         req->scst_state = ISCSI_CMD_STATE_RX_CMD;
1596         conn->rx_task = current;
1597         scst_cmd_init_stage1_done(scst_cmd, SCST_CONTEXT_DIRECT, 0);
1598
1599         if (req->scst_state != ISCSI_CMD_STATE_RX_CMD)
1600                 res = cmnd_rx_continue(req);
1601         else {
1602                 TRACE_DBG("Delaying req %p post processing (scst_state %d)",
1603                         req, req->scst_state);
1604                 res = 1;
1605         }
1606
1607 out:
1608         /* Aborted commands will be freed in cmnd_rx_end() */
1609         TRACE_EXIT_RES(res);
1610         return res;
1611 }
1612
1613 static int data_out_start(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
1614 {
1615         struct iscsi_data_out_hdr *req_hdr =
1616                 (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1617         struct iscsi_cmnd *orig_req = NULL;
1618         u32 offset = be32_to_cpu(req_hdr->buffer_offset);
1619         int res = 0;
1620
1621         TRACE_ENTRY();
1622
1623         /*
1624          * There is no race with send_r2t() and conn_abort(), since
1625          * all functions called from single read thread
1626          */
1627         iscsi_extracheck_is_rd_thread(cmnd->conn);
1628
1629         update_stat_sn(cmnd);
1630
1631         cmnd->cmd_req = orig_req = cmnd_find_hash(conn->session, req_hdr->itt,
1632                                         req_hdr->ttt);
1633         if (unlikely(orig_req == NULL)) {
1634                 /* It might happen if req was aborted and then freed */
1635                 TRACE(TRACE_MGMT_MINOR, "Unable to find scsi task %x %x",
1636                         cmnd_itt(cmnd), cmnd_ttt(cmnd));
1637                 goto out_reject;
1638         }
1639
1640         if (orig_req->is_unsolicited_data) {
1641                 if (unlikely(orig_req->r2t_length < cmnd->pdu.datasize)) {
1642                         PRINT_ERROR("Data size (%d) > R2T length (%d)",
1643                                 cmnd->pdu.datasize, orig_req->r2t_length);
1644                         mark_conn_closed(conn);
1645                         res = -EINVAL;
1646                         goto out;
1647                 }
1648                 orig_req->r2t_length -= cmnd->pdu.datasize;
1649         }
1650
1651         /* Check unsolicited burst data */
1652         if (unlikely((req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) &&
1653                      (orig_req->pdu.bhs.flags & ISCSI_FLG_FINAL))) {
1654                 PRINT_ERROR("Unexpected data from %x %x",
1655                         cmnd_itt(cmnd), cmnd_ttt(cmnd));
1656                 mark_conn_closed(conn);
1657                 res = -EINVAL;
1658                 goto out;
1659         }
1660
1661         TRACE_WRITE("%u %p %p %u %u", req_hdr->ttt, cmnd, orig_req,
1662                 offset, cmnd->pdu.datasize);
1663
1664         res = cmnd_prepare_recv_pdu(conn, orig_req, offset, cmnd->pdu.datasize);
1665
1666 out:
1667         TRACE_EXIT_RES(res);
1668         return res;
1669
1670 out_reject:
1671         sBUG_ON(cmnd->rejected);
1672         cmnd->rejected = 1;
1673         cmnd->reject_reason = ISCSI_REJECT_DATA;
1674         cmnd_prepare_get_rejected_cmd_data(cmnd);
1675         goto out;
1676 }
1677
1678 static void data_out_end(struct iscsi_cmnd *cmnd)
1679 {
1680         struct iscsi_data_out_hdr *req_hdr =
1681                 (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1682         struct iscsi_cmnd *req;
1683
1684         sBUG_ON(cmnd == NULL);
1685         req = cmnd->cmd_req;
1686         sBUG_ON(req == NULL);
1687
1688         TRACE_DBG("cmnd %p, req %p", cmnd, req);
1689
1690         iscsi_extracheck_is_rd_thread(cmnd->conn);
1691
1692         if (!(cmnd->conn->ddigest_type & DIGEST_NONE) &&
1693             !cmnd->ddigest_checked) {
1694                 cmd_add_on_rx_ddigest_list(req, cmnd);
1695                 cmnd_get(cmnd);
1696         }
1697
1698         if (req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1699                 TRACE_DBG("ISCSI_RESERVED_TAG, FINAL %x",
1700                         req_hdr->flags & ISCSI_FLG_FINAL);
1701
1702                 if (req_hdr->flags & ISCSI_FLG_FINAL) {
1703                         req->is_unsolicited_data = 0;
1704                         if (req->pending)
1705                                 goto out_put;
1706                 } else
1707                         goto out_put;
1708         } else {
1709                 TRACE_DBG("FINAL %x, outstanding_r2t %d, r2t_length %d",
1710                         req_hdr->flags & ISCSI_FLG_FINAL,
1711                         req->outstanding_r2t, req->r2t_length);
1712
1713                 if (req_hdr->flags & ISCSI_FLG_FINAL) {
1714                         if (unlikely(req->is_unsolicited_data)) {
1715                                 PRINT_ERROR("Unexpected unsolicited data "
1716                                         "(r2t_length %u, outstanding_r2t %d)",
1717                                         req->r2t_length,
1718                                         req->is_unsolicited_data);
1719                                 mark_conn_closed(req->conn);
1720                                 goto out_put;
1721                         }
1722                         req->outstanding_r2t--;
1723                 } else
1724                         goto out_put;
1725         }
1726
1727         if (req->r2t_length != 0) {
1728                 if (!req->is_unsolicited_data)
1729                         send_r2t(req);
1730         } else
1731                 iscsi_restart_waiting_cmnd(req);
1732
1733 out_put:
1734         cmnd_put(cmnd);
1735         return;
1736 }
1737
1738 static void __cmnd_abort(struct iscsi_cmnd *cmnd)
1739 {
1740         /*
1741          * Here, if cmnd is data_waiting, we should iscsi_fail_waiting_cmnd()
1742          * it. But, since this function can be called from any thread, not only
1743          * from the read one, we at the moment can't do that, because of
1744          * absence of appropriate locking protection. But this isn't a stuff
1745          * for 1.0.0. So, currently a misbehaving initiator, not sending
1746          * data in R2T state for a sharing between targets device, for which
1747          * for some reason an aborting TM command, e.g. TARGET RESET, from
1748          * another initiator is issued, can block response for this TM command
1749          * virtually forever and by this make the issuing initiator eventually
1750          * put the device offline.
1751          *
1752          * ToDo in the next version, possibly a simple connection mutex, taken
1753          * by the read thread before starting any processing and by this
1754          * function, should be sufficient.
1755          */
1756
1757         TRACE_MGMT_DBG("Aborting cmd %p, scst_cmd %p (scst state %x, "
1758                 "ref_cnt %d, itt %x, sn %u, op %x, r2t_len %x, CDB op %x, "
1759                 "size to write %u, is_unsolicited_data %d, "
1760                 "outstanding_r2t %d, data_waiting %d, sess->exp_cmd_sn %u, "
1761                 "conn %p, rd_task %p)", cmnd, cmnd->scst_cmd, cmnd->scst_state,
1762                 atomic_read(&cmnd->ref_cnt), cmnd_itt(cmnd), cmnd->pdu.bhs.sn,
1763                 cmnd_opcode(cmnd), cmnd->r2t_length, cmnd_scsicode(cmnd),
1764                 cmnd_write_size(cmnd), cmnd->is_unsolicited_data,
1765                 cmnd->outstanding_r2t, cmnd->data_waiting,
1766                 cmnd->conn->session->exp_cmd_sn, cmnd->conn,
1767                 cmnd->conn->rd_task);
1768
1769 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
1770         TRACE_MGMT_DBG("net_ref_cnt %d", atomic_read(&cmnd->net_ref_cnt));
1771 #endif
1772
1773         cmnd->tm_aborted = 1;
1774
1775         return;
1776 }
1777
1778 /* Must be called from the read or conn close thread */
1779 static int cmnd_abort(struct iscsi_cmnd *req)
1780 {
1781         struct iscsi_session *session = req->conn->session;
1782         struct iscsi_task_mgt_hdr *req_hdr =
1783                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1784         struct iscsi_cmnd *cmnd;
1785         int err;
1786
1787         req_hdr->ref_cmd_sn = be32_to_cpu(req_hdr->ref_cmd_sn);
1788
1789         if (after(req_hdr->ref_cmd_sn, req_hdr->cmd_sn)) {
1790                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) > CmdSN(%u)",
1791                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn);
1792                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1793                 goto out;
1794         }
1795
1796         cmnd = cmnd_find_hash_get(session, req_hdr->rtt, ISCSI_RESERVED_TAG);
1797         if (cmnd) {
1798                 struct iscsi_conn *conn = cmnd->conn;
1799                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
1800
1801                 if (req_hdr->lun != hdr->lun) {
1802                         PRINT_ERROR("ABORT TASK: LUN mismatch: req LUN "
1803                                     "%llx, cmd LUN %llx, rtt %u",
1804                                     (long long unsigned int)req_hdr->lun,
1805                                     (long long unsigned int)hdr->lun,
1806                                     req_hdr->rtt);
1807                         err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1808                         goto out_put;
1809                 }
1810
1811                 if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
1812                         if (req_hdr->ref_cmd_sn != req_hdr->cmd_sn) {
1813                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != TM "
1814                                         "cmd CmdSN(%u) for immediate command "
1815                                         "%p", req_hdr->ref_cmd_sn,
1816                                         req_hdr->cmd_sn, cmnd);
1817                                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1818                                 goto out_put;
1819                         }
1820                 } else {
1821                         if (req_hdr->ref_cmd_sn != hdr->cmd_sn) {
1822                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != "
1823                                         "CmdSN(%u) for command %p",
1824                                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn,
1825                                         cmnd);
1826                                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1827                                 goto out_put;
1828                         }
1829                 }
1830
1831                 if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
1832                     (req_hdr->cmd_sn == hdr->cmd_sn)) {
1833                         PRINT_ERROR("ABORT TASK: SN mismatch: req SN %x, "
1834                                 "cmd SN %x, rtt %u", req_hdr->cmd_sn,
1835                                 hdr->cmd_sn, req_hdr->rtt);
1836                         err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1837                         goto out_put;
1838                 }
1839
1840                 spin_lock_bh(&conn->cmd_list_lock);
1841                 __cmnd_abort(cmnd);
1842                 spin_unlock_bh(&conn->cmd_list_lock);
1843
1844                 cmnd_put(cmnd);
1845                 err = 0;
1846         } else {
1847                 TRACE_MGMT_DBG("cmd RTT %x not found", req_hdr->rtt);
1848                 err = ISCSI_RESPONSE_UNKNOWN_TASK;
1849         }
1850
1851 out:
1852         return err;
1853
1854 out_put:
1855         cmnd_put(cmnd);
1856         goto out;
1857 }
1858
1859 /* Must be called from the read or conn close thread */
1860 static int target_abort(struct iscsi_cmnd *req, int all)
1861 {
1862         struct iscsi_target *target = req->conn->session->target;
1863         struct iscsi_task_mgt_hdr *req_hdr =
1864                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1865         struct iscsi_session *session;
1866         struct iscsi_conn *conn;
1867         struct iscsi_cmnd *cmnd;
1868
1869         mutex_lock(&target->target_mutex);
1870
1871         list_for_each_entry(session, &target->session_list,
1872                             session_list_entry) {
1873                 list_for_each_entry(conn, &session->conn_list,
1874                                     conn_list_entry) {
1875                         spin_lock_bh(&conn->cmd_list_lock);
1876                         list_for_each_entry(cmnd, &conn->cmd_list,
1877                                             cmd_list_entry) {
1878                                 if (cmnd == req)
1879                                         continue;
1880                                 if (all)
1881                                         __cmnd_abort(cmnd);
1882                                 else if (req_hdr->lun == cmnd_hdr(cmnd)->lun)
1883                                         __cmnd_abort(cmnd);
1884                         }
1885                         spin_unlock_bh(&conn->cmd_list_lock);
1886                 }
1887         }
1888
1889         mutex_unlock(&target->target_mutex);
1890         return 0;
1891 }
1892
1893 /* Must be called from the read or conn close thread */
1894 static void task_set_abort(struct iscsi_cmnd *req)
1895 {
1896         struct iscsi_session *session = req->conn->session;
1897         struct iscsi_task_mgt_hdr *req_hdr =
1898                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1899         struct iscsi_target *target = session->target;
1900         struct iscsi_conn *conn;
1901         struct iscsi_cmnd *cmnd;
1902
1903         mutex_lock(&target->target_mutex);
1904
1905         list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
1906                 spin_lock_bh(&conn->cmd_list_lock);
1907                 list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1908                         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
1909                         if (cmnd == req)
1910                                 continue;
1911                         if (req_hdr->lun != hdr->lun)
1912                                 continue;
1913                         if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
1914                             req_hdr->cmd_sn == hdr->cmd_sn)
1915                                 continue;
1916                         __cmnd_abort(cmnd);
1917                 }
1918                 spin_unlock_bh(&conn->cmd_list_lock);
1919         }
1920
1921         mutex_unlock(&target->target_mutex);
1922         return;
1923 }
1924
1925 /* Must be called from the read or conn close thread */
1926 void conn_abort(struct iscsi_conn *conn)
1927 {
1928         struct iscsi_cmnd *cmnd;
1929
1930         TRACE_MGMT_DBG("Aborting conn %p", conn);
1931
1932         iscsi_extracheck_is_rd_thread(conn);
1933
1934         spin_lock_bh(&conn->cmd_list_lock);
1935 again:
1936         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1937                 __cmnd_abort(cmnd);
1938                 if (cmnd->data_waiting) {
1939                         if (!cmnd_get_check(cmnd)) {
1940                                 spin_unlock_bh(&conn->cmd_list_lock);
1941
1942                                 /* ToDo: this is racy for MC/S */
1943                                 TRACE_MGMT_DBG("Restarting data waiting cmd "
1944                                         "%p", cmnd);
1945                                 iscsi_fail_waiting_cmnd(cmnd);
1946
1947                                 cmnd_put(cmnd);
1948
1949                                 /*
1950                                  * We are in the read thread, so we may not
1951                                  * worry that after cmnd release conn gets
1952                                  * released as well.
1953                                  */
1954                                 spin_lock_bh(&conn->cmd_list_lock);
1955                                 goto again;
1956                         }
1957                 }
1958         }
1959         spin_unlock_bh(&conn->cmd_list_lock);
1960
1961         return;
1962 }
1963
1964 static void execute_task_management(struct iscsi_cmnd *req)
1965 {
1966         struct iscsi_conn *conn = req->conn;
1967         struct iscsi_session *sess = conn->session;
1968         struct iscsi_task_mgt_hdr *req_hdr =
1969                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1970         int rc, status, function = req_hdr->function & ISCSI_FUNCTION_MASK;
1971         struct scst_rx_mgmt_params params;
1972
1973         TRACE((function == ISCSI_FUNCTION_ABORT_TASK) ?
1974                         TRACE_MGMT_MINOR : TRACE_MGMT,
1975                 "TM fn %d", function);
1976
1977         TRACE_MGMT_DBG("TM req %p, itt %x, rtt %x, sn %u, con %p", req,
1978                 cmnd_itt(req), req_hdr->rtt, req_hdr->cmd_sn, conn);
1979
1980         iscsi_extracheck_is_rd_thread(conn);
1981
1982         spin_lock(&sess->sn_lock);
1983         sess->tm_active++;
1984         sess->tm_sn = req_hdr->cmd_sn;
1985         if (sess->tm_rsp != NULL) {
1986                 struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
1987
1988                 TRACE(TRACE_MGMT_MINOR, "Dropping delayed TM rsp %p", tm_rsp);
1989
1990                 sess->tm_rsp = NULL;
1991                 sess->tm_active--;
1992
1993                 spin_unlock(&sess->sn_lock);
1994
1995                 sBUG_ON(sess->tm_active < 0);
1996
1997                 rsp_cmnd_release(tm_rsp);
1998         } else
1999                 spin_unlock(&sess->sn_lock);
2000
2001         memset(&params, 0, sizeof(params));
2002         params.atomic = SCST_NON_ATOMIC;
2003         params.tgt_priv = req;
2004
2005         if ((function != ISCSI_FUNCTION_ABORT_TASK) &&
2006             (req_hdr->rtt != ISCSI_RESERVED_TAG)) {
2007                 PRINT_ERROR("Invalid RTT %x (TM fn %d)", req_hdr->rtt,
2008                         function);
2009                 rc = -1;
2010                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2011                 goto reject;
2012         }
2013
2014         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
2015
2016         switch (function) {
2017         case ISCSI_FUNCTION_ABORT_TASK:
2018                 rc = -1;
2019                 status = cmnd_abort(req);
2020                 if (status == 0) {
2021                         params.fn = SCST_ABORT_TASK;
2022                         params.tag = req_hdr->rtt;
2023                         params.tag_set = 1;
2024                         params.lun = (uint8_t *)&req_hdr->lun;
2025                         params.lun_len = sizeof(req_hdr->lun);
2026                         params.lun_set = 1;
2027                         params.cmd_sn = req_hdr->cmd_sn;
2028                         params.cmd_sn_set = 1;
2029                         rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2030                                 &params);
2031                         status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2032                 }
2033                 break;
2034         case ISCSI_FUNCTION_ABORT_TASK_SET:
2035                 task_set_abort(req);
2036                 params.fn = SCST_ABORT_TASK_SET;
2037                 params.lun = (uint8_t *)&req_hdr->lun;
2038                 params.lun_len = sizeof(req_hdr->lun);
2039                 params.lun_set = 1;
2040                 params.cmd_sn = req_hdr->cmd_sn;
2041                 params.cmd_sn_set = 1;
2042                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2043                         &params);
2044                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2045                 break;
2046         case ISCSI_FUNCTION_CLEAR_TASK_SET:
2047                 task_set_abort(req);
2048                 params.fn = SCST_CLEAR_TASK_SET;
2049                 params.lun = (uint8_t *)&req_hdr->lun;
2050                 params.lun_len = sizeof(req_hdr->lun);
2051                 params.lun_set = 1;
2052                 params.cmd_sn = req_hdr->cmd_sn;
2053                 params.cmd_sn_set = 1;
2054                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2055                         &params);
2056                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2057                 break;
2058         case ISCSI_FUNCTION_CLEAR_ACA:
2059                 params.fn = SCST_CLEAR_ACA;
2060                 params.lun = (uint8_t *)&req_hdr->lun;
2061                 params.lun_len = sizeof(req_hdr->lun);
2062                 params.lun_set = 1;
2063                 params.cmd_sn = req_hdr->cmd_sn;
2064                 params.cmd_sn_set = 1;
2065                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2066                         &params);
2067                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2068                 break;
2069         case ISCSI_FUNCTION_TARGET_COLD_RESET:
2070         case ISCSI_FUNCTION_TARGET_WARM_RESET:
2071                 target_abort(req, 1);
2072                 params.fn = SCST_TARGET_RESET;
2073                 params.cmd_sn = req_hdr->cmd_sn;
2074                 params.cmd_sn_set = 1;
2075                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2076                         &params);
2077                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2078                 break;
2079         case ISCSI_FUNCTION_LOGICAL_UNIT_RESET:
2080                 target_abort(req, 0);
2081                 params.fn = SCST_LUN_RESET;
2082                 params.lun = (uint8_t *)&req_hdr->lun;
2083                 params.lun_len = sizeof(req_hdr->lun);
2084                 params.lun_set = 1;
2085                 params.cmd_sn = req_hdr->cmd_sn;
2086                 params.cmd_sn_set = 1;
2087                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2088                         &params);
2089                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2090                 break;
2091         case ISCSI_FUNCTION_TASK_REASSIGN:
2092                 rc = -1;
2093                 status = ISCSI_RESPONSE_ALLEGIANCE_REASSIGNMENT_UNSUPPORTED;
2094                 break;
2095         default:
2096                 PRINT_ERROR("Unknown TM function %d", function);
2097                 rc = -1;
2098                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2099                 break;
2100         }
2101
2102 reject:
2103         if (rc != 0)
2104                 iscsi_send_task_mgmt_resp(req, status);
2105
2106         return;
2107 }
2108
2109 static void noop_out_exec(struct iscsi_cmnd *req)
2110 {
2111         struct iscsi_cmnd *rsp;
2112         struct iscsi_nop_in_hdr *rsp_hdr;
2113
2114         TRACE_DBG("%p", req);
2115
2116         if (cmnd_itt(req) != cpu_to_be32(ISCSI_RESERVED_TAG)) {
2117                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
2118
2119                 rsp_hdr = (struct iscsi_nop_in_hdr *)&rsp->pdu.bhs;
2120                 rsp_hdr->opcode = ISCSI_OP_NOOP_IN;
2121                 rsp_hdr->flags = ISCSI_FLG_FINAL;
2122                 rsp_hdr->itt = req->pdu.bhs.itt;
2123                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
2124
2125                 if (req->pdu.datasize)
2126                         sBUG_ON(req->sg == NULL);
2127                 else
2128                         sBUG_ON(req->sg != NULL);
2129
2130                 if (req->sg) {
2131                         rsp->sg = req->sg;
2132                         rsp->sg_cnt = req->sg_cnt;
2133                         rsp->bufflen = req->bufflen;
2134                 }
2135
2136                 /* We already checked it in check_segment_length() */
2137                 sBUG_ON(get_pgcnt(req->pdu.datasize, 0) > ISCSI_CONN_IOV_MAX);
2138
2139                 rsp->pdu.datasize = req->pdu.datasize;
2140                 iscsi_cmnd_init_write(rsp,
2141                         ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2142                 req_cmnd_release(req);
2143         } else
2144                 cmnd_put(req);
2145         return;
2146 }
2147
2148 static void logout_exec(struct iscsi_cmnd *req)
2149 {
2150         struct iscsi_logout_req_hdr *req_hdr;
2151         struct iscsi_cmnd *rsp;
2152         struct iscsi_logout_rsp_hdr *rsp_hdr;
2153
2154         PRINT_INFO("Logout received from initiator %s",
2155                 req->conn->session->initiator_name);
2156         TRACE_DBG("%p", req);
2157
2158         req_hdr = (struct iscsi_logout_req_hdr *)&req->pdu.bhs;
2159         rsp = iscsi_cmnd_create_rsp_cmnd(req);
2160         rsp_hdr = (struct iscsi_logout_rsp_hdr *)&rsp->pdu.bhs;
2161         rsp_hdr->opcode = ISCSI_OP_LOGOUT_RSP;
2162         rsp_hdr->flags = ISCSI_FLG_FINAL;
2163         rsp_hdr->itt = req_hdr->itt;
2164         rsp->should_close_conn = 1;
2165         iscsi_cmnd_init_write(rsp,
2166                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2167         req_cmnd_release(req);
2168         return;
2169 }
2170
2171 static void iscsi_cmnd_exec(struct iscsi_cmnd *cmnd)
2172 {
2173         TRACE_ENTRY();
2174
2175         TRACE_DBG("%p,%x,%u", cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn);
2176
2177         iscsi_extracheck_is_rd_thread(cmnd->conn);
2178
2179         if (unlikely(cmnd->tm_aborted)) {
2180                 TRACE_MGMT_DBG("cmnd %p (scst_cmd %p) aborted", cmnd,
2181                         cmnd->scst_cmd);
2182                 req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
2183                 goto out;
2184         }
2185
2186         if (unlikely(cmnd->rejected))
2187                 goto out_rejected;
2188
2189         switch (cmnd_opcode(cmnd)) {
2190         case ISCSI_OP_SCSI_CMD:
2191                 if (cmnd->r2t_length != 0) {
2192                         if (!cmnd->is_unsolicited_data) {
2193                                 send_r2t(cmnd);
2194                                 break;
2195                         }
2196                 } else
2197                         iscsi_restart_cmnd(cmnd);
2198                 break;
2199         case ISCSI_OP_NOOP_OUT:
2200                 noop_out_exec(cmnd);
2201                 break;
2202         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2203                 execute_task_management(cmnd);
2204                 break;
2205         case ISCSI_OP_LOGOUT_CMD:
2206                 logout_exec(cmnd);
2207                 break;
2208         default:
2209                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2210                 req_cmnd_release(cmnd);
2211                 break;
2212         }
2213 out:
2214         TRACE_EXIT();
2215         return;
2216
2217 out_rejected:
2218         TRACE_MGMT_DBG("Rejected cmd %p (reason %d)", cmnd,
2219                 cmnd->reject_reason);
2220         switch (cmnd->reject_reason) {
2221         default:
2222                 PRINT_ERROR("Unexpected reject reason %d",
2223                             cmnd->reject_reason);
2224                 /* go through */
2225         case ISCSI_REJECT_SCSI_CMD:
2226                 req_cmnd_release(cmnd);
2227                 break;
2228         }
2229         goto out;
2230 }
2231
2232 /*
2233  * Note: the code belows passes a kernel space pointer (&opt) to setsockopt()
2234  * while the declaration of setsockopt specifies that it expects a user space
2235  * pointer. This seems to work fine, and this approach is also used in some
2236  * other parts of the Linux kernel (see e.g. fs/ocfs2/cluster/tcp.c).
2237  */
2238 static void set_cork(struct socket *sock, int on)
2239 {
2240         int opt = on;
2241         mm_segment_t oldfs;
2242
2243         oldfs = get_fs();
2244         set_fs(get_ds());
2245         sock->ops->setsockopt(sock, SOL_TCP, TCP_CORK,
2246                               (void __force __user *)&opt, sizeof(opt));
2247         set_fs(oldfs);
2248         return;
2249 }
2250
2251 void cmnd_tx_start(struct iscsi_cmnd *cmnd)
2252 {
2253         struct iscsi_conn *conn = cmnd->conn;
2254
2255         TRACE_DBG("conn %p, cmnd %p, opcode %x", conn, cmnd, cmnd_opcode(cmnd));
2256         iscsi_cmnd_set_length(&cmnd->pdu);
2257
2258         iscsi_extracheck_is_wr_thread(conn);
2259
2260         set_cork(conn->sock, 1);
2261
2262         conn->write_iop = conn->write_iov;
2263         conn->write_iop->iov_base = (void __force __user *)(&cmnd->pdu.bhs);
2264         conn->write_iop->iov_len = sizeof(cmnd->pdu.bhs);
2265         conn->write_iop_used = 1;
2266         conn->write_size = sizeof(cmnd->pdu.bhs) + cmnd->pdu.datasize;
2267         conn->write_offset = 0;
2268
2269         switch (cmnd_opcode(cmnd)) {
2270         case ISCSI_OP_NOOP_IN:
2271                 cmnd_set_sn(cmnd, 1);
2272                 break;
2273         case ISCSI_OP_SCSI_RSP:
2274                 cmnd_set_sn(cmnd, 1);
2275                 break;
2276         case ISCSI_OP_SCSI_TASK_MGT_RSP:
2277                 cmnd_set_sn(cmnd, 1);
2278                 break;
2279         case ISCSI_OP_TEXT_RSP:
2280                 cmnd_set_sn(cmnd, 1);
2281                 break;
2282         case ISCSI_OP_SCSI_DATA_IN:
2283         {
2284                 struct iscsi_data_in_hdr *rsp =
2285                         (struct iscsi_data_in_hdr *)&cmnd->pdu.bhs;
2286                 u32 offset = cpu_to_be32(rsp->buffer_offset);
2287
2288                 TRACE_DBG("cmnd %p, offset %u, datasize %u, bufflen %u", cmnd,
2289                         offset, cmnd->pdu.datasize, cmnd->bufflen);
2290
2291                 sBUG_ON(offset > cmnd->bufflen);
2292                 sBUG_ON(offset + cmnd->pdu.datasize > cmnd->bufflen);
2293
2294                 conn->write_offset = offset;
2295
2296                 cmnd_set_sn(cmnd, (rsp->flags & ISCSI_FLG_FINAL) ? 1 : 0);
2297                 break;
2298         }
2299         case ISCSI_OP_LOGOUT_RSP:
2300                 cmnd_set_sn(cmnd, 1);
2301                 break;
2302         case ISCSI_OP_R2T:
2303                 cmnd->pdu.bhs.sn = cmnd_set_sn(cmnd, 0);
2304                 break;
2305         case ISCSI_OP_ASYNC_MSG:
2306                 cmnd_set_sn(cmnd, 1);
2307                 break;
2308         case ISCSI_OP_REJECT:
2309                 cmnd_set_sn(cmnd, 1);
2310                 break;
2311         default:
2312                 PRINT_ERROR("Unexpected cmnd op %x", cmnd_opcode(cmnd));
2313                 break;
2314         }
2315
2316         iscsi_dump_pdu(&cmnd->pdu);
2317         return;
2318 }
2319
2320 void cmnd_tx_end(struct iscsi_cmnd *cmnd)
2321 {
2322         struct iscsi_conn *conn = cmnd->conn;
2323
2324         TRACE_DBG("%p:%x (should_close_conn %d, should_close_all_conn %d)",
2325                 cmnd, cmnd_opcode(cmnd), cmnd->should_close_conn,
2326                 cmnd->should_close_all_conn);
2327
2328         switch (cmnd_opcode(cmnd)) {
2329         case ISCSI_OP_NOOP_IN:
2330         case ISCSI_OP_SCSI_RSP:
2331         case ISCSI_OP_SCSI_TASK_MGT_RSP:
2332         case ISCSI_OP_TEXT_RSP:
2333         case ISCSI_OP_R2T:
2334         case ISCSI_OP_ASYNC_MSG:
2335         case ISCSI_OP_REJECT:
2336         case ISCSI_OP_SCSI_DATA_IN:
2337         case ISCSI_OP_LOGOUT_RSP:
2338                 break;
2339         default:
2340                 PRINT_CRIT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2341                 sBUG();
2342                 break;
2343         }
2344
2345         if (unlikely(cmnd->should_close_conn)) {
2346                 if (cmnd->should_close_all_conn) {
2347                         PRINT_INFO("Closing all connections for target %x at "
2348                                 "initiator's %s request",
2349                                 cmnd->conn->session->target->tid,
2350                                 conn->session->initiator_name);
2351                         target_del_all_sess(cmnd->conn->session->target, 0);
2352                 } else {
2353                         PRINT_INFO("Closing connection at initiator's %s "
2354                                 "request", conn->session->initiator_name);
2355                         mark_conn_closed(conn);
2356                 }
2357         }
2358
2359         set_cork(cmnd->conn->sock, 0);
2360         return;
2361 }
2362
2363 /*
2364  * Push the command for execution. This functions reorders the commands.
2365  * Called from the read thread.
2366  */
2367 static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd)
2368 {
2369         struct iscsi_session *session = cmnd->conn->session;
2370         struct list_head *entry;
2371         u32 cmd_sn;
2372
2373         TRACE_DBG("%p:%x %u,%u",
2374                 cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn,
2375                 session->exp_cmd_sn);
2376
2377         iscsi_extracheck_is_rd_thread(cmnd->conn);
2378
2379         sBUG_ON(cmnd->parent_req != NULL);
2380
2381         if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
2382                 TRACE_DBG("Immediate cmd %p (cmd_sn %u)", cmnd,
2383                         cmnd->pdu.bhs.sn);
2384                 iscsi_cmnd_exec(cmnd);
2385                 goto out;
2386         }
2387
2388         spin_lock(&session->sn_lock);
2389
2390         cmd_sn = cmnd->pdu.bhs.sn;
2391         if (cmd_sn == session->exp_cmd_sn) {
2392                 while (1) {
2393                         session->exp_cmd_sn = ++cmd_sn;
2394
2395                         if (unlikely(session->tm_active > 0)) {
2396                                 if (before(cmd_sn, session->tm_sn)) {
2397                                         struct iscsi_conn *conn = cmnd->conn;
2398
2399                                         spin_unlock(&session->sn_lock);
2400
2401                                         spin_lock_bh(&conn->cmd_list_lock);
2402                                         __cmnd_abort(cmnd);
2403                                         spin_unlock_bh(&conn->cmd_list_lock);
2404
2405                                         spin_lock(&session->sn_lock);
2406                                 }
2407                                 iscsi_check_send_delayed_tm_resp(session);
2408                         }
2409
2410                         spin_unlock(&session->sn_lock);
2411
2412                         iscsi_cmnd_exec(cmnd);
2413
2414                         spin_lock(&session->sn_lock);
2415
2416                         if (list_empty(&session->pending_list))
2417                                 break;
2418                         cmnd = list_entry(session->pending_list.next,
2419                                           struct iscsi_cmnd,
2420                                           pending_list_entry);
2421                         if (cmnd->pdu.bhs.sn != cmd_sn)
2422                                 break;
2423
2424                         list_del(&cmnd->pending_list_entry);
2425                         cmnd->pending = 0;
2426
2427                         TRACE_DBG("Processing pending cmd %p (cmd_sn %u)",
2428                                 cmnd, cmd_sn);
2429                 }
2430         } else {
2431                 int drop = 0;
2432
2433                 TRACE_DBG("Pending cmd %p (cmd_sn %u, exp_cmd_sn %u)",
2434                         cmnd, cmd_sn, session->exp_cmd_sn);
2435
2436                 /*
2437                  * iSCSI RFC 3720: "The target MUST silently ignore any
2438                  * non-immediate command outside of [from ExpCmdSN to MaxCmdSN
2439                  * inclusive] range". But we won't honor the MaxCmdSN
2440                  * requirement, because, since we adjust MaxCmdSN from the
2441                  * separate write thread, rarery it is possible that initiator
2442                  * can legally send command with CmdSN>MaxSN. But it won't
2443                  * hurt anything, in the worst case it will lead to
2444                  * additional QUEUE FULL status.
2445                  */
2446
2447                 if (unlikely(before(cmd_sn, session->exp_cmd_sn))) {
2448                         PRINT_ERROR("Unexpected cmd_sn (%u,%u)", cmd_sn,
2449                                 session->exp_cmd_sn);
2450                         drop = 1;
2451                 }
2452
2453 #if 0
2454                 if (unlikely(after(cmd_sn, session->exp_cmd_sn +
2455                                         iscsi_get_allowed_cmds(session)))) {
2456                         TRACE_MGMT_DBG("Too large cmd_sn %u (exp_cmd_sn %u, "
2457                                 "max_sn %u)", cmd_sn, session->exp_cmd_sn,
2458                                 iscsi_get_allowed_cmds(session));
2459                 }
2460 #endif
2461
2462                 spin_unlock(&session->sn_lock);
2463
2464                 if (unlikely(drop)) {
2465                         req_cmnd_release_force(cmnd,
2466                                                ISCSI_FORCE_RELEASE_WRITE);
2467                         goto out;
2468                 }
2469
2470                 if (unlikely(cmnd->tm_aborted)) {
2471                         struct iscsi_cmnd *tm_clone;
2472
2473                         TRACE_MGMT_DBG("Pending aborted cmnd %p, creating TM "
2474                                 "clone (scst cmd %p, state %d)", cmnd,
2475                                 cmnd->scst_cmd, cmnd->scst_state);
2476
2477                         tm_clone = cmnd_alloc(cmnd->conn, NULL);
2478                         if (tm_clone != NULL) {
2479                                 tm_clone->tm_aborted = 1;
2480                                 tm_clone->pdu = cmnd->pdu;
2481
2482                                 TRACE_MGMT_DBG("TM clone %p created",
2483                                                tm_clone);
2484
2485                                 iscsi_cmnd_exec(cmnd);
2486                                 cmnd = tm_clone;
2487                         } else
2488                                 PRINT_ERROR("%s", "Unable to create TM clone");
2489                 }
2490
2491                 spin_lock(&session->sn_lock);
2492                 list_for_each(entry, &session->pending_list) {
2493                         struct iscsi_cmnd *tmp =
2494                                 list_entry(entry, struct iscsi_cmnd,
2495                                            pending_list_entry);
2496                         if (before(cmd_sn, tmp->pdu.bhs.sn))
2497                                 break;
2498                 }
2499                 list_add_tail(&cmnd->pending_list_entry, entry);
2500                 cmnd->pending = 1;
2501         }
2502
2503         spin_unlock(&session->sn_lock);
2504 out:
2505         return;
2506 }
2507
2508 static int check_segment_length(struct iscsi_cmnd *cmnd)
2509 {
2510         struct iscsi_conn *conn = cmnd->conn;
2511         struct iscsi_session *session = conn->session;
2512
2513         if (unlikely(cmnd->pdu.datasize > session->sess_param.max_recv_data_length)) {
2514                 PRINT_ERROR("Initiator %s violated negotiated parameters: "
2515                         "data too long (ITT %x, datasize %u, "
2516                         "max_recv_data_length %u", session->initiator_name,
2517                         cmnd_itt(cmnd), cmnd->pdu.datasize,
2518                         session->sess_param.max_recv_data_length);
2519                 mark_conn_closed(conn);
2520                 return -EINVAL;
2521         }
2522         return 0;
2523 }
2524
2525 int cmnd_rx_start(struct iscsi_cmnd *cmnd)
2526 {
2527         struct iscsi_conn *conn = cmnd->conn;
2528         int res, rc;
2529
2530         iscsi_dump_pdu(&cmnd->pdu);
2531
2532         res = check_segment_length(cmnd);
2533         if (res != 0)
2534                 goto out;
2535
2536         switch (cmnd_opcode(cmnd)) {
2537         case ISCSI_OP_NOOP_OUT:
2538                 rc = noop_out_start(cmnd);
2539                 break;
2540         case ISCSI_OP_SCSI_CMD:
2541                 rc = cmnd_insert_hash(cmnd);
2542                 if (likely(rc == 0)) {
2543                         res = scsi_cmnd_start(cmnd);
2544                         if (unlikely(res != 0))
2545                                 goto out;
2546                 }
2547                 break;
2548         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2549                 rc = cmnd_insert_hash(cmnd);
2550                 break;
2551         case ISCSI_OP_SCSI_DATA_OUT:
2552                 res = data_out_start(conn, cmnd);
2553                 rc = 0; /* to avoid compiler warning */
2554                 if (unlikely(res != 0))
2555                         goto out;
2556                 break;
2557         case ISCSI_OP_LOGOUT_CMD:
2558                 rc = cmnd_insert_hash(cmnd);
2559                 break;
2560         case ISCSI_OP_TEXT_CMD:
2561         case ISCSI_OP_SNACK_CMD:
2562                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2563                 break;
2564         default:
2565                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2566                 break;
2567         }
2568
2569         if (unlikely(rc < 0)) {
2570                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
2571                 PRINT_ERROR("Error %d (iSCSI opcode %x, ITT %x, op %x)", rc,
2572                         cmnd_opcode(cmnd), cmnd_itt(cmnd),
2573                         (cmnd_opcode(cmnd) == ISCSI_OP_SCSI_CMD ?
2574                                 hdr->scb[0] : -1));
2575                 iscsi_cmnd_reject(cmnd, -rc);
2576         }
2577
2578 out:
2579         TRACE_EXIT_RES(res);
2580         return res;
2581 }
2582
2583 void cmnd_rx_end(struct iscsi_cmnd *cmnd)
2584 {
2585         TRACE_ENTRY();
2586
2587         TRACE_DBG("%p:%x", cmnd, cmnd_opcode(cmnd));
2588
2589         if (unlikely(cmnd->rejected))
2590                 goto out_rejected;
2591
2592 cont:
2593         switch (cmnd_opcode(cmnd)) {
2594         case ISCSI_OP_SCSI_CMD:
2595         case ISCSI_OP_NOOP_OUT:
2596         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2597         case ISCSI_OP_LOGOUT_CMD:
2598                 iscsi_session_push_cmnd(cmnd);
2599                 break;
2600         case ISCSI_OP_SCSI_DATA_OUT:
2601                 data_out_end(cmnd);
2602                 break;
2603         default:
2604                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2605                 req_cmnd_release(cmnd);
2606                 break;
2607         }
2608
2609 out:
2610         TRACE_EXIT();
2611         return;
2612
2613 out_rejected:
2614         switch (cmnd->reject_reason) {
2615         default:
2616                 PRINT_ERROR("Unexpected reject reason %d",
2617                             cmnd->reject_reason);
2618                 /* go through */
2619         case ISCSI_REJECT_CMD:
2620         case ISCSI_REJECT_DATA:
2621                 req_cmnd_release(cmnd);
2622                 break;
2623         case ISCSI_REJECT_SCSI_CMD:
2624                 goto cont;
2625         }
2626         goto out;
2627 }
2628
2629 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
2630 static int iscsi_alloc_data_buf(struct scst_cmd *cmd)
2631 {
2632         /*
2633          * sock->ops->sendpage() is async zero copy operation,
2634          * so we must be sure not to free and reuse
2635          * the command's buffer before the sending was completed
2636          * by the network layers. It is possible only if we
2637          * don't use SGV cache.
2638          */
2639         EXTRACHECKS_BUG_ON(!(scst_cmd_get_data_direction(cmd) & SCST_DATA_READ));
2640         scst_cmd_set_no_sgv(cmd);
2641         return 1;
2642 }
2643 #endif
2644
2645 static inline void iscsi_set_state_wake_up(struct iscsi_cmnd *req,
2646         int new_state)
2647 {
2648         if (req->conn->rx_task == current)
2649                 req->scst_state = new_state;
2650         else {
2651                 /*
2652                  * We wait for the state change without any protection, so
2653                  * without cmnd_get() it is possible that req will die
2654                  * "immediately" after the state assignment and
2655                  * iscsi_make_conn_rd_active() will operate on dead data.
2656                  * We use the ordered version of cmnd_get(), because "get"
2657                  * must be done before the state assignment.
2658                  */
2659                 cmnd_get_ordered(req);
2660                 req->scst_state = new_state;
2661                 iscsi_make_conn_rd_active(req->conn);
2662                 if (unlikely(req->conn->closing)) {
2663                         TRACE_DBG("Waiking up closing conn %p", req->conn);
2664                         wake_up(&req->conn->read_state_waitQ);
2665                 }
2666                 cmnd_put(req);
2667         }
2668         return;
2669 }
2670
2671 static void iscsi_preprocessing_done(struct scst_cmd *scst_cmd)
2672 {
2673         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
2674                                 scst_cmd_get_tgt_priv(scst_cmd);
2675
2676         TRACE_DBG("req %p", req);
2677
2678         EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_RX_CMD);
2679
2680         iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_AFTER_PREPROC);
2681         return;
2682 }
2683
2684 /*
2685  * No locks.
2686  *
2687  * IMPORTANT! Connection conn must be protected by additional conn_get()
2688  * upon entrance in this function, because otherwise it could be destroyed
2689  * inside as a result of iscsi_send(), which releases sent commands.
2690  */
2691 static void iscsi_try_local_processing(struct iscsi_conn *conn)
2692 {
2693         int local;
2694
2695         TRACE_ENTRY();
2696
2697         spin_lock_bh(&iscsi_wr_lock);
2698         switch (conn->wr_state) {
2699         case ISCSI_CONN_WR_STATE_IN_LIST:
2700                 list_del(&conn->wr_list_entry);
2701                 /* go through */
2702         case ISCSI_CONN_WR_STATE_IDLE:
2703 #ifdef CONFIG_SCST_EXTRACHECKS
2704                 conn->wr_task = current;
2705 #endif
2706                 conn->wr_state = ISCSI_CONN_WR_STATE_PROCESSING;
2707                 conn->wr_space_ready = 0;
2708                 local = 1;
2709                 break;
2710         default:
2711                 local = 0;
2712                 break;
2713         }
2714         spin_unlock_bh(&iscsi_wr_lock);
2715
2716         if (local) {
2717                 int rc = 1;
2718
2719                 if (test_write_ready(conn))
2720                         rc = iscsi_send(conn);
2721
2722                 spin_lock_bh(&iscsi_wr_lock);
2723 #ifdef CONFIG_SCST_EXTRACHECKS
2724                 conn->wr_task = NULL;
2725 #endif
2726                 if ((rc <= 0) || test_write_ready(conn)) {
2727                         list_add_tail(&conn->wr_list_entry, &iscsi_wr_list);
2728                         conn->wr_state = ISCSI_CONN_WR_STATE_IN_LIST;
2729                         wake_up(&iscsi_wr_waitQ);
2730                 } else
2731                         conn->wr_state = ISCSI_CONN_WR_STATE_IDLE;
2732                 spin_unlock_bh(&iscsi_wr_lock);
2733         }
2734
2735         TRACE_EXIT();
2736         return;
2737 }
2738
2739 static int iscsi_xmit_response(struct scst_cmd *scst_cmd)
2740 {
2741         int is_send_status = scst_cmd_get_is_send_status(scst_cmd);
2742         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
2743                                         scst_cmd_get_tgt_priv(scst_cmd);
2744         struct iscsi_conn *conn = req->conn;
2745         int status = scst_cmd_get_status(scst_cmd);
2746         u8 *sense = scst_cmd_get_sense_buffer(scst_cmd);
2747         int sense_len = scst_cmd_get_sense_buffer_len(scst_cmd);
2748         int old_state = req->scst_state;
2749
2750         if (scst_cmd_atomic(scst_cmd))
2751                 return SCST_TGT_RES_NEED_THREAD_CTX;
2752
2753         scst_cmd_set_tgt_priv(scst_cmd, NULL);
2754
2755         req->tm_aborted |= scst_cmd_aborted(scst_cmd) ? 1 : 0;
2756         if (unlikely(req->tm_aborted)) {
2757                 TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
2758                         req->scst_cmd);
2759
2760                 scst_set_delivery_status(req->scst_cmd,
2761                         SCST_CMD_DELIVERY_ABORTED);
2762
2763                 if (old_state == ISCSI_CMD_STATE_RESTARTED) {
2764                         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
2765                         req_cmnd_release_force(req, ISCSI_FORCE_RELEASE_WRITE);
2766                 } else
2767                         iscsi_set_state_wake_up(req,
2768                                                 ISCSI_CMD_STATE_PROCESSED);
2769
2770                 goto out;
2771         }
2772
2773         if (unlikely(old_state != ISCSI_CMD_STATE_RESTARTED)) {
2774                 TRACE_DBG("req %p on %d state", req, old_state);
2775
2776                 /*
2777                  * We could preliminary have finished req before we knew its
2778                  * device, so check if we return correct sense format.
2779                  */
2780                 scst_check_convert_sense(scst_cmd);
2781
2782                 create_status_rsp(req, status, sense, sense_len);
2783
2784                 switch (old_state) {
2785                 case ISCSI_CMD_STATE_RX_CMD:
2786                 case ISCSI_CMD_STATE_AFTER_PREPROC:
2787                         break;
2788                 default:
2789                         sBUG();
2790                 }
2791
2792                 iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_PROCESSED);
2793                 goto out;
2794         }
2795
2796         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
2797
2798         req->bufflen = scst_cmd_get_resp_data_len(scst_cmd);
2799         req->sg = scst_cmd_get_sg(scst_cmd);
2800         req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
2801
2802         TRACE_DBG("req %p, is_send_status=%x, req->bufflen=%d, req->sg=%p, "
2803                 "req->sg_cnt %d", req, is_send_status, req->bufflen, req->sg,
2804                 req->sg_cnt);
2805
2806         if (unlikely((req->bufflen != 0) && !is_send_status)) {
2807                 PRINT_CRIT_ERROR("%s", "Sending DATA without STATUS is "
2808                         "unsupported");
2809                 scst_set_cmd_error(scst_cmd,
2810                         SCST_LOAD_SENSE(scst_sense_hardw_error));
2811                 sBUG();
2812         }
2813
2814         if (req->bufflen != 0) {
2815                 /*
2816                  * Check above makes sure that is_send_status is set,
2817                  * so status is valid here, but in future that could change.
2818                  * ToDo
2819                  */
2820                 if ((status != SAM_STAT_CHECK_CONDITION) &&
2821                     ((cmnd_hdr(req)->flags & (ISCSI_CMD_WRITE|ISCSI_CMD_READ)) !=
2822                                 (ISCSI_CMD_WRITE|ISCSI_CMD_READ))) {
2823                         send_data_rsp(req, status, is_send_status);
2824                 } else {
2825                         struct iscsi_cmnd *rsp;
2826                         send_data_rsp(req, 0, 0);
2827                         if (is_send_status) {
2828                                 rsp = create_status_rsp(req, status, sense,
2829                                         sense_len);
2830                                 iscsi_set_resid(req, rsp, true);
2831                                 iscsi_cmnd_init_write(rsp,
2832                                         ISCSI_INIT_WRITE_REMOVE_HASH);
2833                         }
2834                 }
2835         } else if (is_send_status) {
2836                 struct iscsi_cmnd *rsp;
2837                 rsp = create_status_rsp(req, status, sense, sense_len);
2838                 iscsi_set_resid(req, rsp, false);
2839                 iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH);
2840         }
2841 #ifdef CONFIG_SCST_EXTRACHECKS
2842         else
2843                 sBUG();
2844 #endif
2845
2846         /*
2847          * "_ordered" here to protect from reorder, which can lead to
2848          * preliminary connection destroy in req_cmnd_release(). Just in
2849          * case, actually, because reordering shouldn't go so far, but who
2850          * knows..
2851          */
2852         conn_get_ordered(conn);
2853         req_cmnd_release(req);
2854         iscsi_try_local_processing(conn);
2855         conn_put(conn);
2856
2857 out:
2858         return SCST_TGT_RES_SUCCESS;
2859 }
2860
2861 /* Called under sn_lock */
2862 static bool iscsi_is_delay_tm_resp(struct iscsi_cmnd *rsp)
2863 {
2864         bool res = 0;
2865         struct iscsi_task_mgt_hdr *req_hdr =
2866                 (struct iscsi_task_mgt_hdr *)&rsp->parent_req->pdu.bhs;
2867         int function = req_hdr->function & ISCSI_FUNCTION_MASK;
2868         struct iscsi_session *sess = rsp->conn->session;
2869
2870         TRACE_ENTRY();
2871
2872         /* This should be checked for immediate TM commands as well */
2873
2874         switch (function) {
2875         default:
2876                 if (before(sess->exp_cmd_sn, req_hdr->cmd_sn))
2877                         res = 1;
2878                 break;
2879         }
2880
2881         TRACE_EXIT_RES(res);
2882         return res;
2883 }
2884
2885 /* Called under sn_lock, but might drop it inside, then reaquire */
2886 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess)
2887         __acquires(&sn_lock)
2888         __releases(&sn_lock)
2889 {
2890         struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
2891
2892         TRACE_ENTRY();
2893
2894         if (tm_rsp == NULL)
2895                 goto out;
2896
2897         if (iscsi_is_delay_tm_resp(tm_rsp))
2898                 goto out;
2899
2900         TRACE(TRACE_MGMT_MINOR, "Sending delayed rsp %p", tm_rsp);
2901
2902         sess->tm_rsp = NULL;
2903         sess->tm_active--;
2904
2905         spin_unlock(&sess->sn_lock);
2906
2907         sBUG_ON(sess->tm_active < 0);
2908
2909         iscsi_cmnd_init_write(tm_rsp,
2910                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2911
2912         spin_lock(&sess->sn_lock);
2913
2914 out:
2915         TRACE_EXIT();
2916         return;
2917 }
2918
2919 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status)
2920 {
2921         struct iscsi_cmnd *rsp;
2922         struct iscsi_task_mgt_hdr *req_hdr =
2923                                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2924         struct iscsi_task_rsp_hdr *rsp_hdr;
2925         struct iscsi_session *sess = req->conn->session;
2926         int fn = req_hdr->function & ISCSI_FUNCTION_MASK;
2927
2928         TRACE_ENTRY();
2929
2930         TRACE_MGMT_DBG("TM req %p finished", req);
2931         TRACE((req_hdr->function == ISCSI_FUNCTION_ABORT_TASK) ?
2932                          TRACE_MGMT_MINOR : TRACE_MGMT,
2933                 "TM fn %d finished, status %d", fn, status);
2934
2935         rsp = iscsi_cmnd_create_rsp_cmnd(req);
2936         rsp_hdr = (struct iscsi_task_rsp_hdr *)&rsp->pdu.bhs;
2937
2938         rsp_hdr->opcode = ISCSI_OP_SCSI_TASK_MGT_RSP;
2939         rsp_hdr->flags = ISCSI_FLG_FINAL;
2940         rsp_hdr->itt = req_hdr->itt;
2941         rsp_hdr->response = status;
2942
2943         if (fn == ISCSI_FUNCTION_TARGET_COLD_RESET) {
2944                 rsp->should_close_conn = 1;
2945                 rsp->should_close_all_conn = 1;
2946         }
2947
2948         sBUG_ON(sess->tm_rsp != NULL);
2949
2950         spin_lock(&sess->sn_lock);
2951         if (iscsi_is_delay_tm_resp(rsp)) {
2952                 TRACE(TRACE_MGMT_MINOR, "Delaying TM fn %d response %p "
2953                         "(req %p), because not all affected commands received "
2954                         "(TM cmd sn %u, exp sn %u)",
2955                         req_hdr->function & ISCSI_FUNCTION_MASK, rsp, req,
2956                         req_hdr->cmd_sn, sess->exp_cmd_sn);
2957                 sess->tm_rsp = rsp;
2958                 spin_unlock(&sess->sn_lock);
2959                 goto out_release;
2960         }
2961         sess->tm_active--;
2962         spin_unlock(&sess->sn_lock);
2963
2964         sBUG_ON(sess->tm_active < 0);
2965
2966         iscsi_cmnd_init_write(rsp,
2967                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2968
2969 out_release:
2970         req_cmnd_release(req);
2971
2972         TRACE_EXIT();
2973         return;
2974 }
2975
2976 static inline int iscsi_get_mgmt_response(int status)
2977 {
2978         switch (status) {
2979         case SCST_MGMT_STATUS_SUCCESS:
2980                 return ISCSI_RESPONSE_FUNCTION_COMPLETE;
2981
2982         case SCST_MGMT_STATUS_TASK_NOT_EXIST:
2983                 return ISCSI_RESPONSE_UNKNOWN_TASK;
2984
2985         case SCST_MGMT_STATUS_LUN_NOT_EXIST:
2986                 return ISCSI_RESPONSE_UNKNOWN_LUN;
2987
2988         case SCST_MGMT_STATUS_FN_NOT_SUPPORTED:
2989                 return ISCSI_RESPONSE_FUNCTION_UNSUPPORTED;
2990
2991         case SCST_MGMT_STATUS_REJECTED:
2992         case SCST_MGMT_STATUS_FAILED:
2993         default:
2994                 return ISCSI_RESPONSE_FUNCTION_REJECTED;
2995         }
2996 }
2997
2998 static void iscsi_task_mgmt_fn_done(struct scst_mgmt_cmd *scst_mcmd)
2999 {
3000         int fn = scst_mgmt_cmd_get_fn(scst_mcmd);
3001         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
3002                                 scst_mgmt_cmd_get_tgt_priv(scst_mcmd);
3003         int status =
3004                 iscsi_get_mgmt_response(scst_mgmt_cmd_get_status(scst_mcmd));
3005
3006         TRACE_MGMT_DBG("req %p, scst_mcmd %p, fn %d, scst status %d",
3007                 req, scst_mcmd, fn, scst_mgmt_cmd_get_status(scst_mcmd));
3008
3009         switch (fn) {
3010         case SCST_NEXUS_LOSS_SESS:
3011         case SCST_ABORT_ALL_TASKS_SESS:
3012                 /* They are internal */
3013                 break;
3014         default:
3015                 iscsi_send_task_mgmt_resp(req, status);
3016                 scst_mgmt_cmd_set_tgt_priv(scst_mcmd, NULL);
3017                 break;
3018         }
3019         return;
3020 }
3021
3022 static int iscsi_scsi_aen(struct scst_aen *aen)
3023 {
3024         int res = SCST_AEN_RES_SUCCESS;
3025         uint64_t lun = scst_aen_get_lun(aen);
3026         const uint8_t *sense = scst_aen_get_sense(aen);
3027         int sense_len = scst_aen_get_sense_len(aen);
3028         struct iscsi_session *sess = scst_sess_get_tgt_priv(
3029                                         scst_aen_get_sess(aen));
3030         struct iscsi_conn *conn;
3031         bool found;
3032         struct iscsi_cmnd *fake_req, *rsp;
3033         struct iscsi_async_msg_hdr *rsp_hdr;
3034         struct scatterlist *sg;
3035
3036         TRACE_ENTRY();
3037
3038         TRACE_MGMT_DBG("SCSI AEN to sess %p (initiator %s)", sess,
3039                 sess->initiator_name);
3040
3041         mutex_lock(&sess->target->target_mutex);
3042
3043         found = false;
3044         list_for_each_entry_reverse(conn, &sess->conn_list, conn_list_entry) {
3045                 if (!test_bit(ISCSI_CONN_SHUTTINGDOWN, &conn->conn_aflags) &&
3046                     (conn->conn_reinst_successor == NULL)) {
3047                         found = true;
3048                         break;
3049                 }
3050         }
3051         if (!found) {
3052                 TRACE_MGMT_DBG("Unable to find alive conn for sess %p", sess);
3053                 goto out_err;
3054         }
3055
3056         /* Create a fake request */
3057         fake_req = cmnd_alloc(conn, NULL);
3058         if (fake_req == NULL) {
3059                 PRINT_ERROR("%s", "Unable to alloc fake AEN request");
3060                 goto out_err;
3061         }
3062
3063         mutex_unlock(&sess->target->target_mutex);
3064
3065         rsp = iscsi_cmnd_create_rsp_cmnd(fake_req);
3066         if (rsp == NULL) {
3067                 PRINT_ERROR("%s", "Unable to alloc AEN rsp");
3068                 goto out_err_free_req;
3069         }
3070
3071         fake_req->scst_state = ISCSI_CMD_STATE_AEN;
3072         fake_req->scst_aen = aen;
3073
3074         rsp_hdr = (struct iscsi_async_msg_hdr *)&rsp->pdu.bhs;
3075
3076         rsp_hdr->opcode = ISCSI_OP_ASYNC_MSG;
3077         rsp_hdr->flags = ISCSI_FLG_FINAL;
3078         rsp_hdr->lun = lun; /* it's already in SCSI form */
3079         rsp_hdr->ffffffff = 0xffffffff;
3080         rsp_hdr->async_event = ISCSI_ASYNC_SCSI;
3081
3082         sg = rsp->sg = rsp->rsp_sg;
3083         rsp->sg_cnt = 2;
3084         rsp->own_sg = 1;
3085
3086         sg_init_table(sg, 2);
3087         sg_set_buf(&sg[0], &rsp->sense_hdr, sizeof(rsp->sense_hdr));
3088         sg_set_buf(&sg[1], sense, sense_len);
3089
3090         rsp->sense_hdr.length = cpu_to_be16(sense_len);
3091         rsp->pdu.datasize = sizeof(rsp->sense_hdr) + sense_len;
3092         rsp->bufflen = rsp->pdu.datasize;
3093
3094         iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_WAKE);
3095
3096         req_cmnd_release(fake_req);
3097
3098 out:
3099         TRACE_EXIT_RES(res);
3100         return res;
3101
3102 out_err_free_req:
3103         req_cmnd_release(fake_req);
3104
3105 out_err:
3106         mutex_unlock(&sess->target->target_mutex);
3107         res = SCST_AEN_RES_FAILED;
3108         goto out;
3109 }
3110
3111 static int iscsi_report_aen(struct scst_aen *aen)
3112 {
3113         int res;
3114         int event_fn = scst_aen_get_event_fn(aen);
3115
3116         TRACE_ENTRY();
3117
3118         switch (event_fn) {
3119         case SCST_AEN_SCSI:
3120                 res = iscsi_scsi_aen(aen);
3121                 break;
3122         default:
3123                 TRACE_MGMT_DBG("Unsupported AEN %d", event_fn);
3124                 res = SCST_AEN_RES_NOT_SUPPORTED;
3125                 break;
3126         }
3127
3128         TRACE_EXIT_RES(res);
3129         return res;
3130 }
3131
3132 static int iscsi_target_detect(struct scst_tgt_template *templ)
3133 {
3134         /* Nothing to do */
3135         return 0;
3136 }
3137
3138 static int iscsi_target_release(struct scst_tgt *scst_tgt)
3139 {
3140         /* Nothing to do */
3141         return 0;
3142 }
3143
3144 struct scst_tgt_template iscsi_template = {
3145         .name = "iscsi",
3146         .sg_tablesize = 0xFFFF /* no limit */,
3147         .threads_num = 0,
3148         .no_clustering = 1,
3149         .xmit_response_atomic = 0,
3150         .detect = iscsi_target_detect,
3151         .release = iscsi_target_release,
3152         .xmit_response = iscsi_xmit_response,
3153 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
3154         .alloc_data_buf = iscsi_alloc_data_buf,
3155 #endif
3156         .preprocessing_done = iscsi_preprocessing_done,
3157         .pre_exec = iscsi_pre_exec,
3158         .task_mgmt_affected_cmds_done = iscsi_task_mgmt_affected_cmds_done,
3159         .task_mgmt_fn_done = iscsi_task_mgmt_fn_done,
3160         .report_aen = iscsi_report_aen,
3161 };
3162
3163 static __init int iscsi_run_threads(int count, char *name, int (*fn)(void *))
3164 {
3165         int res = 0;
3166         int i;
3167         struct iscsi_thread_t *thr;
3168
3169         for (i = 0; i < count; i++) {
3170                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
3171                 if (!thr) {
3172                         res = -ENOMEM;
3173                         PRINT_ERROR("Failed to allocate thr %d", res);
3174                         goto out;
3175                 }
3176                 thr->thr = kthread_run(fn, NULL, "%s%d", name, i);
3177                 if (IS_ERR(thr->thr)) {
3178                         res = PTR_ERR(thr->thr);
3179                         PRINT_ERROR("kthread_create() failed: %d", res);
3180                         kfree(thr);
3181                         goto out;
3182                 }
3183                 list_add_tail(&thr->threads_list_entry, &iscsi_threads_list);
3184         }
3185
3186 out:
3187         return res;
3188 }
3189
3190 static void iscsi_stop_threads(void)
3191 {
3192         struct iscsi_thread_t *t, *tmp;
3193
3194         list_for_each_entry_safe(t, tmp, &iscsi_threads_list,
3195                                 threads_list_entry) {
3196                 int rc = kthread_stop(t->thr);
3197                 if (rc < 0)
3198                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
3199                 list_del(&t->threads_list_entry);
3200                 kfree(t);
3201         }
3202         return;
3203 }
3204
3205 static int __init iscsi_init(void)
3206 {
3207         int err = 0;
3208         int num;
3209
3210         PRINT_INFO("iSCSI SCST Target - version %s", ISCSI_VERSION_STRING);
3211
3212         dummy_page = alloc_pages(GFP_KERNEL, 0);
3213         if (dummy_page == NULL) {
3214                 PRINT_ERROR("%s", "Dummy page allocation failed");
3215                 goto out;
3216         }
3217
3218         sg_init_table(&dummy_sg, 1);
3219         sg_set_page(&dummy_sg, dummy_page, PAGE_SIZE, 0);
3220
3221 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
3222         err = net_set_get_put_page_callbacks(iscsi_get_page_callback,
3223                         iscsi_put_page_callback);
3224         if (err != 0) {
3225                 PRINT_INFO("Unable to set page callbackes: %d", err);
3226                 goto out_free_dummy;
3227         }
3228 #else
3229         PRINT_WARNING("%s",
3230                 "CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION "
3231                 "not enabled in your kernel. ISCSI-SCST will be working with "
3232                 "not the best performance. Refer README file for details.");
3233 #endif
3234
3235         ctr_major = register_chrdev(0, ctr_name, &ctr_fops);
3236         if (ctr_major < 0) {
3237                 PRINT_ERROR("failed to register the control device %d",
3238                             ctr_major);
3239                 err = ctr_major;
3240                 goto out_callb;
3241         }
3242
3243         err = event_init();
3244         if (err < 0)
3245                 goto out_reg;
3246
3247         iscsi_cmnd_cache = KMEM_CACHE(iscsi_cmnd, SCST_SLAB_FLAGS);
3248         if (!iscsi_cmnd_cache) {
3249                 err = -ENOMEM;
3250                 goto out_event;
3251         }
3252
3253         err = scst_register_target_template(&iscsi_template);
3254         if (err < 0)
3255                 goto out_kmem;
3256
3257         iscsi_template_registered = 1;
3258
3259 #ifdef CONFIG_SCST_PROC
3260         err = iscsi_procfs_init();
3261         if (err < 0)
3262                 goto out_reg_tmpl;
3263 #endif
3264
3265         num = max((int)num_online_cpus(), 2);
3266
3267         err = iscsi_run_threads(num, "iscsird", istrd);
3268         if (err != 0)
3269                 goto out_thr;
3270
3271         err = iscsi_run_threads(num, "iscsiwr", istwr);
3272         if (err != 0)
3273                 goto out_thr;
3274
3275 out:
3276         return err;
3277
3278 out_thr:
3279 #ifdef CONFIG_SCST_PROC
3280         iscsi_procfs_exit();
3281 #endif
3282         iscsi_stop_threads();
3283
3284 #ifdef CONFIG_SCST_PROC
3285 out_reg_tmpl:
3286 #endif
3287         scst_unregister_target_template(&iscsi_template);
3288
3289 out_kmem:
3290         kmem_cache_destroy(iscsi_cmnd_cache);
3291
3292 out_event:
3293         event_exit();
3294
3295 out_reg:
3296         unregister_chrdev(ctr_major, ctr_name);
3297
3298 out_callb:
3299 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
3300         net_set_get_put_page_callbacks(NULL, NULL);
3301
3302 out_free_dummy:
3303 #endif
3304         __free_pages(dummy_page, 0);
3305         goto out;
3306 }
3307
3308 static void __exit iscsi_exit(void)
3309 {
3310         iscsi_stop_threads();
3311
3312         unregister_chrdev(ctr_major, ctr_name);
3313
3314 #ifdef CONFIG_SCST_PROC
3315         iscsi_procfs_exit();
3316 #endif
3317         event_exit();
3318
3319         kmem_cache_destroy(iscsi_cmnd_cache);
3320
3321         scst_unregister_target_template(&iscsi_template);
3322
3323 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
3324         net_set_get_put_page_callbacks(NULL, NULL);
3325 #endif
3326
3327         __free_pages(dummy_page, 0);
3328         return;
3329 }
3330
3331 module_init(iscsi_init);
3332 module_exit(iscsi_exit);
3333
3334 MODULE_LICENSE("GPL");