iscsi-scst/kernel/iscsi.c

   1 /*
   2  *  Copyright (C) 2002 - 2003 Ardis Technolgies <roman@ardistech.com>
   3  *  Copyright (C) 2007 - 2008 Vladislav Bolkhovitin
   4  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
   5  *
   6  *  This program is free software; you can redistribute it and/or
   7  *  modify it under the terms of the GNU General Public License
   8  *  as published by the Free Software Foundation, version 2
   9  *  of the License.
  10  *
  11  *  This program is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14  *  GNU General Public License for more details.
  15  */
  16
  17 #include <linux/module.h>
  18 #include <linux/hash.h>
  19 #include <linux/kthread.h>
  20 #include <net/tcp.h>
  21 #include <scsi/scsi.h>
  22
  23 #include "iscsi.h"
  24 #include "digest.h"
  25
  26 #ifndef NET_PAGE_CALLBACKS_DEFINED
  27 #warning "Patch put_page_callback-<kernel-version>.patch not applied on your \
  28         kernel. ISCSI-SCST will run in the performance degraded mode. Refer \
  29         README file for details."
  30 #endif
  31
  32 #define ISCSI_INIT_WRITE_WAKE           0x1
  33 #define ISCSI_INIT_WRITE_REMOVE_HASH    0x2
  34
  35 static int ctr_major;
  36 static char ctr_name[] = "iscsi-scst-ctl";
  37 static int iscsi_template_registered;
  38
  39 #if defined(DEBUG) || defined(TRACING)
  40 unsigned long iscsi_trace_flag = ISCSI_DEFAULT_LOG_FLAGS;
  41 #endif
  42
  43 static struct kmem_cache *iscsi_cmnd_cache;
  44
  45 DEFINE_SPINLOCK(iscsi_rd_lock);
  46 LIST_HEAD(iscsi_rd_list);
  47 DECLARE_WAIT_QUEUE_HEAD(iscsi_rd_waitQ);
  48
  49 DEFINE_SPINLOCK(iscsi_wr_lock);
  50 LIST_HEAD(iscsi_wr_list);
  51 DECLARE_WAIT_QUEUE_HEAD(iscsi_wr_waitQ);
  52
  53 static char dummy_data[1024];
  54
  55 struct iscsi_thread_t {
  56         struct task_struct *thr;
  57         struct list_head threads_list_entry;
  58 };
  59
  60 static LIST_HEAD(iscsi_threads_list);
  61
  62 static void cmnd_remove_hash(struct iscsi_cmnd *cmnd);
  63 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status);
  64 static void cmnd_prepare_get_rejected_cmd_data(struct iscsi_cmnd *cmnd);
  65 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess);
  66 static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd);
  67
  68 static inline u32 cmnd_write_size(struct iscsi_cmnd *cmnd)
  69 {
  70         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
  71
  72         if (hdr->flags & ISCSI_CMD_WRITE)
  73                 return be32_to_cpu(hdr->data_length);
  74         return 0;
  75 }
  76
  77 static inline u32 cmnd_read_size(struct iscsi_cmnd *cmnd)
  78 {
  79         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
  80
  81         if (hdr->flags & ISCSI_CMD_READ) {
  82                 struct iscsi_rlength_ahdr *ahdr =
  83                         (struct iscsi_rlength_ahdr *)cmnd->pdu.ahs;
  84
  85                 if (!(hdr->flags & ISCSI_CMD_WRITE))
  86                         return be32_to_cpu(hdr->data_length);
  87                 if (ahdr && ahdr->ahstype == ISCSI_AHSTYPE_RLENGTH)
  88                         return be32_to_cpu(ahdr->read_length);
  89         }
  90         return 0;
  91 }
  92
  93 static inline void iscsi_restart_cmnd(struct iscsi_cmnd *cmnd)
  94 {
  95         EXTRACHECKS_BUG_ON(cmnd->data_waiting);
  96
  97         cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
  98         scst_restart_cmd(cmnd->scst_cmd, SCST_PREPROCESS_STATUS_SUCCESS,
  99                 SCST_CONTEXT_THREAD);
 100 }
 101
 102 static inline void iscsi_restart_waiting_cmnd(struct iscsi_cmnd *cmnd)
 103 {
 104         /*
 105          * There is no race with conn_abort(), since all functions
 106          * called from single read thread
 107          */
 108         iscsi_extracheck_is_rd_thread(cmnd->conn);
 109         cmnd->data_waiting = 0;
 110
 111         iscsi_restart_cmnd(cmnd);
 112 }
 113
 114 static inline void iscsi_fail_waiting_cmnd(struct iscsi_cmnd *cmnd)
 115 {
 116         TRACE_MGMT_DBG("Failing data waiting cmd %p", cmnd);
 117
 118         /*
 119          * There is no race with conn_abort(), since all functions
 120          * called from single read thread
 121          */
 122         iscsi_extracheck_is_rd_thread(cmnd->conn);
 123         cmnd->data_waiting = 0;
 124
 125         req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
 126 }
 127
 128 struct iscsi_cmnd *cmnd_alloc(struct iscsi_conn *conn, struct iscsi_cmnd *parent)
 129 {
 130         struct iscsi_cmnd *cmnd;
 131
 132         /* ToDo: __GFP_NOFAIL?? */
 133 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17)
 134         cmnd = kmem_cache_alloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
 135         memset(cmnd, 0, sizeof(*cmnd));
 136 #else
 137         cmnd = kmem_cache_zalloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
 138 #endif
 139
 140         atomic_set(&cmnd->ref_cnt, 1);
 141         cmnd->scst_state = ISCSI_CMD_STATE_NEW;
 142         cmnd->conn = conn;
 143         cmnd->parent_req = parent;
 144         init_waitqueue_head(&cmnd->scst_waitQ);
 145
 146         if (parent == NULL) {
 147                 conn_get(conn);
 148
 149 #ifdef NET_PAGE_CALLBACKS_DEFINED
 150                 atomic_set(&cmnd->net_ref_cnt, 0);
 151 #endif
 152                 spin_lock_init(&cmnd->rsp_cmd_lock);
 153                 INIT_LIST_HEAD(&cmnd->rsp_cmd_list);
 154                 INIT_LIST_HEAD(&cmnd->rx_ddigest_cmd_list);
 155
 156                 spin_lock_bh(&conn->cmd_list_lock);
 157                 list_add_tail(&cmnd->cmd_list_entry, &conn->cmd_list);
 158                 spin_unlock_bh(&conn->cmd_list_lock);
 159         }
 160
 161         TRACE_DBG("conn %p, parent %p, cmnd %p", conn, parent, cmnd);
 162         return cmnd;
 163 }
 164
 165 /* Frees a command. Also frees the additional header. */
 166 static void cmnd_free(struct iscsi_cmnd *cmnd)
 167 {
 168         TRACE_DBG("%p", cmnd);
 169
 170         if (unlikely(cmnd->tm_aborted)) {
 171                 TRACE_MGMT_DBG("Free aborted cmd %p (scst cmd %p, state %d, "
 172                         "parent_req %p)", cmnd, cmnd->scst_cmd, cmnd->scst_state,
 173                         cmnd->parent_req);
 174         }
 175
 176         /* Catch users from cmd_list or rsp_cmd_list */
 177         EXTRACHECKS_BUG_ON(atomic_read(&cmnd->ref_cnt) != 0);
 178
 179         kfree(cmnd->pdu.ahs);
 180
 181         if (unlikely(cmnd->on_write_list || cmnd->on_written_list)) {
 182                 struct iscsi_scsi_cmd_hdr *req = cmnd_hdr(cmnd);
 183
 184                 PRINT_CRIT_ERROR("cmnd %p still on some list?, %x, %x, %x, %x, %x, %x, %x",
 185                         cmnd, req->opcode, req->scb[0], req->flags, req->itt,
 186                         be32_to_cpu(req->data_length),
 187                         req->cmd_sn, be32_to_cpu(cmnd->pdu.datasize));
 188
 189                 if (unlikely(cmnd->parent_req)) {
 190                         struct iscsi_scsi_cmd_hdr *preq =
 191                                         cmnd_hdr(cmnd->parent_req);
 192                         PRINT_CRIT_ERROR("%p %x %u", preq, preq->opcode, preq->scb[0]);
 193                 }
 194                 sBUG();
 195         }
 196
 197         kmem_cache_free(iscsi_cmnd_cache, cmnd);
 198         return;
 199 }
 200
 201 void cmnd_done(struct iscsi_cmnd *cmnd)
 202 {
 203         TRACE_DBG("%p", cmnd);
 204
 205         if (unlikely(cmnd->tm_aborted)) {
 206                 TRACE_MGMT_DBG("Done aborted cmd %p (scst cmd %p, state %d, "
 207                         "parent_req %p)", cmnd, cmnd->scst_cmd, cmnd->scst_state,
 208                         cmnd->parent_req);
 209         }
 210
 211         EXTRACHECKS_BUG_ON(cmnd->on_rx_digest_list);
 212
 213         if (cmnd->on_written_list) {
 214                 struct iscsi_conn *conn = cmnd->conn;
 215                 TRACE_DBG("Deleting cmd %p from conn %p written_list", cmnd,
 216                         conn);
 217                 spin_lock_bh(&conn->write_list_lock);
 218                 list_del(&cmnd->write_list_entry);
 219                 cmnd->on_written_list = 0;
 220                 spin_unlock_bh(&conn->write_list_lock);
 221         }
 222
 223         if (cmnd->parent_req == NULL) {
 224                 struct iscsi_conn *conn = cmnd->conn;
 225                 TRACE_DBG("Deleting req %p from conn %p", cmnd, conn);
 226
 227                 spin_lock_bh(&conn->cmd_list_lock);
 228                 list_del(&cmnd->cmd_list_entry);
 229                 spin_unlock_bh(&conn->cmd_list_lock);
 230
 231                 conn_put(conn);
 232
 233                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rsp_cmd_list));
 234                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rx_ddigest_cmd_list));
 235
 236                 /* Order between above and below code is important! */
 237
 238                 if (cmnd->scst_cmd) {
 239                         switch (cmnd->scst_state) {
 240                         case ISCSI_CMD_STATE_PROCESSED:
 241                                 TRACE_DBG("cmd %p PROCESSED", cmnd);
 242                                 scst_tgt_cmd_done(cmnd->scst_cmd);
 243                                 break;
 244                         case ISCSI_CMD_STATE_AFTER_PREPROC:
 245                         {
 246                                 struct scst_cmd *scst_cmd = cmnd->scst_cmd;
 247                                 TRACE_DBG("cmd %p AFTER_PREPROC", cmnd);
 248                                 cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
 249                                 cmnd->scst_cmd = NULL;
 250                                 scst_restart_cmd(scst_cmd,
 251                                         SCST_PREPROCESS_STATUS_ERROR_FATAL,
 252                                         SCST_CONTEXT_THREAD);
 253                                 break;
 254                         }
 255                         default:
 256                                 PRINT_CRIT_ERROR("Unexpected cmnd scst state %d",
 257                                                  cmnd->scst_state);
 258                                 sBUG();
 259                                 break;
 260                         }
 261                 }
 262         } else {
 263                 EXTRACHECKS_BUG_ON(cmnd->scst_cmd != NULL);
 264                 TRACE_DBG("Deleting rsp %p from parent %p", cmnd,
 265                         cmnd->parent_req);
 266
 267                 spin_lock_bh(&cmnd->parent_req->rsp_cmd_lock);
 268                 list_del(&cmnd->rsp_cmd_list_entry);
 269                 spin_unlock_bh(&cmnd->parent_req->rsp_cmd_lock);
 270
 271                 cmnd_put(cmnd->parent_req);
 272         }
 273
 274         /* Order between above and below code is important! */
 275
 276         if (cmnd->own_sg) {
 277                 TRACE_DBG("%s", "own_sg");
 278                 scst_free(cmnd->sg, cmnd->sg_cnt);
 279 #ifdef DEBUG
 280                 cmnd->own_sg = 0;
 281                 cmnd->sg = NULL;
 282                 cmnd->sg_cnt = -1;
 283 #endif
 284         }
 285
 286         if (cmnd->dec_active_cmnds) {
 287                 struct iscsi_session *sess = cmnd->conn->session;
 288                 TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
 289                         "new value %d)", cmnd, sess,
 290                         atomic_read(&sess->active_cmds)-1);
 291                 atomic_dec(&sess->active_cmds);
 292 #ifdef EXTRACHECKS
 293                 if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
 294                         PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
 295                                 atomic_read(&sess->active_cmds));
 296                         sBUG();
 297                 }
 298 #endif
 299         }
 300
 301         cmnd_free(cmnd);
 302         return;
 303 }
 304
 305 /*
 306  * Corresponding conn may also gets destroyed atfer this function, except only
 307  * if it's called from the read thread!
 308  *
 309  * It can't be called in parallel with iscsi_cmnds_init_write()!
 310  */
 311 void req_cmnd_release_force(struct iscsi_cmnd *req, int flags)
 312 {
 313         struct iscsi_cmnd *rsp, *t;
 314         struct iscsi_conn *conn = req->conn;
 315         LIST_HEAD(cmds_list);
 316
 317         TRACE_ENTRY();
 318
 319         TRACE_MGMT_DBG("%p", req);
 320
 321         sBUG_ON(req == conn->read_cmnd);
 322
 323         if (flags & ISCSI_FORCE_RELEASE_WRITE) {
 324                 spin_lock_bh(&conn->write_list_lock);
 325                 list_for_each_entry_safe(rsp, t, &conn->write_list,
 326                                                 write_list_entry) {
 327                         if (rsp->parent_req != req)
 328                                 continue;
 329
 330                         cmd_del_from_write_list(rsp);
 331
 332                         list_add_tail(&rsp->write_list_entry, &cmds_list);
 333                 }
 334                 spin_unlock_bh(&conn->write_list_lock);
 335
 336                 list_for_each_entry_safe(rsp, t, &cmds_list, write_list_entry) {
 337                         TRACE_MGMT_DBG("Putting write rsp %p", rsp);
 338                         list_del(&rsp->write_list_entry);
 339                         cmnd_put(rsp);
 340                 }
 341         }
 342
 343 again_rsp:
 344         spin_lock_bh(&req->rsp_cmd_lock);
 345         list_for_each_entry_reverse(rsp, &req->rsp_cmd_list, rsp_cmd_list_entry) {
 346                 bool r;
 347
 348                 if (rsp->force_cleanup_done)
 349                         continue;
 350
 351                 rsp->force_cleanup_done = 1;
 352
 353                 if (cmnd_get_check(rsp))
 354                         continue;
 355
 356                 spin_unlock_bh(&req->rsp_cmd_lock);
 357
 358                 spin_lock_bh(&conn->write_list_lock);
 359                 r = rsp->on_write_list || rsp->write_processing_started;
 360                 spin_unlock_bh(&conn->write_list_lock);
 361
 362                 cmnd_put(rsp);
 363
 364                 if (r)
 365                         goto again_rsp;
 366
 367                 /*
 368                  * If both on_write_list and write_processing_started not set,
 369                  * we can safely put() rsp.
 370                  */
 371                 TRACE_MGMT_DBG("Putting rsp %p", rsp);
 372                 cmnd_put(rsp);
 373                 goto again_rsp;
 374         }
 375         spin_unlock_bh(&req->rsp_cmd_lock);
 376
 377         req_cmnd_release(req);
 378
 379         TRACE_EXIT();
 380         return;
 381 }
 382
 383 /*
 384  * Corresponding conn may also gets destroyed atfer this function, except only
 385  * if it's called from the read thread!
 386  */
 387 void req_cmnd_release(struct iscsi_cmnd *req)
 388 {
 389         struct iscsi_cmnd *c, *t;
 390
 391         TRACE_ENTRY();
 392
 393         TRACE_DBG("%p", req);
 394
 395 #ifdef EXTRACHECKS
 396         sBUG_ON(req->release_called);
 397         req->release_called = 1;
 398 #endif
 399
 400         if (unlikely(req->tm_aborted)) {
 401                 TRACE_MGMT_DBG("Release aborted req cmd %p (scst cmd %p, "
 402                         "state %d)", req, req->scst_cmd, req->scst_state);
 403         }
 404
 405         sBUG_ON(req->parent_req != NULL);
 406
 407         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
 408                                 rx_ddigest_cmd_list_entry) {
 409                 cmd_del_from_rx_ddigest_list(c);
 410                 cmnd_put(c);
 411         }
 412
 413         if (req->hashed)
 414                 cmnd_remove_hash(req);
 415
 416         if (req->dec_active_cmnds) {
 417                 struct iscsi_session *sess = req->conn->session;
 418                 TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
 419                         "new value %d)", req, sess,
 420                         atomic_read(&sess->active_cmds)-1);
 421                 atomic_dec(&sess->active_cmds);
 422                 req->dec_active_cmnds = 0;
 423 #ifdef EXTRACHECKS
 424                 if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
 425                         PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
 426                                 atomic_read(&sess->active_cmds));
 427                         sBUG();
 428                 }
 429 #endif
 430         }
 431
 432         cmnd_put(req);
 433
 434         TRACE_EXIT();
 435         return;
 436 }
 437
 438 /*
 439  * Corresponding conn may also gets destroyed atfer this function, except only
 440  * if it's called from the read thread!
 441  */
 442 void rsp_cmnd_release(struct iscsi_cmnd *cmnd)
 443 {
 444         TRACE_DBG("%p", cmnd);
 445
 446 #ifdef EXTRACHECKS
 447         sBUG_ON(cmnd->release_called);
 448         cmnd->release_called = 1;
 449 #endif
 450
 451         sBUG_ON(cmnd->hashed);
 452         sBUG_ON(cmnd->parent_req == NULL);
 453
 454         cmnd_put(cmnd);
 455         return;
 456 }
 457
 458 /**
 459  * create a new command used as response.
 460  *
 461  * iscsi_cmnd_create_rsp_cmnd -
 462  * @cmnd: ptr to request command
 463  *
 464  * @return    ptr to response command or NULL
 465  */
 466 static struct iscsi_cmnd *iscsi_cmnd_create_rsp_cmnd(struct iscsi_cmnd *parent)
 467 {
 468         struct iscsi_cmnd *rsp;
 469
 470         rsp = cmnd_alloc(parent->conn, parent);
 471
 472         spin_lock_bh(&parent->rsp_cmd_lock);
 473         TRACE_DBG("Adding rsp %p to parent %p", rsp, parent);
 474         list_add_tail(&rsp->rsp_cmd_list_entry, &parent->rsp_cmd_list);
 475         spin_unlock_bh(&parent->rsp_cmd_lock);
 476         cmnd_get(parent);
 477         return rsp;
 478 }
 479
 480 static inline struct iscsi_cmnd *get_rsp_cmnd(struct iscsi_cmnd *req)
 481 {
 482         struct iscsi_cmnd *res = NULL;
 483
 484         /* Currently this lock isn't needed, but just in case.. */
 485         spin_lock_bh(&req->rsp_cmd_lock);
 486         if (!list_empty(&req->rsp_cmd_list)) {
 487                 res = list_entry(req->rsp_cmd_list.prev, struct iscsi_cmnd,
 488                         rsp_cmd_list_entry);
 489         }
 490         spin_unlock_bh(&req->rsp_cmd_lock);
 491
 492         return res;
 493 }
 494
 495 static void iscsi_cmnds_init_write(struct list_head *send, int flags)
 496 {
 497         struct iscsi_cmnd *rsp = list_entry(send->next, struct iscsi_cmnd,
 498                                                 write_list_entry);
 499         struct iscsi_conn *conn = rsp->conn;
 500         struct list_head *pos, *next;
 501
 502         sBUG_ON(list_empty(send));
 503
 504         /*
 505          * If we don't remove hashed req cmd from the hash list here, before
 506          * submitting it for transmittion, we will have a race, when for
 507          * some reason cmd's release is delayed after transmittion and
 508          * initiator sends cmd with the same ITT => this command will be
 509          * erroneously rejected as a duplicate.
 510          */
 511         if ((flags & ISCSI_INIT_WRITE_REMOVE_HASH) && rsp->parent_req->hashed &&
 512             (rsp->parent_req->r2t_length == 0) &&
 513             (rsp->parent_req->outstanding_r2t == 0))
 514                 cmnd_remove_hash(rsp->parent_req);
 515
 516         if (!(conn->ddigest_type & DIGEST_NONE)) {
 517                 list_for_each(pos, send) {
 518                         rsp = list_entry(pos, struct iscsi_cmnd,
 519                                                 write_list_entry);
 520
 521                         if (rsp->pdu.datasize != 0) {
 522                                 TRACE_DBG("Doing data digest (%p:%x)", rsp,
 523                                         cmnd_opcode(rsp));
 524                                 digest_tx_data(rsp);
 525                         }
 526                 }
 527         }
 528
 529         spin_lock_bh(&conn->write_list_lock);
 530         list_for_each_safe(pos, next, send) {
 531                 rsp = list_entry(pos, struct iscsi_cmnd, write_list_entry);
 532
 533                 TRACE_DBG("%p:%x", rsp, cmnd_opcode(rsp));
 534
 535                 sBUG_ON(conn != rsp->conn);
 536
 537                 list_del(&rsp->write_list_entry);
 538                 cmd_add_on_write_list(conn, rsp);
 539         }
 540         spin_unlock_bh(&conn->write_list_lock);
 541
 542         if (flags & ISCSI_INIT_WRITE_WAKE)
 543                 iscsi_make_conn_wr_active(conn);
 544 }
 545
 546 static void iscsi_cmnd_init_write(struct iscsi_cmnd *rsp, int flags)
 547 {
 548         LIST_HEAD(head);
 549
 550         if (unlikely(rsp->on_write_list)) {
 551                 PRINT_CRIT_ERROR("cmd already on write list (%x %x %x %x %u %u "
 552                         "%u %u %u %u %u %d %d",
 553                         cmnd_itt(rsp), cmnd_ttt(rsp), cmnd_opcode(rsp),
 554                         cmnd_scsicode(rsp), rsp->r2t_sn,
 555                         rsp->r2t_length, rsp->is_unsolicited_data,
 556                         rsp->target_task_tag, rsp->outstanding_r2t,
 557                         rsp->hdigest, rsp->ddigest,
 558                         list_empty(&rsp->rsp_cmd_list), rsp->hashed);
 559                 sBUG();
 560         }
 561         list_add(&rsp->write_list_entry, &head);
 562         iscsi_cmnds_init_write(&head, flags);
 563 }
 564
 565 static void iscsi_set_datasize(struct iscsi_cmnd *cmnd, u32 offset, u32 size)
 566 {
 567         cmnd->pdu.datasize = size;
 568
 569         if (size & 3) {
 570                 u32 last_off = offset + size;
 571                 int idx = last_off >> PAGE_SHIFT;
 572                 u8 *p = (u8 *)page_address(sg_page(&cmnd->sg[idx])) +
 573                         (last_off & ~PAGE_MASK);
 574                 int i = 4 - (size & 3);
 575                 while (i--)
 576                     *p++ = 0;
 577         }
 578 }
 579
 580 static void send_data_rsp(struct iscsi_cmnd *req, u8 status, int send_status)
 581 {
 582         struct iscsi_cmnd *rsp;
 583         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
 584         struct iscsi_data_in_hdr *rsp_hdr;
 585         u32 pdusize, expsize, scsisize, size, offset, sn;
 586         LIST_HEAD(send);
 587
 588         TRACE_DBG("req %p", req);
 589
 590         pdusize = req->conn->session->sess_param.max_xmit_data_length;
 591         expsize = cmnd_read_size(req);
 592         size = min(expsize, (u32)req->bufflen);
 593         offset = 0;
 594         sn = 0;
 595
 596         while (1) {
 597                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
 598                 TRACE_DBG("rsp %p", rsp);
 599                 rsp->sg = req->sg;
 600                 rsp->sg_cnt = req->sg_cnt;
 601                 rsp->bufflen = req->bufflen;
 602                 rsp_hdr = (struct iscsi_data_in_hdr *)&rsp->pdu.bhs;
 603
 604                 rsp_hdr->opcode = ISCSI_OP_SCSI_DATA_IN;
 605                 rsp_hdr->itt = req_hdr->itt;
 606                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
 607                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
 608                 rsp_hdr->data_sn = cpu_to_be32(sn);
 609
 610                 if (size <= pdusize) {
 611                         TRACE_DBG("offset %d, size %d", offset, size);
 612                         iscsi_set_datasize(rsp, offset, size);
 613                         if (send_status) {
 614                                 TRACE_DBG("status %x", status);
 615                                 rsp_hdr->flags =
 616                                         ISCSI_FLG_FINAL | ISCSI_FLG_STATUS;
 617                                 rsp_hdr->cmd_status = status;
 618                         }
 619                         scsisize = req->bufflen;
 620                         if (scsisize < expsize) {
 621                                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
 622                                 size = expsize - scsisize;
 623                         } else if (scsisize > expsize) {
 624                                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
 625                                 size = scsisize - expsize;
 626                         } else
 627                                 size = 0;
 628                         rsp_hdr->residual_count = cpu_to_be32(size);
 629                         list_add_tail(&rsp->write_list_entry, &send);
 630                         break;
 631                 }
 632
 633                 TRACE_DBG("pdusize %d, offset %d, size %d", pdusize, offset,
 634                         size);
 635
 636                 iscsi_set_datasize(rsp, offset, pdusize);
 637
 638                 size -= pdusize;
 639                 offset += pdusize;
 640                 sn++;
 641
 642                 list_add_tail(&rsp->write_list_entry, &send);
 643         }
 644         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_REMOVE_HASH);
 645 }
 646
 647 static struct iscsi_cmnd *create_status_rsp(struct iscsi_cmnd *req, int status,
 648         const u8 *sense_buf, int sense_len)
 649 {
 650         struct iscsi_cmnd *rsp;
 651         struct iscsi_scsi_rsp_hdr *rsp_hdr;
 652         struct iscsi_sense_data *sense;
 653         struct scatterlist *sg;
 654
 655         rsp = iscsi_cmnd_create_rsp_cmnd(req);
 656         TRACE_DBG("%p", rsp);
 657
 658         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
 659         rsp_hdr->opcode = ISCSI_OP_SCSI_RSP;
 660         rsp_hdr->flags = ISCSI_FLG_FINAL;
 661         rsp_hdr->response = ISCSI_RESPONSE_COMMAND_COMPLETED;
 662         rsp_hdr->cmd_status = status;
 663         rsp_hdr->itt = cmnd_hdr(req)->itt;
 664
 665         if (SCST_SENSE_VALID(sense_buf)) {
 666                 TRACE_DBG("%s", "SENSE VALID");
 667                 /* ToDo: __GFP_NOFAIL ?? */
 668                 sg = rsp->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
 669                                         &rsp->sg_cnt);
 670                 if (sg == NULL) {
 671                         ;/* ToDo */;
 672                 }
 673                 rsp->own_sg = 1;
 674                 sense = (struct iscsi_sense_data *)page_address(sg_page(&sg[0]));
 675                 sense->length = cpu_to_be16(sense_len);
 676                 memcpy(sense->data, sense_buf, sense_len);
 677                 rsp->pdu.datasize = sizeof(struct iscsi_sense_data) + sense_len;
 678                 rsp->bufflen = (rsp->pdu.datasize + 3) & -4;
 679                 if (rsp->bufflen - rsp->pdu.datasize) {
 680                     int i = rsp->pdu.datasize;
 681                     u8 *p = (u8 *)sense + i;
 682
 683                     while (i < rsp->bufflen) {
 684                         *p++ = 0;
 685                         i++;
 686                     }
 687                 }
 688         } else {
 689                 rsp->pdu.datasize = 0;
 690                 rsp->bufflen = 0;
 691         }
 692
 693         return rsp;
 694 }
 695
 696 static struct iscsi_cmnd *create_sense_rsp(struct iscsi_cmnd *req,
 697         u8 sense_key, u8 asc, u8 ascq)
 698 {
 699         u8 sense[14];
 700         memset(sense, 0, sizeof(sense));
 701         sense[0] = 0xf0;
 702         sense[2] = sense_key;
 703         sense[7] = 6;   /* Additional sense length */
 704         sense[12] = asc;
 705         sense[13] = ascq;
 706         return create_status_rsp(req, SAM_STAT_CHECK_CONDITION, sense,
 707                 sizeof(sense));
 708 }
 709
 710 static void iscsi_cmnd_reject(struct iscsi_cmnd *req, int reason)
 711 {
 712         struct iscsi_cmnd *rsp;
 713         struct iscsi_reject_hdr *rsp_hdr;
 714         struct scatterlist *sg;
 715         char *addr;
 716
 717         TRACE_MGMT_DBG("Reject: req %p, reason %x", req, reason);
 718
 719         sBUG_ON(req->rejected);
 720         req->rejected = 1;
 721         req->reject_reason = ISCSI_REJECT_CMD;
 722
 723         rsp = iscsi_cmnd_create_rsp_cmnd(req);
 724         rsp_hdr = (struct iscsi_reject_hdr *)&rsp->pdu.bhs;
 725
 726         rsp_hdr->opcode = ISCSI_OP_REJECT;
 727         rsp_hdr->ffffffff = ISCSI_RESERVED_TAG;
 728         rsp_hdr->reason = reason;
 729
 730         /* ToDo: __GFP_NOFAIL ?? */
 731         sg = rsp->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
 732                                 &rsp->sg_cnt);
 733         if (sg == NULL) {
 734                 ;/* ToDo */;
 735         }
 736         rsp->own_sg = 1;
 737         addr = page_address(sg_page(&sg[0]));
 738         clear_page(addr);
 739         memcpy(addr, &req->pdu.bhs, sizeof(struct iscsi_hdr));
 740         rsp->bufflen = rsp->pdu.datasize = sizeof(struct iscsi_hdr);
 741
 742         iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH |
 743                                          ISCSI_INIT_WRITE_WAKE);
 744
 745         cmnd_prepare_get_rejected_cmd_data(req);
 746 }
 747
 748 static inline int iscsi_get_allowed_cmds(struct iscsi_session *sess)
 749 {
 750         int res = max(-1, (int)sess->max_queued_cmnds -
 751                                 atomic_read(&sess->active_cmds)-1);
 752         TRACE_DBG("allowed cmds %d (sess %p, active_cmds %d)", res,
 753                 sess, atomic_read(&sess->active_cmds));
 754         return res;
 755 }
 756
 757 static u32 cmnd_set_sn(struct iscsi_cmnd *cmnd, int set_stat_sn)
 758 {
 759         struct iscsi_conn *conn = cmnd->conn;
 760         struct iscsi_session *sess = conn->session;
 761         u32 res;
 762
 763         spin_lock(&sess->sn_lock);
 764
 765         if (set_stat_sn)
 766                 cmnd->pdu.bhs.sn = cpu_to_be32(conn->stat_sn++);
 767         cmnd->pdu.bhs.exp_sn = cpu_to_be32(sess->exp_cmd_sn);
 768         cmnd->pdu.bhs.max_sn = cpu_to_be32(sess->exp_cmd_sn +
 769                                  iscsi_get_allowed_cmds(sess));
 770
 771         res = cpu_to_be32(conn->stat_sn);
 772
 773         spin_unlock(&sess->sn_lock);
 774         return res;
 775 }
 776
 777 /* Called under sn_lock */
 778 static void __update_stat_sn(struct iscsi_cmnd *cmnd)
 779 {
 780         struct iscsi_conn *conn = cmnd->conn;
 781         u32 exp_stat_sn;
 782
 783         cmnd->pdu.bhs.exp_sn = exp_stat_sn = be32_to_cpu(cmnd->pdu.bhs.exp_sn);
 784         TRACE_DBG("%x,%x", cmnd_opcode(cmnd), exp_stat_sn);
 785         if ((int)(exp_stat_sn - conn->exp_stat_sn) > 0 &&
 786             (int)(exp_stat_sn - conn->stat_sn) <= 0) {
 787                 /* free pdu resources */
 788                 cmnd->conn->exp_stat_sn = exp_stat_sn;
 789         }
 790 }
 791
 792 static inline void update_stat_sn(struct iscsi_cmnd *cmnd)
 793 {
 794         spin_lock(&cmnd->conn->session->sn_lock);
 795         __update_stat_sn(cmnd);
 796         spin_unlock(&cmnd->conn->session->sn_lock);
 797 }
 798
 799 /* Called under sn_lock */
 800 static int check_cmd_sn(struct iscsi_cmnd *cmnd)
 801 {
 802         struct iscsi_session *session = cmnd->conn->session;
 803         u32 cmd_sn;
 804
 805         cmnd->pdu.bhs.sn = cmd_sn = be32_to_cpu(cmnd->pdu.bhs.sn);
 806         TRACE_DBG("%d(%d)", cmd_sn, session->exp_cmd_sn);
 807         if (likely((s32)(cmd_sn - session->exp_cmd_sn) >= 0))
 808                 return 0;
 809         PRINT_ERROR("sequence error (%x,%x)", cmd_sn, session->exp_cmd_sn);
 810         return -ISCSI_REASON_PROTOCOL_ERROR;
 811 }
 812
 813 static inline struct iscsi_cmnd *__cmnd_find_hash(struct iscsi_session *session,
 814         u32 itt, u32 ttt)
 815 {
 816         struct list_head *head;
 817         struct iscsi_cmnd *cmnd;
 818
 819         head = &session->cmnd_hash[cmnd_hashfn(itt)];
 820
 821         list_for_each_entry(cmnd, head, hash_list_entry) {
 822                 if (cmnd->pdu.bhs.itt == itt) {
 823                         if ((ttt != ISCSI_RESERVED_TAG) && (ttt != cmnd->target_task_tag))
 824                                 continue;
 825                         return cmnd;
 826                 }
 827         }
 828         return NULL;
 829 }
 830
 831 static struct iscsi_cmnd *cmnd_find_hash(struct iscsi_session *session,
 832         u32 itt, u32 ttt)
 833 {
 834         struct iscsi_cmnd *cmnd;
 835
 836         spin_lock(&session->cmnd_hash_lock);
 837         cmnd = __cmnd_find_hash(session, itt, ttt);
 838         spin_unlock(&session->cmnd_hash_lock);
 839
 840         return cmnd;
 841 }
 842
 843 static struct iscsi_cmnd *cmnd_find_hash_get(struct iscsi_session *session,
 844         u32 itt, u32 ttt)
 845 {
 846         struct iscsi_cmnd *cmnd;
 847
 848         spin_lock(&session->cmnd_hash_lock);
 849         cmnd = __cmnd_find_hash(session, itt, ttt);
 850         if (cmnd != NULL) {
 851                 if (unlikely(cmnd_get_check(cmnd)))
 852                         cmnd = NULL;
 853         }
 854         spin_unlock(&session->cmnd_hash_lock);
 855
 856         return cmnd;
 857 }
 858
 859 static int cmnd_insert_hash(struct iscsi_cmnd *cmnd)
 860 {
 861         struct iscsi_session *session = cmnd->conn->session;
 862         struct iscsi_cmnd *tmp;
 863         struct list_head *head;
 864         int err = 0;
 865         u32 itt = cmnd->pdu.bhs.itt;
 866
 867         TRACE_DBG("%p:%x", cmnd, itt);
 868         if (unlikely(itt == ISCSI_RESERVED_TAG)) {
 869                 PRINT_ERROR("%s", "ITT is RESERVED_TAG");
 870                 PRINT_BUFFER("Incorrect BHS", &cmnd->pdu.bhs,
 871                         sizeof(cmnd->pdu.bhs));
 872                 err = -ISCSI_REASON_PROTOCOL_ERROR;
 873                 goto out;
 874         }
 875
 876         spin_lock(&session->cmnd_hash_lock);
 877
 878         head = &session->cmnd_hash[cmnd_hashfn(cmnd->pdu.bhs.itt)];
 879
 880         tmp = __cmnd_find_hash(session, itt, ISCSI_RESERVED_TAG);
 881         if (likely(!tmp)) {
 882                 list_add_tail(&cmnd->hash_list_entry, head);
 883                 cmnd->hashed = 1;
 884         } else {
 885                 PRINT_ERROR("Task %x in progress, cmnd %p", itt, cmnd);
 886                 err = -ISCSI_REASON_TASK_IN_PROGRESS;
 887         }
 888
 889         spin_unlock(&session->cmnd_hash_lock);
 890
 891         if (likely(!err)) {
 892                 spin_lock(&session->sn_lock);
 893                 __update_stat_sn(cmnd);
 894                 err = check_cmd_sn(cmnd);
 895                 spin_unlock(&session->sn_lock);
 896         }
 897
 898 out:
 899         return err;
 900 }
 901
 902 static void cmnd_remove_hash(struct iscsi_cmnd *cmnd)
 903 {
 904         struct iscsi_session *session = cmnd->conn->session;
 905         struct iscsi_cmnd *tmp;
 906
 907         spin_lock(&session->cmnd_hash_lock);
 908
 909         tmp = __cmnd_find_hash(session, cmnd->pdu.bhs.itt, ISCSI_RESERVED_TAG);
 910
 911         if (likely(tmp && tmp == cmnd)) {
 912                 list_del(&cmnd->hash_list_entry);
 913                 cmnd->hashed = 0;
 914         } else {
 915                 PRINT_ERROR("%p:%x not found", cmnd, cmnd_itt(cmnd));
 916         }
 917
 918         spin_unlock(&session->cmnd_hash_lock);
 919 }
 920
 921 static void cmnd_prepare_get_rejected_cmd_data(struct iscsi_cmnd *cmnd)
 922 {
 923         struct iscsi_conn *conn = cmnd->conn;
 924         struct scatterlist *sg = cmnd->sg;
 925         char *addr;
 926         u32 size;
 927         int i;
 928
 929         TRACE_MGMT_DBG("Skipping (%p, %x %x %x %u, %p, scst state %d)", cmnd,
 930                 cmnd_itt(cmnd), cmnd_opcode(cmnd), cmnd_hdr(cmnd)->scb[0],
 931                 cmnd->pdu.datasize, cmnd->scst_cmd, cmnd->scst_state);
 932
 933         iscsi_extracheck_is_rd_thread(conn);
 934
 935         size = cmnd->pdu.datasize;
 936         if (!size)
 937                 return;
 938
 939         if (sg == NULL) {
 940                 /* ToDo: __GFP_NOFAIL ?? */
 941                 sg = cmnd->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
 942                                         &cmnd->sg_cnt);
 943                 if (sg == NULL) {
 944                         ;/* ToDo */;
 945                 }
 946                 cmnd->own_sg = 1;
 947                 cmnd->bufflen = PAGE_SIZE;
 948         }
 949
 950         addr = page_address(sg_page(&sg[0]));
 951         sBUG_ON(addr == NULL);
 952         size = (size + 3) & -4;
 953         conn->read_size = size;
 954         for (i = 0; size > PAGE_SIZE; i++, size -= cmnd->bufflen) {
 955                 sBUG_ON(i >= ISCSI_CONN_IOV_MAX);
 956                 conn->read_iov[i].iov_base = addr;
 957                 conn->read_iov[i].iov_len = cmnd->bufflen;
 958         }
 959         conn->read_iov[i].iov_base = addr;
 960         conn->read_iov[i].iov_len = size;
 961         conn->read_msg.msg_iov = conn->read_iov;
 962         conn->read_msg.msg_iovlen = ++i;
 963
 964         return;
 965 }
 966
 967 static void cmnd_reject_scsi_cmd(struct iscsi_cmnd *req)
 968 {
 969         struct iscsi_cmnd *rsp;
 970         struct iscsi_scsi_rsp_hdr *rsp_hdr;
 971         u32 size;
 972
 973         TRACE_DBG("%p", req);
 974
 975         sBUG_ON(req->rejected);
 976         req->rejected = 1;
 977         req->reject_reason = ISCSI_REJECT_SCSI_CMD;
 978
 979         rsp = get_rsp_cmnd(req);
 980         if (rsp == NULL) {
 981                 /* That can be true for aborted commands */
 982                 goto out_reject;
 983         }
 984
 985         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
 986
 987         sBUG_ON(cmnd_opcode(rsp) != ISCSI_OP_SCSI_RSP);
 988
 989         size = cmnd_write_size(req);
 990         if (size) {
 991                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
 992                 rsp_hdr->residual_count = cpu_to_be32(size);
 993         }
 994         size = cmnd_read_size(req);
 995         if (size) {
 996                 if (cmnd_hdr(req)->flags & ISCSI_CMD_WRITE) {
 997                         rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_UNDERFLOW;
 998                         rsp_hdr->bi_residual_count = cpu_to_be32(size);
 999                 } else {
1000                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
1001                         rsp_hdr->residual_count = cpu_to_be32(size);
1002                 }
1003         }
1004
1005         iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH |
1006                                          ISCSI_INIT_WRITE_WAKE);
1007
1008 out_reject:
1009         cmnd_prepare_get_rejected_cmd_data(req);
1010         return;
1011 }
1012
1013 static int cmnd_prepare_recv_pdu(struct iscsi_conn *conn,
1014         struct iscsi_cmnd *cmd, u32 offset, u32 size)
1015 {
1016         struct scatterlist *sg = cmd->sg;
1017         int bufflen = cmd->bufflen;
1018         int idx, i;
1019         char *addr;
1020         int res = 0;
1021
1022         TRACE_DBG("%p %u,%u", cmd->sg, offset, size);
1023
1024         iscsi_extracheck_is_rd_thread(conn);
1025
1026         if (unlikely((offset >= bufflen) ||
1027                      (offset + size > bufflen))) {
1028                 PRINT_ERROR("Wrong ltn (%u %u %u)", offset, size, bufflen);
1029                 mark_conn_closed(conn);
1030                 res = -EIO;
1031                 goto out;
1032         }
1033
1034         offset += sg[0].offset;
1035         idx = offset >> PAGE_SHIFT;
1036         offset &= ~PAGE_MASK;
1037
1038         conn->read_msg.msg_iov = conn->read_iov;
1039         conn->read_size = size = (size + 3) & -4;
1040
1041         i = 0;
1042         while (1) {
1043                 addr = page_address(sg_page(&sg[idx]));
1044                 sBUG_ON(addr == NULL);
1045                 conn->read_iov[i].iov_base = addr + offset;
1046                 if (offset + size <= PAGE_SIZE) {
1047                         TRACE_DBG("idx=%d, offset=%u, size=%d, addr=%p",
1048                                 idx, offset, size, addr);
1049                         conn->read_iov[i].iov_len = size;
1050                         conn->read_msg.msg_iovlen = ++i;
1051                         break;
1052                 }
1053                 conn->read_iov[i].iov_len = PAGE_SIZE - offset;
1054                 TRACE_DBG("idx=%d, offset=%u, size=%d, iov_len=%zd, addr=%p",
1055                         idx, offset, size, conn->read_iov[i].iov_len, addr);
1056                 size -= conn->read_iov[i].iov_len;
1057                 offset = 0;
1058                 if (unlikely(++i >= ISCSI_CONN_IOV_MAX)) {
1059                         PRINT_ERROR("Initiator %s violated negotiated "
1060                                 "parameters by sending too much data (size "
1061                                 "left %d)", conn->session->initiator_name, size);
1062                         mark_conn_closed(conn);
1063                         res = -EINVAL;
1064                         break;
1065                 }
1066                 idx++;
1067         }
1068         TRACE_DBG("msg_iov=%p, msg_iovlen=%zd",
1069                 conn->read_msg.msg_iov, conn->read_msg.msg_iovlen);
1070
1071 out:
1072         return res;
1073 }
1074
1075 static void send_r2t(struct iscsi_cmnd *req)
1076 {
1077         struct iscsi_session *session = req->conn->session;
1078         struct iscsi_cmnd *rsp;
1079         struct iscsi_r2t_hdr *rsp_hdr;
1080         u32 offset, burst;
1081         LIST_HEAD(send);
1082
1083         if (unlikely(req->tm_aborted)) {
1084                 TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted on R2T "
1085                         "(r2t_length %d, outstanding_r2t %d)", req,
1086                         req->scst_cmd, req->r2t_length, req->outstanding_r2t);
1087                 if (req->outstanding_r2t == 0)
1088                         iscsi_fail_waiting_cmnd(req);
1089                 goto out;
1090         }
1091
1092         /*
1093          * There is no race with data_out_start() and conn_abort(), since
1094          * all functions called from single read thread
1095          */
1096         iscsi_extracheck_is_rd_thread(req->conn);
1097
1098         burst = session->sess_param.max_burst_length;
1099         offset = be32_to_cpu(cmnd_hdr(req)->data_length) - req->r2t_length;
1100
1101         do {
1102                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
1103                 rsp->pdu.bhs.ttt = req->target_task_tag;
1104                 rsp_hdr = (struct iscsi_r2t_hdr *)&rsp->pdu.bhs;
1105                 rsp_hdr->opcode = ISCSI_OP_R2T;
1106                 rsp_hdr->flags = ISCSI_FLG_FINAL;
1107                 rsp_hdr->lun = cmnd_hdr(req)->lun;
1108                 rsp_hdr->itt = cmnd_hdr(req)->itt;
1109                 rsp_hdr->r2t_sn = cpu_to_be32(req->r2t_sn++);
1110                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
1111                 if (req->r2t_length > burst) {
1112                         rsp_hdr->data_length = cpu_to_be32(burst);
1113                         req->r2t_length -= burst;
1114                         offset += burst;
1115                 } else {
1116                         rsp_hdr->data_length = cpu_to_be32(req->r2t_length);
1117                         req->r2t_length = 0;
1118                 }
1119
1120                 TRACE_WRITE("%x %u %u %u %u", cmnd_itt(req),
1121                         be32_to_cpu(rsp_hdr->data_length),
1122                         be32_to_cpu(rsp_hdr->buffer_offset),
1123                         be32_to_cpu(rsp_hdr->r2t_sn), req->outstanding_r2t);
1124
1125                 list_add_tail(&rsp->write_list_entry, &send);
1126
1127                 if (++req->outstanding_r2t >= session->sess_param.max_outstanding_r2t)
1128                         break;
1129
1130         } while (req->r2t_length != 0);
1131
1132         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_WAKE);
1133
1134 out:
1135         return;
1136 }
1137
1138 static int iscsi_pre_exec(struct scst_cmd *scst_cmd)
1139 {
1140         int res = SCST_PREPROCESS_STATUS_SUCCESS;
1141         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
1142                 scst_cmd_get_tgt_priv(scst_cmd);
1143         struct iscsi_cmnd *c, *t;
1144
1145         TRACE_ENTRY();
1146
1147         EXTRACHECKS_BUG_ON(scst_cmd_atomic(scst_cmd));
1148
1149         if (scst_cmd_get_data_direction(scst_cmd) == SCST_DATA_READ) {
1150                 if (!(req->conn->ddigest_type & DIGEST_NONE))
1151                         scst_set_long_xmit(scst_cmd);
1152 #ifndef NET_PAGE_CALLBACKS_DEFINED
1153                 else if (cmnd_hdr(req)->data_length > 8*1024)
1154                         scst_set_long_xmit(scst_cmd);
1155 #endif
1156                 EXTRACHECKS_BUG_ON(!list_empty(&req->rx_ddigest_cmd_list));
1157                 goto out;
1158         }
1159
1160         /* If data digest isn't used this list will be empty */
1161         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
1162                                 rx_ddigest_cmd_list_entry) {
1163                 TRACE_DBG("Checking digest of RX ddigest cmd %p", c);
1164                 if (digest_rx_data(c) != 0) {
1165                         scst_set_cmd_error(scst_cmd,
1166                                 SCST_LOAD_SENSE(iscsi_sense_crc_error));
1167                         res = SCST_PREPROCESS_STATUS_ERROR_SENSE_SET;
1168                         /*
1169                          * The rest of rx_ddigest_cmd_list will be freed
1170                          * in req_cmnd_release()
1171                          */
1172                         goto out;
1173                 }
1174                 cmd_del_from_rx_ddigest_list(c);
1175                 cmnd_put(c);
1176         }
1177
1178 out:
1179         TRACE_EXIT_RES(res);
1180         return res;
1181 }
1182
1183 static int noop_out_start(struct iscsi_cmnd *cmnd)
1184 {
1185         struct iscsi_conn *conn = cmnd->conn;
1186         u32 size, tmp;
1187         int i, err = 0;
1188
1189         TRACE_DBG("%p", cmnd);
1190
1191         iscsi_extracheck_is_rd_thread(conn);
1192
1193         if (unlikely(cmnd_ttt(cmnd) != cpu_to_be32(ISCSI_RESERVED_TAG))) {
1194                 /*
1195                  * We don't request a NOP-Out by sending a NOP-In.
1196                  * See 10.18.2 in the draft 20.
1197                  */
1198                 PRINT_ERROR("Initiator sent command with not RESERVED tag and "
1199                         "TTT %x", cmnd_itt(cmnd));
1200                 err = -ISCSI_REASON_PROTOCOL_ERROR;
1201                 goto out;
1202         }
1203
1204         if (cmnd_itt(cmnd) == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1205                 if (unlikely(!(cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE)))
1206                         PRINT_ERROR("%s", "Initiator sent RESERVED tag for "
1207                                 "non-immediate command");
1208                 spin_lock(&conn->session->sn_lock);
1209                 __update_stat_sn(cmnd);
1210                 err = check_cmd_sn(cmnd);
1211                 spin_unlock(&conn->session->sn_lock);
1212                 if (unlikely(err))
1213                         goto out;
1214         } else {
1215                 err = cmnd_insert_hash(cmnd);
1216                 if (unlikely(err < 0)) {
1217                         PRINT_ERROR("Can't insert in hash: ignore this request %x",
1218                                     cmnd_itt(cmnd));
1219                         goto out;
1220                 }
1221         }
1222
1223         size = cmnd->pdu.datasize;
1224         if (size) {
1225                 size = (size + 3) & -4;
1226                 conn->read_msg.msg_iov = conn->read_iov;
1227                 if (cmnd->pdu.bhs.itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1228                         struct scatterlist *sg;
1229
1230                         /* ToDo: __GFP_NOFAIL ?? */
1231                         cmnd->sg = sg = scst_alloc(size,
1232                                 GFP_KERNEL|__GFP_NOFAIL, &cmnd->sg_cnt);
1233                         if (sg == NULL) {
1234                                 ;/* ToDo */;
1235                         }
1236                         if (cmnd->sg_cnt > ISCSI_CONN_IOV_MAX) {
1237                                 ;/* ToDo */;
1238                         }
1239                         cmnd->own_sg = 1;
1240                         cmnd->bufflen = size;
1241
1242                         for (i = 0; i < cmnd->sg_cnt; i++) {
1243                                 conn->read_iov[i].iov_base =
1244                                         page_address(sg_page(&sg[i]));
1245                                 tmp = min_t(u32, size, PAGE_SIZE);
1246                                 conn->read_iov[i].iov_len = tmp;
1247                                 conn->read_size += tmp;
1248                                 size -= tmp;
1249                         }
1250                 } else {
1251                         /*
1252                          * There are no problems with the safety from concurrent
1253                          * accesses to dummy_data, since for ISCSI_RESERVED_TAG
1254                          * the data only read and then discarded.
1255                          */
1256                         for (i = 0; i < ISCSI_CONN_IOV_MAX; i++) {
1257                                 conn->read_iov[i].iov_base = dummy_data;
1258                                 tmp = min_t(u32, size, sizeof(dummy_data));
1259                                 conn->read_iov[i].iov_len = tmp;
1260                                 conn->read_size += tmp;
1261                                 size -= tmp;
1262                         }
1263                 }
1264                 sBUG_ON(size == 0);
1265                 conn->read_msg.msg_iovlen = i;
1266                 TRACE_DBG("msg_iov=%p, msg_iovlen=%zd", conn->read_msg.msg_iov,
1267                         conn->read_msg.msg_iovlen);
1268         }
1269 out:
1270         return err;
1271 }
1272
1273 static inline u32 get_next_ttt(struct iscsi_conn *conn)
1274 {
1275         u32 ttt;
1276         struct iscsi_session *session = conn->session;
1277
1278         iscsi_extracheck_is_rd_thread(conn);
1279
1280         if (session->next_ttt == ISCSI_RESERVED_TAG)
1281                 session->next_ttt++;
1282         ttt = session->next_ttt++;
1283
1284         return cpu_to_be32(ttt);
1285 }
1286
1287 static int scsi_cmnd_start(struct iscsi_cmnd *req)
1288 {
1289         struct iscsi_conn *conn = req->conn;
1290         struct iscsi_session *session = conn->session;
1291         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1292         struct scst_cmd *scst_cmd;
1293         scst_data_direction dir;
1294         int res = 0;
1295
1296         TRACE_ENTRY();
1297
1298         TRACE_DBG("scsi command: %02x", req_hdr->scb[0]);
1299
1300         TRACE_DBG("Incrementing active_cmds (cmd %p, sess %p, "
1301                 "new value %d)", req, session,
1302                 atomic_read(&session->active_cmds)+1);
1303         atomic_inc(&session->active_cmds);
1304         req->dec_active_cmnds = 1;
1305
1306         scst_cmd = scst_rx_cmd(session->scst_sess,
1307                 (uint8_t *)&req_hdr->lun, sizeof(req_hdr->lun),
1308                 req_hdr->scb, sizeof(req_hdr->scb), SCST_NON_ATOMIC);
1309         if (scst_cmd == NULL) {
1310                 create_status_rsp(req, SAM_STAT_BUSY, NULL, 0);
1311                 cmnd_reject_scsi_cmd(req);
1312                 goto out;
1313         }
1314
1315         req->scst_cmd = scst_cmd;
1316         scst_cmd_set_tag(scst_cmd, req_hdr->itt);
1317         scst_cmd_set_tgt_priv(scst_cmd, req);
1318 #ifndef NET_PAGE_CALLBACKS_DEFINED
1319         scst_cmd_set_data_buf_tgt_alloc(scst_cmd);
1320 #endif
1321
1322         if (req_hdr->flags & ISCSI_CMD_READ)
1323                 dir = SCST_DATA_READ;
1324         else if (req_hdr->flags & ISCSI_CMD_WRITE)
1325                 dir = SCST_DATA_WRITE;
1326         else
1327                 dir = SCST_DATA_NONE;
1328         scst_cmd_set_expected(scst_cmd, dir, be32_to_cpu(req_hdr->data_length));
1329
1330         switch (req_hdr->flags & ISCSI_CMD_ATTR_MASK) {
1331         case ISCSI_CMD_SIMPLE:
1332                 scst_cmd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1333                 break;
1334         case ISCSI_CMD_HEAD_OF_QUEUE:
1335                 scst_cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1336                 break;
1337         case ISCSI_CMD_ORDERED:
1338                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1339                 break;
1340         case ISCSI_CMD_ACA:
1341                 scst_cmd->queue_type = SCST_CMD_QUEUE_ACA;
1342                 break;
1343         case ISCSI_CMD_UNTAGGED:
1344                 scst_cmd->queue_type = SCST_CMD_QUEUE_UNTAGGED;
1345                 break;
1346         default:
1347                 PRINT_ERROR("Unknown task code %x, use ORDERED instead",
1348                         req_hdr->flags & ISCSI_CMD_ATTR_MASK);
1349                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1350                 break;
1351         }
1352
1353         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
1354         scst_cmd_set_tgt_sn(scst_cmd, req_hdr->cmd_sn);
1355
1356         TRACE_DBG("START Command (tag %d, queue_type %d)",
1357                 req_hdr->itt, scst_cmd->queue_type);
1358         req->scst_state = ISCSI_CMD_STATE_RX_CMD;
1359         scst_cmd_init_stage1_done(scst_cmd, SCST_CONTEXT_DIRECT, 0);
1360
1361         wait_event(req->scst_waitQ, (req->scst_state != ISCSI_CMD_STATE_RX_CMD));
1362
1363         if (unlikely(req->scst_state != ISCSI_CMD_STATE_AFTER_PREPROC)) {
1364                 TRACE_DBG("req %p is in %x state", req, req->scst_state);
1365                 if (req->scst_state == ISCSI_CMD_STATE_PROCESSED) {
1366                         cmnd_reject_scsi_cmd(req);
1367                         goto out;
1368                 }
1369                 if (unlikely(req->tm_aborted)) {
1370                         TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
1371                                 req->scst_cmd);
1372                         cmnd_prepare_get_rejected_cmd_data(req);
1373                         goto out;
1374                 }
1375                 sBUG();
1376         }
1377
1378         dir = scst_cmd_get_data_direction(scst_cmd);
1379         if (dir != SCST_DATA_WRITE) {
1380                 if (unlikely(!(req_hdr->flags & ISCSI_CMD_FINAL) ||
1381                              req->pdu.datasize)) {
1382                         PRINT_ERROR("Unexpected unsolicited data (ITT %x "
1383                                 "CDB %x", cmnd_itt(req), req_hdr->scb[0]);
1384                         create_sense_rsp(req, ABORTED_COMMAND, 0xc, 0xc);
1385                         cmnd_reject_scsi_cmd(req);
1386                         goto out;
1387                 }
1388         }
1389
1390         if (dir == SCST_DATA_WRITE) {
1391                 req->is_unsolicited_data = !(req_hdr->flags & ISCSI_CMD_FINAL);
1392                 req->r2t_length = be32_to_cpu(req_hdr->data_length) - req->pdu.datasize;
1393                 if (req->r2t_length > 0)
1394                         req->data_waiting = 1;
1395         }
1396         req->target_task_tag = get_next_ttt(conn);
1397         req->sg = scst_cmd_get_sg(scst_cmd);
1398         req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
1399         req->bufflen = scst_cmd_get_bufflen(scst_cmd);
1400         if (unlikely(req->r2t_length > req->bufflen)) {
1401                 PRINT_ERROR("req->r2t_length %d > req->bufflen %d",
1402                         req->r2t_length, req->bufflen);
1403                 req->r2t_length = req->bufflen;
1404         }
1405
1406         TRACE_DBG("req=%p, dir=%d, is_unsolicited_data=%d, "
1407                 "r2t_length=%d, bufflen=%d", req, dir,
1408                 req->is_unsolicited_data, req->r2t_length, req->bufflen);
1409
1410         if (unlikely(!session->sess_param.immediate_data &&
1411                      req->pdu.datasize)) {
1412                 PRINT_ERROR("Initiator %s violated negotiated paremeters: "
1413                         "forbidden immediate data sent (ITT %x, op  %x)",
1414                         session->initiator_name, cmnd_itt(req), req_hdr->scb[0]);
1415                 res = -EINVAL;
1416                 goto out;
1417         }
1418
1419         if (unlikely(session->sess_param.initial_r2t &&
1420                      !(req_hdr->flags & ISCSI_CMD_FINAL))) {
1421                 PRINT_ERROR("Initiator %s violated negotiated paremeters: "
1422                         "initial R2T is required (ITT %x, op  %x)",
1423                         session->initiator_name, cmnd_itt(req), req_hdr->scb[0]);
1424                 res = -EINVAL;
1425                 goto out;
1426         }
1427
1428         if (req->pdu.datasize) {
1429                 if (unlikely(dir != SCST_DATA_WRITE)) {
1430                         PRINT_ERROR("pdu.datasize(%d) >0, but dir(%x) isn't WRITE",
1431                                 req->pdu.datasize, dir);
1432                         create_sense_rsp(req, ABORTED_COMMAND, 0xc, 0xc);
1433                         cmnd_reject_scsi_cmd(req);
1434                 } else
1435                         res = cmnd_prepare_recv_pdu(conn, req, 0, req->pdu.datasize);
1436         }
1437 out:
1438         /* Aborted commands will be freed in cmnd_rx_end() */
1439         TRACE_EXIT_RES(res);
1440         return res;
1441 }
1442
1443 static int data_out_start(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
1444 {
1445         struct iscsi_data_out_hdr *req_hdr = (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1446         struct iscsi_cmnd *orig_req = NULL;
1447         u32 offset = be32_to_cpu(req_hdr->buffer_offset);
1448         int res = 0;
1449
1450         TRACE_ENTRY();
1451
1452         /*
1453          * There is no race with send_r2t() and conn_abort(), since
1454          * all functions called from single read thread
1455          */
1456         iscsi_extracheck_is_rd_thread(cmnd->conn);
1457
1458         update_stat_sn(cmnd);
1459
1460         cmnd->cmd_req = orig_req = cmnd_find_hash(conn->session, req_hdr->itt,
1461                                         req_hdr->ttt);
1462         if (unlikely(orig_req == NULL)) {
1463                 /* It might happen if req was aborted and then freed */
1464                 TRACE(TRACE_MGMT_MINOR, "Unable to find scsi task %x %x",
1465                         cmnd_itt(cmnd), cmnd_ttt(cmnd));
1466                 goto out_reject;
1467         }
1468
1469         if (orig_req->is_unsolicited_data) {
1470                 if (unlikely(orig_req->r2t_length < cmnd->pdu.datasize)) {
1471                         PRINT_ERROR("Data size (%d) > R2T length (%d)",
1472                                 cmnd->pdu.datasize, orig_req->r2t_length);
1473                         mark_conn_closed(conn);
1474                         res = -EINVAL;
1475                         goto out;
1476                 }
1477                 orig_req->r2t_length -= cmnd->pdu.datasize;
1478         }
1479
1480         /* Check unsolicited burst data */
1481         if (unlikely((req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) &&
1482                      (orig_req->pdu.bhs.flags & ISCSI_FLG_FINAL))) {
1483                 PRINT_ERROR("Unexpected data from %x %x",
1484                         cmnd_itt(cmnd), cmnd_ttt(cmnd));
1485                 mark_conn_closed(conn);
1486                 res = -EINVAL;
1487                 goto out;
1488         }
1489
1490         TRACE_WRITE("%u %p %p %u %u", req_hdr->ttt, cmnd, orig_req,
1491                 offset, cmnd->pdu.datasize);
1492
1493         res = cmnd_prepare_recv_pdu(conn, orig_req, offset, cmnd->pdu.datasize);
1494
1495 out:
1496         TRACE_EXIT_RES(res);
1497         return res;
1498
1499 out_reject:
1500         sBUG_ON(cmnd->rejected);
1501         cmnd->rejected = 1;
1502         cmnd->reject_reason = ISCSI_REJECT_DATA;
1503         cmnd_prepare_get_rejected_cmd_data(cmnd);
1504         goto out;
1505 }
1506
1507 static void data_out_end(struct iscsi_cmnd *cmnd)
1508 {
1509         struct iscsi_data_out_hdr *req_hdr = (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1510         struct iscsi_cmnd *req;
1511
1512         sBUG_ON(cmnd == NULL);
1513         req = cmnd->cmd_req;
1514         sBUG_ON(req == NULL);
1515
1516         TRACE_DBG("cmnd %p, req %p", cmnd, req);
1517
1518         iscsi_extracheck_is_rd_thread(cmnd->conn);
1519
1520         if (!(cmnd->conn->ddigest_type & DIGEST_NONE) &&
1521             !cmnd->ddigest_checked) {
1522                 cmd_add_on_rx_ddigest_list(req, cmnd);
1523                 cmnd_get(cmnd);
1524         }
1525
1526         if (req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1527                 TRACE_DBG("ISCSI_RESERVED_TAG, FINAL %x",
1528                         req_hdr->flags & ISCSI_FLG_FINAL);
1529
1530                 if (req_hdr->flags & ISCSI_FLG_FINAL) {
1531                         req->is_unsolicited_data = 0;
1532                         if (req->pending)
1533                                 goto out_put;
1534                 } else
1535                         goto out_put;
1536         } else {
1537                 TRACE_DBG("FINAL %x, outstanding_r2t %d, r2t_length %d",
1538                         req_hdr->flags & ISCSI_FLG_FINAL,
1539                         req->outstanding_r2t, req->r2t_length);
1540
1541                 if (req_hdr->flags & ISCSI_FLG_FINAL) {
1542                         if (unlikely(req->is_unsolicited_data)) {
1543                                 PRINT_ERROR("Unexpected unsolicited data "
1544                                         "(r2t_length %u, outstanding_r2t %d)",
1545                                         req->r2t_length, req->is_unsolicited_data);
1546                                 mark_conn_closed(req->conn);
1547                                 goto out_put;
1548                         }
1549                         req->outstanding_r2t--;
1550                 } else
1551                         goto out_put;
1552         }
1553
1554         if (req->r2t_length != 0) {
1555                 if (!req->is_unsolicited_data)
1556                         send_r2t(req);
1557         } else
1558                 iscsi_restart_waiting_cmnd(req);
1559
1560 out_put:
1561         cmnd_put(cmnd);
1562         return;
1563 }
1564
1565 static void __cmnd_abort(struct iscsi_cmnd *cmnd)
1566 {
1567         /*
1568          * Here, if cmnd is data_waiting, we should iscsi_fail_waiting_cmnd()
1569          * it. But, since this function can be called from any thread, not only
1570          * from the read one, we at the moment can't do that, because of
1571          * absence of appropriate locking protection. But this isn't a stuff
1572          * for 1.0.0. So, currently a misbehaving initiator, not sending
1573          * data in R2T state for a sharing between targets device, for which
1574          * for some reason an aborting TM command, e.g. TARGET RESET, from
1575          * another initiator is issued, can block response for this TM command
1576          * virtually forever and by this make the issuing initiator eventually
1577          * put the device offline.
1578          *
1579          * ToDo in the next version, possibly a simple connection mutex, taken
1580          * by the read thread before starting any processing and by this
1581          * function, should be sufficient.
1582          */
1583
1584         TRACE_MGMT_DBG("Aborting cmd %p, scst_cmd %p (scst state %x, "
1585                 "ref_cnt %d, itt %x, sn %u, op %x, r2t_len %x, CDB op %x, "
1586                 "size to write %u, is_unsolicited_data %d, "
1587                 "outstanding_r2t %d, data_waiting %d, sess->exp_cmd_sn %u, "
1588                 "conn %p, rd_task %p)", cmnd, cmnd->scst_cmd, cmnd->scst_state,
1589                 atomic_read(&cmnd->ref_cnt), cmnd_itt(cmnd), cmnd->pdu.bhs.sn,
1590                 cmnd_opcode(cmnd), cmnd->r2t_length, cmnd_scsicode(cmnd),
1591                 cmnd_write_size(cmnd), cmnd->is_unsolicited_data,
1592                 cmnd->outstanding_r2t, cmnd->data_waiting,
1593                 cmnd->conn->session->exp_cmd_sn, cmnd->conn,
1594                 cmnd->conn->rd_task);
1595
1596 #ifdef NET_PAGE_CALLBACKS_DEFINED
1597         TRACE_MGMT_DBG("net_ref_cnt %d", atomic_read(&cmnd->net_ref_cnt));
1598 #endif
1599
1600         cmnd->tm_aborted = 1;
1601
1602         return;
1603 }
1604
1605 /* Must be called from the read thread */
1606 static int cmnd_abort(struct iscsi_cmnd *req)
1607 {
1608         struct iscsi_session *session = req->conn->session;
1609         struct iscsi_task_mgt_hdr *req_hdr =
1610                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1611         struct iscsi_cmnd *cmnd;
1612         int err;
1613
1614         req_hdr->ref_cmd_sn = be32_to_cpu(req_hdr->ref_cmd_sn);
1615
1616         if (after(req_hdr->ref_cmd_sn, req_hdr->cmd_sn)) {
1617                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) > CmdSN(%u)",
1618                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn);
1619                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1620                 goto out;
1621         }
1622
1623         cmnd = cmnd_find_hash_get(session, req_hdr->rtt, ISCSI_RESERVED_TAG);
1624         if (cmnd) {
1625                 struct iscsi_conn *conn = cmnd->conn;
1626                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
1627
1628                 if (req_hdr->lun != hdr->lun) {
1629                          PRINT_ERROR("ABORT TASK: LUN mismatch: req LUN "
1630                                      "%Lx, cmd LUN %Lx, rtt %u",
1631                                      (long long unsigned int)req_hdr->lun,
1632                                      (long long unsigned int)hdr->lun,
1633                                      req_hdr->rtt);
1634                         err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1635                         goto out_put;
1636                 }
1637
1638                 if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
1639                         if (req_hdr->ref_cmd_sn != req_hdr->cmd_sn) {
1640                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != TM cmd "
1641                                         "CmdSN(%u) for immediate command %p",
1642                                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn,
1643                                         cmnd);
1644                                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1645                                 goto out_put;
1646                         }
1647                 } else {
1648                         if (req_hdr->ref_cmd_sn != hdr->cmd_sn) {
1649                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != "
1650                                         "CmdSN(%u) for command %p",
1651                                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn,
1652                                         cmnd);
1653                                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1654                                 goto out_put;
1655                         }
1656                 }
1657
1658                 if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
1659                     (req_hdr->cmd_sn == hdr->cmd_sn)) {
1660                         PRINT_ERROR("ABORT TASK: SN mismatch: req SN %x, "
1661                                 "cmd SN %x, rtt %u", req_hdr->cmd_sn,
1662                                 hdr->cmd_sn, req_hdr->rtt);
1663                         err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1664                         goto out_put;
1665                 }
1666
1667                 spin_lock_bh(&conn->cmd_list_lock);
1668                 __cmnd_abort(cmnd);
1669                 spin_unlock_bh(&conn->cmd_list_lock);
1670
1671                 cmnd_put(cmnd);
1672                 err = 0;
1673         } else {
1674                 TRACE_MGMT_DBG("cmd RTT %x not found", req_hdr->rtt);
1675                 err = ISCSI_RESPONSE_UNKNOWN_TASK;
1676         }
1677
1678 out:
1679         return err;
1680
1681 out_put:
1682         cmnd_put(cmnd);
1683         goto out;
1684 }
1685
1686 /* Must be called from the read thread */
1687 static int target_abort(struct iscsi_cmnd *req, int all)
1688 {
1689         struct iscsi_target *target = req->conn->session->target;
1690         struct iscsi_task_mgt_hdr *req_hdr =
1691                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1692         struct iscsi_session *session;
1693         struct iscsi_conn *conn;
1694         struct iscsi_cmnd *cmnd;
1695
1696         mutex_lock(&target->target_mutex);
1697
1698         list_for_each_entry(session, &target->session_list, session_list_entry) {
1699                 list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
1700                         spin_lock_bh(&conn->cmd_list_lock);
1701                         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1702                                 if (cmnd == req)
1703                                         continue;
1704                                 if (all)
1705                                         __cmnd_abort(cmnd);
1706                                 else if (req_hdr->lun == cmnd_hdr(cmnd)->lun)
1707                                         __cmnd_abort(cmnd);
1708                         }
1709                         spin_unlock_bh(&conn->cmd_list_lock);
1710                 }
1711         }
1712
1713         mutex_unlock(&target->target_mutex);
1714         return 0;
1715 }
1716
1717 /* Must be called from the read thread */
1718 static void task_set_abort(struct iscsi_cmnd *req)
1719 {
1720         struct iscsi_session *session = req->conn->session;
1721         struct iscsi_task_mgt_hdr *req_hdr =
1722                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1723         struct iscsi_target *target = session->target;
1724         struct iscsi_conn *conn;
1725         struct iscsi_cmnd *cmnd;
1726
1727         mutex_lock(&target->target_mutex);
1728
1729         list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
1730                 spin_lock_bh(&conn->cmd_list_lock);
1731                 list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1732                         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
1733                         if (cmnd == req)
1734                                 continue;
1735                         if (req_hdr->lun != hdr->lun)
1736                                 continue;
1737                         if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
1738                             req_hdr->cmd_sn == hdr->cmd_sn)
1739                                 continue;
1740                         __cmnd_abort(cmnd);
1741                 }
1742                 spin_unlock_bh(&conn->cmd_list_lock);
1743         }
1744
1745         mutex_unlock(&target->target_mutex);
1746         return;
1747 }
1748
1749 /* Must be called from the read thread */
1750 void conn_abort(struct iscsi_conn *conn)
1751 {
1752         struct iscsi_cmnd *cmnd;
1753
1754         TRACE_MGMT_DBG("Aborting conn %p", conn);
1755
1756         iscsi_extracheck_is_rd_thread(conn);
1757
1758         spin_lock_bh(&conn->cmd_list_lock);
1759 again:
1760         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1761                 __cmnd_abort(cmnd);
1762                 if (cmnd->data_waiting) {
1763                         if (!cmnd_get_check(cmnd)) {
1764                                 spin_unlock_bh(&conn->cmd_list_lock);
1765
1766                                 /* ToDo: this is racy for MC/S */
1767                                 TRACE_MGMT_DBG("Restarting data waiting cmd %p",
1768                                         cmnd);
1769                                 iscsi_fail_waiting_cmnd(cmnd);
1770
1771                                 cmnd_put(cmnd);
1772
1773                                 /*
1774                                  * We are in the read thread, so we may not
1775                                  * worry that after cmnd release conn gets
1776                                  * released as well.
1777                                  */
1778                                 spin_lock_bh(&conn->cmd_list_lock);
1779                                 goto again;
1780                         }
1781                 }
1782         }
1783         spin_unlock_bh(&conn->cmd_list_lock);
1784
1785         return;
1786 }
1787
1788 static void execute_task_management(struct iscsi_cmnd *req)
1789 {
1790         struct iscsi_conn *conn = req->conn;
1791         struct iscsi_session *sess = conn->session;
1792         struct iscsi_task_mgt_hdr *req_hdr =
1793                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1794         int rc, status, function = req_hdr->function & ISCSI_FUNCTION_MASK;
1795         struct scst_rx_mgmt_params params;
1796
1797         TRACE((function == ISCSI_FUNCTION_ABORT_TASK) ? TRACE_MGMT_MINOR : TRACE_MGMT,
1798                 "TM fn %d", function);
1799
1800         TRACE_MGMT_DBG("TM req %p, itt %x, rtt %x, sn %u, con %p", req,
1801                 cmnd_itt(req), req_hdr->rtt, req_hdr->cmd_sn, conn);
1802
1803         iscsi_extracheck_is_rd_thread(conn);
1804
1805         spin_lock(&sess->sn_lock);
1806         sess->tm_active++;
1807         sess->tm_sn = req_hdr->cmd_sn;
1808         if (sess->tm_rsp != NULL) {
1809                 struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
1810
1811                 TRACE(TRACE_MGMT_MINOR, "Dropping delayed TM rsp %p", tm_rsp);
1812
1813                 sess->tm_rsp = NULL;
1814                 sess->tm_active--;
1815
1816                 spin_unlock(&sess->sn_lock);
1817
1818                 sBUG_ON(sess->tm_active < 0);
1819
1820                 rsp_cmnd_release(tm_rsp);
1821         } else
1822                 spin_unlock(&sess->sn_lock);
1823
1824         memset(&params, 0, sizeof(params));
1825         params.atomic = SCST_NON_ATOMIC;
1826         params.tgt_priv = req;
1827
1828         if ((function != ISCSI_FUNCTION_ABORT_TASK) &&
1829             (req_hdr->rtt != ISCSI_RESERVED_TAG)) {
1830                 PRINT_ERROR("Invalid RTT %x (TM fn %x)", req_hdr->rtt,
1831                         function);
1832                 rc = -1;
1833                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1834                 goto reject;
1835         }
1836
1837         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
1838
1839         switch (function) {
1840         case ISCSI_FUNCTION_ABORT_TASK:
1841                 rc = -1;
1842                 status = cmnd_abort(req);
1843                 if (status == 0) {
1844                         params.fn = SCST_ABORT_TASK;
1845                         params.tag = req_hdr->rtt;
1846                         params.tag_set = 1;
1847                         params.lun = (uint8_t *)&req_hdr->lun;
1848                         params.lun_len = sizeof(req_hdr->lun);
1849                         params.lun_set = 1;
1850                         params.cmd_sn = req_hdr->cmd_sn;
1851                         params.cmd_sn_set = 1;
1852                         rc = scst_rx_mgmt_fn(conn->session->scst_sess,
1853                                 &params);
1854                         status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1855                 }
1856                 break;
1857         case ISCSI_FUNCTION_ABORT_TASK_SET:
1858                 task_set_abort(req);
1859                 params.fn = SCST_ABORT_TASK_SET;
1860                 params.lun = (uint8_t *)&req_hdr->lun;
1861                 params.lun_len = sizeof(req_hdr->lun);
1862                 params.lun_set = 1;
1863                 params.cmd_sn = req_hdr->cmd_sn;
1864                 params.cmd_sn_set = 1;
1865                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
1866                         &params);
1867                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1868                 break;
1869         case ISCSI_FUNCTION_CLEAR_TASK_SET:
1870                 task_set_abort(req);
1871                 params.fn = SCST_CLEAR_TASK_SET;
1872                 params.lun = (uint8_t *)&req_hdr->lun;
1873                 params.lun_len = sizeof(req_hdr->lun);
1874                 params.lun_set = 1;
1875                 params.cmd_sn = req_hdr->cmd_sn;
1876                 params.cmd_sn_set = 1;
1877                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
1878                         &params);
1879                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1880                 break;
1881         case ISCSI_FUNCTION_CLEAR_ACA:
1882                 params.fn = SCST_CLEAR_ACA;
1883                 params.lun = (uint8_t *)&req_hdr->lun;
1884                 params.lun_len = sizeof(req_hdr->lun);
1885                 params.lun_set = 1;
1886                 params.cmd_sn = req_hdr->cmd_sn;
1887                 params.cmd_sn_set = 1;
1888                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
1889                         &params);
1890                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1891                 break;
1892         case ISCSI_FUNCTION_TARGET_COLD_RESET:
1893         case ISCSI_FUNCTION_TARGET_WARM_RESET:
1894                 target_abort(req, 1);
1895                 params.fn = SCST_TARGET_RESET;
1896                 params.cmd_sn = req_hdr->cmd_sn;
1897                 params.cmd_sn_set = 1;
1898                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
1899                         &params);
1900                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1901                 break;
1902         case ISCSI_FUNCTION_LOGICAL_UNIT_RESET:
1903                 target_abort(req, 0);
1904                 params.fn = SCST_LUN_RESET;
1905                 params.lun = (uint8_t *)&req_hdr->lun;
1906                 params.lun_len = sizeof(req_hdr->lun);
1907                 params.lun_set = 1;
1908                 params.cmd_sn = req_hdr->cmd_sn;
1909                 params.cmd_sn_set = 1;
1910                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
1911                         &params);
1912                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1913                 break;
1914         case ISCSI_FUNCTION_TASK_REASSIGN:
1915                 rc = -1;
1916                 status = ISCSI_RESPONSE_FUNCTION_UNSUPPORTED;
1917                 break;
1918         default:
1919                 PRINT_ERROR("Unknown TM function %d", function);
1920                 rc = -1;
1921                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1922                 break;
1923         }
1924
1925 reject:
1926         if (rc != 0)
1927                 iscsi_send_task_mgmt_resp(req, status);
1928
1929         return;
1930 }
1931
1932 static void noop_out_exec(struct iscsi_cmnd *req)
1933 {
1934         struct iscsi_cmnd *rsp;
1935         struct iscsi_nop_in_hdr *rsp_hdr;
1936
1937         TRACE_DBG("%p", req);
1938
1939         if (cmnd_itt(req) != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1940                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
1941
1942                 rsp_hdr = (struct iscsi_nop_in_hdr *)&rsp->pdu.bhs;
1943                 rsp_hdr->opcode = ISCSI_OP_NOOP_IN;
1944                 rsp_hdr->flags = ISCSI_FLG_FINAL;
1945                 rsp_hdr->itt = req->pdu.bhs.itt;
1946                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
1947
1948                 if (req->pdu.datasize)
1949                         sBUG_ON(req->sg == NULL);
1950                 else
1951                         sBUG_ON(req->sg != NULL);
1952
1953                 if (req->sg) {
1954                         rsp->sg = req->sg;
1955                         rsp->sg_cnt = req->sg_cnt;
1956                         rsp->bufflen = req->bufflen;
1957                 }
1958
1959                 sBUG_ON(get_pgcnt(req->pdu.datasize, 0) > ISCSI_CONN_IOV_MAX);
1960
1961                 rsp->pdu.datasize = req->pdu.datasize;
1962                 iscsi_cmnd_init_write(rsp,
1963                         ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
1964                 req_cmnd_release(req);
1965         } else
1966                 cmnd_put(req);
1967 }
1968
1969 static void logout_exec(struct iscsi_cmnd *req)
1970 {
1971         struct iscsi_logout_req_hdr *req_hdr;
1972         struct iscsi_cmnd *rsp;
1973         struct iscsi_logout_rsp_hdr *rsp_hdr;
1974
1975         PRINT_INFO("Logout received from initiator %s",
1976                 req->conn->session->initiator_name);
1977         TRACE_DBG("%p", req);
1978
1979         req_hdr = (struct iscsi_logout_req_hdr *)&req->pdu.bhs;
1980         rsp = iscsi_cmnd_create_rsp_cmnd(req);
1981         rsp_hdr = (struct iscsi_logout_rsp_hdr *)&rsp->pdu.bhs;
1982         rsp_hdr->opcode = ISCSI_OP_LOGOUT_RSP;
1983         rsp_hdr->flags = ISCSI_FLG_FINAL;
1984         rsp_hdr->itt = req_hdr->itt;
1985         rsp->should_close_conn = 1;
1986         iscsi_cmnd_init_write(rsp,
1987                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
1988         req_cmnd_release(req);
1989 }
1990
1991 static void iscsi_cmnd_exec(struct iscsi_cmnd *cmnd)
1992 {
1993         TRACE_ENTRY();
1994
1995         TRACE_DBG("%p,%x,%u", cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn);
1996
1997         iscsi_extracheck_is_rd_thread(cmnd->conn);
1998
1999         if (unlikely(cmnd->tm_aborted)) {
2000                 TRACE_MGMT_DBG("cmnd %p (scst_cmd %p) aborted", cmnd,
2001                         cmnd->scst_cmd);
2002                 req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
2003                 goto out;
2004         }
2005
2006         if (unlikely(cmnd->rejected))
2007                 goto out_rejected;
2008
2009         switch (cmnd_opcode(cmnd)) {
2010         case ISCSI_OP_SCSI_CMD:
2011                 if (cmnd->r2t_length != 0) {
2012                         if (!cmnd->is_unsolicited_data) {
2013                                 send_r2t(cmnd);
2014                                 break;
2015                         }
2016                 } else
2017                         iscsi_restart_cmnd(cmnd);
2018                 break;
2019         case ISCSI_OP_NOOP_OUT:
2020                 noop_out_exec(cmnd);
2021                 break;
2022         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2023                 execute_task_management(cmnd);
2024                 break;
2025         case ISCSI_OP_LOGOUT_CMD:
2026                 logout_exec(cmnd);
2027                 break;
2028         default:
2029                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2030                 req_cmnd_release(cmnd);
2031                 break;
2032         }
2033 out:
2034         TRACE_EXIT();
2035         return;
2036
2037 out_rejected:
2038         TRACE_MGMT_DBG("Rejected cmd %p (reason %d)", cmnd,
2039                 cmnd->reject_reason);
2040         switch (cmnd->reject_reason) {
2041         default:
2042                 PRINT_ERROR("Unexpected reject reason %d", cmnd->reject_reason);
2043                 /* go through */
2044         case ISCSI_REJECT_SCSI_CMD:
2045                 req_cmnd_release(cmnd);
2046                 break;
2047         }
2048         goto out;
2049 }
2050
2051 static void __cmnd_send_pdu(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd,
2052         u32 offset, u32 size)
2053 {
2054         TRACE_DBG("%p %u,%u,%u", cmnd, offset, size, cmnd->bufflen);
2055
2056         iscsi_extracheck_is_wr_thread(conn);
2057
2058         sBUG_ON(offset > cmnd->bufflen);
2059         sBUG_ON(offset + size > cmnd->bufflen);
2060
2061         conn->write_offset = offset;
2062         conn->write_size += size;
2063 }
2064
2065 static void cmnd_send_pdu(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
2066 {
2067         u32 size;
2068
2069         if (!cmnd->pdu.datasize)
2070                 return;
2071
2072         size = (cmnd->pdu.datasize + 3) & -4;
2073         sBUG_ON(cmnd->sg == NULL);
2074         sBUG_ON(cmnd->bufflen != size);
2075         __cmnd_send_pdu(conn, cmnd, 0, size);
2076 }
2077
2078 static void set_cork(struct socket *sock, int on)
2079 {
2080         int opt = on;
2081         mm_segment_t oldfs;
2082
2083         oldfs = get_fs();
2084         set_fs(get_ds());
2085         sock->ops->setsockopt(sock, SOL_TCP, TCP_CORK, (void *)&opt, sizeof(opt));
2086         set_fs(oldfs);
2087 }
2088
2089 void cmnd_tx_start(struct iscsi_cmnd *cmnd)
2090 {
2091         struct iscsi_conn *conn = cmnd->conn;
2092
2093         TRACE_DBG("%p:%p:%x", conn, cmnd, cmnd_opcode(cmnd));
2094         iscsi_cmnd_set_length(&cmnd->pdu);
2095
2096         iscsi_extracheck_is_wr_thread(conn);
2097
2098         set_cork(conn->sock, 1);
2099
2100         conn->write_iop = conn->write_iov;
2101         conn->write_iop->iov_base = &cmnd->pdu.bhs;
2102         conn->write_iop->iov_len = sizeof(cmnd->pdu.bhs);
2103         conn->write_iop_used = 1;
2104         conn->write_size = sizeof(cmnd->pdu.bhs);
2105
2106         switch (cmnd_opcode(cmnd)) {
2107         case ISCSI_OP_NOOP_IN:
2108                 cmnd_set_sn(cmnd, 1);
2109                 cmnd_send_pdu(conn, cmnd);
2110                 break;
2111         case ISCSI_OP_SCSI_RSP:
2112                 cmnd_set_sn(cmnd, 1);
2113                 cmnd_send_pdu(conn, cmnd);
2114                 break;
2115         case ISCSI_OP_SCSI_TASK_MGT_RSP:
2116                 cmnd_set_sn(cmnd, 1);
2117                 break;
2118         case ISCSI_OP_TEXT_RSP:
2119                 cmnd_set_sn(cmnd, 1);
2120                 break;
2121         case ISCSI_OP_SCSI_DATA_IN:
2122         {
2123                 struct iscsi_data_in_hdr *rsp = (struct iscsi_data_in_hdr *)&cmnd->pdu.bhs;
2124                 u32 offset = cpu_to_be32(rsp->buffer_offset);
2125
2126                 cmnd_set_sn(cmnd, (rsp->flags & ISCSI_FLG_FINAL) ? 1 : 0);
2127                 __cmnd_send_pdu(conn, cmnd, offset, cmnd->pdu.datasize);
2128                 break;
2129         }
2130         case ISCSI_OP_LOGOUT_RSP:
2131                 cmnd_set_sn(cmnd, 1);
2132                 break;
2133         case ISCSI_OP_R2T:
2134                 cmnd->pdu.bhs.sn = cmnd_set_sn(cmnd, 0);
2135                 break;
2136         case ISCSI_OP_ASYNC_MSG:
2137                 cmnd_set_sn(cmnd, 1);
2138                 break;
2139         case ISCSI_OP_REJECT:
2140                 cmnd_set_sn(cmnd, 1);
2141                 cmnd_send_pdu(conn, cmnd);
2142                 break;
2143         default:
2144                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2145                 break;
2146         }
2147
2148         /* move this? */
2149         conn->write_size = (conn->write_size + 3) & -4;
2150         iscsi_dump_pdu(&cmnd->pdu);
2151 }
2152
2153 void cmnd_tx_end(struct iscsi_cmnd *cmnd)
2154 {
2155         struct iscsi_conn *conn = cmnd->conn;
2156
2157         TRACE_DBG("%p:%x (should_close_conn %d)", cmnd, cmnd_opcode(cmnd),
2158                 cmnd->should_close_conn);
2159
2160         switch (cmnd_opcode(cmnd)) {
2161         case ISCSI_OP_NOOP_IN:
2162         case ISCSI_OP_SCSI_RSP:
2163         case ISCSI_OP_SCSI_TASK_MGT_RSP:
2164         case ISCSI_OP_TEXT_RSP:
2165         case ISCSI_OP_R2T:
2166         case ISCSI_OP_ASYNC_MSG:
2167         case ISCSI_OP_REJECT:
2168         case ISCSI_OP_SCSI_DATA_IN:
2169         case ISCSI_OP_LOGOUT_RSP:
2170                 break;
2171         default:
2172                 PRINT_CRIT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2173                 sBUG();
2174                 break;
2175         }
2176
2177         if (cmnd->should_close_conn) {
2178                 PRINT_INFO("Closing connection at initiator %s request",
2179                         conn->session->initiator_name);
2180                 mark_conn_closed(conn);
2181         }
2182
2183         set_cork(cmnd->conn->sock, 0);
2184 }
2185
2186 /*
2187  * Push the command for execution. This functions reorders the commands.
2188  * Called from the read thread.
2189  */
2190 static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd)
2191 {
2192         struct iscsi_session *session = cmnd->conn->session;
2193         struct list_head *entry;
2194         u32 cmd_sn;
2195
2196         TRACE_DBG("%p:%x %u,%u",
2197                 cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn, session->exp_cmd_sn);
2198
2199         iscsi_extracheck_is_rd_thread(cmnd->conn);
2200
2201         sBUG_ON(cmnd->parent_req != NULL);
2202
2203         if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
2204                 TRACE_DBG("Immediate cmd %p (cmd_sn %u)", cmnd,
2205                         cmnd->pdu.bhs.sn);
2206                 iscsi_cmnd_exec(cmnd);
2207                 goto out;
2208         }
2209
2210         spin_lock(&session->sn_lock);
2211
2212         cmd_sn = cmnd->pdu.bhs.sn;
2213         if (cmd_sn == session->exp_cmd_sn) {
2214                 while (1) {
2215                         session->exp_cmd_sn = ++cmd_sn;
2216
2217                         if (unlikely(session->tm_active > 0)) {
2218                                 if (before(cmd_sn, session->tm_sn)) {
2219                                         struct iscsi_conn *conn = cmnd->conn;
2220
2221                                         spin_unlock(&session->sn_lock);
2222
2223                                         spin_lock_bh(&conn->cmd_list_lock);
2224                                         __cmnd_abort(cmnd);
2225                                         spin_unlock_bh(&conn->cmd_list_lock);
2226
2227                                         spin_lock(&session->sn_lock);
2228                                 }
2229                                 iscsi_check_send_delayed_tm_resp(session);
2230                         }
2231
2232                         spin_unlock(&session->sn_lock);
2233
2234                         iscsi_cmnd_exec(cmnd);
2235
2236                         if (list_empty(&session->pending_list))
2237                                 break;
2238                         cmnd = list_entry(session->pending_list.next, struct iscsi_cmnd,
2239                                                 pending_list_entry);
2240                         if (cmnd->pdu.bhs.sn != cmd_sn)
2241                                 break;
2242
2243                         list_del(&cmnd->pending_list_entry);
2244                         cmnd->pending = 0;
2245
2246                         TRACE_DBG("Processing pending cmd %p (cmd_sn %u)",
2247                                 cmnd, cmd_sn);
2248
2249                         spin_lock(&session->sn_lock);
2250                 }
2251         } else {
2252                 int drop = 0;
2253
2254                 TRACE_DBG("Pending cmd %p (cmd_sn %u, exp_cmd_sn %u)",
2255                         cmnd, cmd_sn, session->exp_cmd_sn);
2256
2257                 /*
2258                  * iSCSI RFC 3720: "The target MUST silently ignore any
2259                  * non-immediate command outside of [from ExpCmdSN to MaxCmdSN
2260                  * inclusive] range". But we won't honor the MaxCmdSN
2261                  * requirement, because, since we adjust MaxCmdSN from the
2262                  * separate write thread, rarery it is possible that initiator
2263                  * can legally send command with CmdSN>MaxSN. But it won't
2264                  * hurt anything, in the worst case it will lead to
2265                  * additional QUEUE FULL status.
2266                  */
2267
2268                 if (unlikely(before(cmd_sn, session->exp_cmd_sn))) {
2269                         PRINT_ERROR("Unexpected cmd_sn (%u,%u)", cmd_sn,
2270                                 session->exp_cmd_sn);
2271                         drop = 1;
2272                 }
2273
2274 #if 0
2275                 if (unlikely(after(cmd_sn, session->exp_cmd_sn +
2276                                         iscsi_get_allowed_cmds(session)))) {
2277                         TRACE_MGMT_DBG("Too large cmd_sn %u (exp_cmd_sn %u, "
2278                                 "max_sn %u)", cmd_sn, session->exp_cmd_sn,
2279                                 iscsi_get_allowed_cmds(session));
2280                 }
2281 #endif
2282
2283                 spin_unlock(&session->sn_lock);
2284
2285                 if (unlikely(drop)) {
2286                         req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
2287                         goto out;
2288                 }
2289
2290                 if (unlikely(cmnd->tm_aborted)) {
2291                         struct iscsi_cmnd *tm_clone;
2292
2293                         TRACE_MGMT_DBG("Pending aborted cmnd %p, creating TM "
2294                                 "clone (scst cmd %p, state %d)", cmnd,
2295                                 cmnd->scst_cmd, cmnd->scst_state);
2296
2297                         tm_clone = cmnd_alloc(cmnd->conn, NULL);
2298                         if (tm_clone != NULL) {
2299                                 tm_clone->tm_aborted = 1;
2300                                 tm_clone->pdu = cmnd->pdu;
2301
2302                                 TRACE_MGMT_DBG("TM clone %p created", tm_clone);
2303
2304                                 iscsi_cmnd_exec(cmnd);
2305                                 cmnd = tm_clone;
2306                         } else
2307                                 PRINT_ERROR("%s", "Unable to create TM clone");
2308                 }
2309
2310                 list_for_each(entry, &session->pending_list) {
2311                         struct iscsi_cmnd *tmp = list_entry(entry, struct iscsi_cmnd,
2312                                                         pending_list_entry);
2313                         if (before(cmd_sn, tmp->pdu.bhs.sn))
2314                                 break;
2315                 }
2316
2317                 list_add_tail(&cmnd->pending_list_entry, entry);
2318                 cmnd->pending = 1;
2319         }
2320 out:
2321         return;
2322 }
2323
2324 static int check_segment_length(struct iscsi_cmnd *cmnd)
2325 {
2326         struct iscsi_conn *conn = cmnd->conn;
2327         struct iscsi_session *session = conn->session;
2328
2329         if (unlikely(cmnd->pdu.datasize > session->sess_param.max_recv_data_length)) {
2330                 PRINT_ERROR("Initiator %s violated negotiated parameters: "
2331                         "data too long (ITT %x, datasize %u, "
2332                         "max_recv_data_length %u", session->initiator_name,
2333                         cmnd_itt(cmnd), cmnd->pdu.datasize,
2334                         session->sess_param.max_recv_data_length);
2335                 mark_conn_closed(conn);
2336                 return -EINVAL;
2337         }
2338         return 0;
2339 }
2340
2341 int cmnd_rx_start(struct iscsi_cmnd *cmnd)
2342 {
2343         struct iscsi_conn *conn = cmnd->conn;
2344         int res, rc;
2345
2346         iscsi_dump_pdu(&cmnd->pdu);
2347
2348         res = check_segment_length(cmnd);
2349         if (res != 0)
2350                 goto out;
2351
2352         switch (cmnd_opcode(cmnd)) {
2353         case ISCSI_OP_NOOP_OUT:
2354                 rc = noop_out_start(cmnd);
2355                 break;
2356         case ISCSI_OP_SCSI_CMD:
2357                 rc = cmnd_insert_hash(cmnd);
2358                 if (likely(rc == 0)) {
2359                         res = scsi_cmnd_start(cmnd);
2360                         if (unlikely(res != 0))
2361                                 goto out;
2362                 }
2363                 break;
2364         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2365                 rc = cmnd_insert_hash(cmnd);
2366                 break;
2367         case ISCSI_OP_SCSI_DATA_OUT:
2368                 res = data_out_start(conn, cmnd);
2369                 rc = 0; /* to avoid compiler warning */
2370                 if (unlikely(res != 0))
2371                         goto out;
2372                 break;
2373         case ISCSI_OP_LOGOUT_CMD:
2374                 rc = cmnd_insert_hash(cmnd);
2375                 break;
2376         case ISCSI_OP_TEXT_CMD:
2377         case ISCSI_OP_SNACK_CMD:
2378                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2379                 break;
2380         default:
2381                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2382                 break;
2383         }
2384
2385         if (unlikely(rc < 0)) {
2386                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
2387                 PRINT_ERROR("Error %d (iSCSI opcode %x, ITT %x, op %x)", rc,
2388                         cmnd_opcode(cmnd), cmnd_itt(cmnd),
2389                         (cmnd_opcode(cmnd) == ISCSI_OP_SCSI_CMD ?
2390                                 hdr->scb[0] : -1));
2391                 iscsi_cmnd_reject(cmnd, -rc);
2392         }
2393
2394 out:
2395         TRACE_EXIT_RES(res);
2396         return res;
2397 }
2398
2399 void cmnd_rx_end(struct iscsi_cmnd *cmnd)
2400 {
2401         TRACE_ENTRY();
2402
2403         TRACE_DBG("%p:%x", cmnd, cmnd_opcode(cmnd));
2404
2405         if (unlikely(cmnd->rejected))
2406                 goto out_rejected;
2407
2408 cont:
2409         switch (cmnd_opcode(cmnd)) {
2410         case ISCSI_OP_SCSI_CMD:
2411         case ISCSI_OP_NOOP_OUT:
2412         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2413         case ISCSI_OP_LOGOUT_CMD:
2414                 iscsi_session_push_cmnd(cmnd);
2415                 break;
2416         case ISCSI_OP_SCSI_DATA_OUT:
2417                 data_out_end(cmnd);
2418                 break;
2419         default:
2420                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2421                 req_cmnd_release(cmnd);
2422                 break;
2423         }
2424
2425 out:
2426         TRACE_EXIT();
2427         return;
2428
2429 out_rejected:
2430         switch (cmnd->reject_reason) {
2431         default:
2432                 PRINT_ERROR("Unexpected reject reason %d", cmnd->reject_reason);
2433                 /* go through */
2434         case ISCSI_REJECT_CMD:
2435         case ISCSI_REJECT_DATA:
2436                 req_cmnd_release(cmnd);
2437                 break;
2438         case ISCSI_REJECT_SCSI_CMD:
2439                 goto cont;
2440         }
2441         goto out;
2442 }
2443
2444 #ifndef NET_PAGE_CALLBACKS_DEFINED
2445 static int iscsi_alloc_data_buf(struct scst_cmd *cmd)
2446 {
2447         if (scst_cmd_get_data_direction(cmd) == SCST_DATA_READ) {
2448                 /*
2449                  * sock->ops->sendpage() is async zero copy operation,
2450                  * so we must be sure not to free and reuse
2451                  * the command's buffer before the sending was completed
2452                  * by the network layers. It is possible only if we
2453                  * don't use SGV cache.
2454                  */
2455                 scst_cmd_set_no_sgv(cmd);
2456         }
2457         return 1;
2458 }
2459 #endif
2460
2461 static inline void iscsi_set_state_wake_up(struct iscsi_cmnd *req,
2462         int new_state)
2463 {
2464         /*
2465          * We use wait_event() to wait for the state change, but it checks its
2466          * condition without any protection, so without cmnd_get() it is
2467          * possible that req will die "immediately" after the state assignment
2468          * and wake_up() will operate on dead data.
2469          */
2470         cmnd_get_ordered(req);
2471         req->scst_state = new_state;
2472         wake_up(&req->scst_waitQ);
2473         cmnd_put(req);
2474         return;
2475 }
2476
2477 static void iscsi_preprocessing_done(struct scst_cmd *scst_cmd)
2478 {
2479         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
2480                                 scst_cmd_get_tgt_priv(scst_cmd);
2481
2482         TRACE_DBG("req %p", req);
2483
2484         EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_RX_CMD);
2485
2486         iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_AFTER_PREPROC);
2487         return;
2488 }
2489
2490 /*
2491  * No locks.
2492  *
2493  * IMPORTANT! Connection conn must be protected by additional conn_get()
2494  * upon entrance in this function, because otherwise it could be destroyed
2495  * inside as a result of iscsi_send(), which releases sent commands.
2496  */
2497 static void iscsi_try_local_processing(struct iscsi_conn *conn,
2498         bool single_only)
2499 {
2500         int local;
2501
2502         TRACE_ENTRY();
2503
2504         spin_lock_bh(&iscsi_wr_lock);
2505         switch (conn->wr_state) {
2506         case ISCSI_CONN_WR_STATE_IN_LIST:
2507                 list_del(&conn->wr_list_entry);
2508                 /* go through */
2509         case ISCSI_CONN_WR_STATE_IDLE:
2510 #ifdef EXTRACHECKS
2511                 conn->wr_task = current;
2512 #endif
2513                 conn->wr_state = ISCSI_CONN_WR_STATE_PROCESSING;
2514                 conn->wr_space_ready = 0;
2515                 local = 1;
2516                 break;
2517         default:
2518                 local = 0;
2519                 break;
2520         }
2521         spin_unlock_bh(&iscsi_wr_lock);
2522
2523         if (local) {
2524                 int rc = 1;
2525                 while (test_write_ready(conn)) {
2526                         rc = iscsi_send(conn);
2527                         if ((rc <= 0) || single_only)
2528                                 break;
2529                 }
2530
2531                 spin_lock_bh(&iscsi_wr_lock);
2532 #ifdef EXTRACHECKS
2533                 conn->wr_task = NULL;
2534 #endif
2535                 if ((rc <= 0) || test_write_ready(conn)) {
2536                         list_add_tail(&conn->wr_list_entry, &iscsi_wr_list);
2537                         conn->wr_state = ISCSI_CONN_WR_STATE_IN_LIST;
2538                         wake_up(&iscsi_wr_waitQ);
2539                 } else
2540                         conn->wr_state = ISCSI_CONN_WR_STATE_IDLE;
2541                 spin_unlock_bh(&iscsi_wr_lock);
2542         }
2543
2544         TRACE_EXIT();
2545         return;
2546 }
2547
2548 static int iscsi_xmit_response(struct scst_cmd *scst_cmd)
2549 {
2550         int is_send_status = scst_cmd_get_is_send_status(scst_cmd);
2551         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
2552                                         scst_cmd_get_tgt_priv(scst_cmd);
2553         struct iscsi_conn *conn = req->conn;
2554         int status = scst_cmd_get_status(scst_cmd);
2555         u8 *sense = scst_cmd_get_sense_buffer(scst_cmd);
2556         int sense_len = scst_cmd_get_sense_buffer_len(scst_cmd);
2557         int old_state = req->scst_state;
2558 #if 0 /* temp. ToDo */
2559         bool single_only = !scst_get_long_xmit(scst_cmd);
2560 #else
2561         bool single_only = 0;
2562 #endif
2563
2564         if (scst_cmd_atomic(scst_cmd))
2565                 return SCST_TGT_RES_NEED_THREAD_CTX;
2566
2567         scst_cmd_set_tgt_priv(scst_cmd, NULL);
2568
2569         req->tm_aborted |= scst_cmd_aborted(scst_cmd) ? 1 : 0;
2570         if (unlikely(req->tm_aborted)) {
2571                 TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
2572                         req->scst_cmd);
2573
2574                 scst_set_delivery_status(req->scst_cmd,
2575                         SCST_CMD_DELIVERY_ABORTED);
2576
2577                 if (old_state == ISCSI_CMD_STATE_RESTARTED) {
2578                         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
2579                         req_cmnd_release_force(req, ISCSI_FORCE_RELEASE_WRITE);
2580                 } else
2581                         iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_PROCESSED);
2582
2583                 goto out;
2584         }
2585
2586         if (unlikely(old_state != ISCSI_CMD_STATE_RESTARTED)) {
2587                 TRACE_DBG("req %p on %d state", req, old_state);
2588
2589                 create_status_rsp(req, status, sense, sense_len);
2590
2591                 switch (old_state) {
2592                 case ISCSI_CMD_STATE_RX_CMD:
2593                 case ISCSI_CMD_STATE_AFTER_PREPROC:
2594                         break;
2595                 default:
2596                         sBUG();
2597                 }
2598
2599                 iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_PROCESSED);
2600                 goto out;
2601         }
2602
2603         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
2604
2605         req->bufflen = scst_cmd_get_resp_data_len(scst_cmd);
2606         req->sg = scst_cmd_get_sg(scst_cmd);
2607         req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
2608
2609         TRACE_DBG("req %p, is_send_status=%x, req->bufflen=%d, req->sg=%p, "
2610                 "req->sg_cnt %d", req, is_send_status, req->bufflen, req->sg,
2611                 req->sg_cnt);
2612
2613         if (unlikely((req->bufflen != 0) && !is_send_status)) {
2614                 PRINT_CRIT_ERROR("%s", "Sending DATA without STATUS is unsupported");
2615                 scst_set_cmd_error(scst_cmd,
2616                         SCST_LOAD_SENSE(scst_sense_hardw_error));
2617                 sBUG();
2618         }
2619
2620         if (req->bufflen != 0) {
2621                 /*
2622                  * Check above makes sure that is_send_status is set,
2623                  * so status is valid here, but in future that could change.
2624                  * ToDo
2625                  */
2626                 if (status != SAM_STAT_CHECK_CONDITION) {
2627                         send_data_rsp(req, status, is_send_status);
2628                 } else {
2629                         struct iscsi_cmnd *rsp;
2630                         struct iscsi_scsi_rsp_hdr *rsp_hdr;
2631                         int resid;
2632                         send_data_rsp(req, 0, 0);
2633                         if (is_send_status) {
2634                                 rsp = create_status_rsp(req, status, sense,
2635                                         sense_len);
2636                                 rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
2637                                 resid = cmnd_read_size(req) - req->bufflen;
2638                                 if (resid > 0) {
2639                                         rsp_hdr->flags |=
2640                                                 ISCSI_FLG_RESIDUAL_UNDERFLOW;
2641                                         rsp_hdr->residual_count = cpu_to_be32(resid);
2642                                 } else if (resid < 0) {
2643                                         rsp_hdr->flags |=
2644                                                 ISCSI_FLG_RESIDUAL_OVERFLOW;
2645                                         rsp_hdr->residual_count = cpu_to_be32(-resid);
2646                                 }
2647                                 iscsi_cmnd_init_write(rsp,
2648                                         ISCSI_INIT_WRITE_REMOVE_HASH);
2649                         }
2650                 }
2651         } else if (is_send_status) {
2652                 struct iscsi_cmnd *rsp;
2653                 struct iscsi_scsi_rsp_hdr *rsp_hdr;
2654                 u32 resid;
2655                 rsp = create_status_rsp(req, status, sense, sense_len);
2656                 rsp_hdr = (struct iscsi_scsi_rsp_hdr *) &rsp->pdu.bhs;
2657                 resid = cmnd_read_size(req);
2658                 if (resid != 0) {
2659                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
2660                         rsp_hdr->residual_count = cpu_to_be32(resid);
2661                 }
2662                 iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH);
2663         }
2664 #ifdef EXTRACHECKS
2665         else
2666                 sBUG();
2667 #endif
2668
2669         conn_get_ordered(conn);
2670         req_cmnd_release(req);
2671         iscsi_try_local_processing(conn, single_only);
2672         conn_put(conn);
2673
2674 out:
2675         return SCST_TGT_RES_SUCCESS;
2676 }
2677
2678 /* Called under sn_lock */
2679 static bool iscsi_is_delay_tm_resp(struct iscsi_cmnd *rsp)
2680 {
2681         bool res = 0;
2682         struct iscsi_task_mgt_hdr *req_hdr =
2683                 (struct iscsi_task_mgt_hdr *)&rsp->parent_req->pdu.bhs;
2684         int function = req_hdr->function & ISCSI_FUNCTION_MASK;
2685         struct iscsi_session *sess = rsp->conn->session;
2686
2687         TRACE_ENTRY();
2688
2689         /* This should be checked for immediate TM commands as well */
2690
2691         switch (function) {
2692         default:
2693                 if (before(sess->exp_cmd_sn, req_hdr->cmd_sn))
2694                         res = 1;
2695                 break;
2696         }
2697
2698         TRACE_EXIT_RES(res);
2699         return res;
2700 }
2701
2702 /* Called under sn_lock, but might drop it inside, then reaquire */
2703 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess)
2704 {
2705         struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
2706
2707         TRACE_ENTRY();
2708
2709         if (tm_rsp == NULL)
2710                 goto out;
2711
2712         if (iscsi_is_delay_tm_resp(tm_rsp))
2713                 goto out;
2714
2715         TRACE(TRACE_MGMT_MINOR, "Sending delayed rsp %p", tm_rsp);
2716
2717         sess->tm_rsp = NULL;
2718         sess->tm_active--;
2719
2720         spin_unlock(&sess->sn_lock);
2721
2722         sBUG_ON(sess->tm_active < 0);
2723
2724         iscsi_cmnd_init_write(tm_rsp,
2725                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2726
2727         spin_lock(&sess->sn_lock);
2728
2729 out:
2730         TRACE_EXIT();
2731         return;
2732 }
2733
2734 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status)
2735 {
2736         struct iscsi_cmnd *rsp;
2737         struct iscsi_task_mgt_hdr *req_hdr =
2738                                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2739         struct iscsi_task_rsp_hdr *rsp_hdr;
2740         struct iscsi_session *sess = req->conn->session;
2741         int fn = req_hdr->function & ISCSI_FUNCTION_MASK;
2742
2743         TRACE_ENTRY();
2744
2745         TRACE_MGMT_DBG("TM req %p finished", req);
2746         TRACE((req_hdr->function == ISCSI_FUNCTION_ABORT_TASK) ?
2747                          TRACE_MGMT_MINOR : TRACE_MGMT,
2748                 "TM fn %d finished, status %d", fn, status);
2749
2750         rsp = iscsi_cmnd_create_rsp_cmnd(req);
2751         rsp_hdr = (struct iscsi_task_rsp_hdr *)&rsp->pdu.bhs;
2752
2753         rsp_hdr->opcode = ISCSI_OP_SCSI_TASK_MGT_RSP;
2754         rsp_hdr->flags = ISCSI_FLG_FINAL;
2755         rsp_hdr->itt = req_hdr->itt;
2756         rsp_hdr->response = status;
2757
2758         if (fn == ISCSI_FUNCTION_TARGET_COLD_RESET)
2759                 rsp->should_close_conn = 1;
2760
2761         sBUG_ON(sess->tm_rsp != NULL);
2762
2763         spin_lock(&sess->sn_lock);
2764         if (iscsi_is_delay_tm_resp(rsp)) {
2765                 TRACE(TRACE_MGMT_MINOR, "Delaying TM fn %x response %p "
2766                         "(req %p), because not all affected commands received "
2767                         "(TM cmd sn %u, exp sn %u)",
2768                         req_hdr->function & ISCSI_FUNCTION_MASK, rsp, req,
2769                         req_hdr->cmd_sn, sess->exp_cmd_sn);
2770                 sess->tm_rsp = rsp;
2771                 spin_unlock(&sess->sn_lock);
2772                 goto out_release;
2773         }
2774         sess->tm_active--;
2775         spin_unlock(&sess->sn_lock);
2776
2777         sBUG_ON(sess->tm_active < 0);
2778
2779         iscsi_cmnd_init_write(rsp,
2780                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2781
2782 out_release:
2783         req_cmnd_release(req);
2784
2785         TRACE_EXIT();
2786         return;
2787 }
2788
2789 static inline int iscsi_get_mgmt_response(int status)
2790 {
2791         switch (status) {
2792         case SCST_MGMT_STATUS_SUCCESS:
2793                 return ISCSI_RESPONSE_FUNCTION_COMPLETE;
2794
2795         case SCST_MGMT_STATUS_TASK_NOT_EXIST:
2796                 return ISCSI_RESPONSE_UNKNOWN_TASK;
2797
2798         case SCST_MGMT_STATUS_LUN_NOT_EXIST:
2799                 return ISCSI_RESPONSE_UNKNOWN_LUN;
2800
2801         case SCST_MGMT_STATUS_FN_NOT_SUPPORTED:
2802                 return ISCSI_RESPONSE_FUNCTION_UNSUPPORTED;
2803
2804         case SCST_MGMT_STATUS_REJECTED:
2805         case SCST_MGMT_STATUS_FAILED:
2806         default:
2807                 return ISCSI_RESPONSE_FUNCTION_REJECTED;
2808         }
2809 }
2810
2811 static void iscsi_task_mgmt_fn_done(struct scst_mgmt_cmd *scst_mcmd)
2812 {
2813         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
2814                                 scst_mgmt_cmd_get_tgt_priv(scst_mcmd);
2815         int status = iscsi_get_mgmt_response(scst_mgmt_cmd_get_status(scst_mcmd));
2816
2817         TRACE_MGMT_DBG("req %p, scst_mcmd %p, fn %d, scst status %d",
2818                 req, scst_mcmd, scst_mgmt_cmd_get_fn(scst_mcmd),
2819                 scst_mgmt_cmd_get_status(scst_mcmd));
2820
2821         iscsi_send_task_mgmt_resp(req, status);
2822
2823         scst_mgmt_cmd_set_tgt_priv(scst_mcmd, NULL);
2824
2825         return;
2826 }
2827
2828 static int iscsi_target_detect(struct scst_tgt_template *templ)
2829 {
2830         /* Nothing to do */
2831         return 0;
2832 }
2833
2834 static int iscsi_target_release(struct scst_tgt *scst_tgt)
2835 {
2836         /* Nothing to do */
2837         return 0;
2838 }
2839
2840 struct scst_tgt_template iscsi_template = {
2841         .name = "iscsi",
2842         .sg_tablesize = ISCSI_CONN_IOV_MAX,
2843         .threads_num = 0,
2844         .no_clustering = 1,
2845         .xmit_response_atomic = 0,
2846         .detect = iscsi_target_detect,
2847         .release = iscsi_target_release,
2848         .xmit_response = iscsi_xmit_response,
2849 #ifndef NET_PAGE_CALLBACKS_DEFINED
2850         .alloc_data_buf = iscsi_alloc_data_buf,
2851 #endif
2852         .preprocessing_done = iscsi_preprocessing_done,
2853         .pre_exec = iscsi_pre_exec,
2854         .task_mgmt_fn_done = iscsi_task_mgmt_fn_done,
2855 };
2856
2857 static __init int iscsi_run_threads(int count, char *name, int (*fn)(void *))
2858 {
2859         int res = 0;
2860         int i;
2861         struct iscsi_thread_t *thr;
2862
2863         for (i = 0; i < count; i++) {
2864                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
2865                 if (!thr) {
2866                         res = -ENOMEM;
2867                         PRINT_ERROR("Failed to allocate thr %d", res);
2868                         goto out;
2869                 }
2870                 thr->thr = kthread_run(fn, NULL, "%s%d", name, i);
2871                 if (IS_ERR(thr->thr)) {
2872                         res = PTR_ERR(thr->thr);
2873                         PRINT_ERROR("kthread_create() failed: %d", res);
2874                         kfree(thr);
2875                         goto out;
2876                 }
2877                 list_add(&thr->threads_list_entry, &iscsi_threads_list);
2878         }
2879
2880 out:
2881         return res;
2882 }
2883
2884 static void iscsi_stop_threads(void)
2885 {
2886         struct iscsi_thread_t *t, *tmp;
2887
2888         list_for_each_entry_safe(t, tmp, &iscsi_threads_list,
2889                                 threads_list_entry) {
2890                 int rc = kthread_stop(t->thr);
2891                 if (rc < 0)
2892                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
2893                 list_del(&t->threads_list_entry);
2894                 kfree(t);
2895         }
2896 }
2897
2898 static int __init iscsi_init(void)
2899 {
2900         int err;
2901         int num;
2902
2903         PRINT_INFO("iSCSI SCST Target - version %s", ISCSI_VERSION_STRING);
2904
2905 #ifdef NET_PAGE_CALLBACKS_DEFINED
2906         err = net_set_get_put_page_callbacks(iscsi_get_page_callback,
2907                         iscsi_put_page_callback);
2908         if (err != 0) {
2909                 PRINT_INFO("Unable to set page callbackes: %d", err);
2910                 goto out;
2911         }
2912 #else
2913         PRINT_INFO("%s", "Patch put_page_callback-<kernel-version>.patch "
2914                 "not applied on your kernel. Running in the performance "
2915                 "degraded mode. Refer README file for details");
2916 #endif
2917
2918         ctr_major = register_chrdev(0, ctr_name, &ctr_fops);
2919         if (ctr_major < 0) {
2920                 PRINT_ERROR("failed to register the control device %d", ctr_major);
2921                 err = ctr_major;
2922                 goto out_callb;
2923         }
2924
2925         err = event_init();
2926         if (err < 0)
2927                 goto out_reg;
2928
2929         iscsi_cmnd_cache = KMEM_CACHE(iscsi_cmnd, SCST_SLAB_FLAGS);
2930         if (!iscsi_cmnd_cache) {
2931                 err = -ENOMEM;
2932                 goto out_event;
2933         }
2934
2935         err = scst_register_target_template(&iscsi_template);
2936         if (err < 0)
2937                 goto out_kmem;
2938
2939         iscsi_template_registered = 1;
2940
2941         err = iscsi_procfs_init();
2942         if (err < 0)
2943                 goto out_reg_tmpl;
2944
2945         num = max(num_online_cpus(), 2);
2946
2947         err = iscsi_run_threads(num, "iscsird", istrd);
2948         if (err != 0)
2949                 goto out_thr;
2950
2951         err = iscsi_run_threads(num, "iscsiwr", istwr);
2952         if (err != 0)
2953                 goto out_thr;
2954
2955 out:
2956         return err;
2957
2958 out_thr:
2959         iscsi_procfs_exit();
2960         iscsi_stop_threads();
2961
2962 out_reg_tmpl:
2963         scst_unregister_target_template(&iscsi_template);
2964
2965 out_kmem:
2966         kmem_cache_destroy(iscsi_cmnd_cache);
2967
2968 out_event:
2969         event_exit();
2970
2971 out_reg:
2972         unregister_chrdev(ctr_major, ctr_name);
2973
2974 out_callb:
2975 #ifdef NET_PAGE_CALLBACKS_DEFINED
2976         net_set_get_put_page_callbacks(NULL, NULL);
2977 #endif
2978         goto out;
2979 }
2980
2981 static void __exit iscsi_exit(void)
2982 {
2983         iscsi_stop_threads();
2984
2985         unregister_chrdev(ctr_major, ctr_name);
2986
2987         iscsi_procfs_exit();
2988         event_exit();
2989
2990         kmem_cache_destroy(iscsi_cmnd_cache);
2991
2992         scst_unregister_target_template(&iscsi_template);
2993
2994 #ifdef NET_PAGE_CALLBACKS_DEFINED
2995         net_set_get_put_page_callbacks(NULL, NULL);
2996 #endif
2997 }
2998
2999 module_init(iscsi_init);
3000 module_exit(iscsi_exit);
3001
3002 MODULE_LICENSE("GPL");