0e2e5a706c05f14295cdd6ce06d639d07b7b2ab2
[mirror/scst/.git] / iscsi-scst / kernel / iscsi.c
1 /*
2  *  Copyright (C) 2002-2003 Ardis Technolgies <roman@ardistech.com>
3  *  Copyright (C) 2007 Vladislav Bolkhovitin
4  *  Copyright (C) 2007 CMS Distribution Limited
5  * 
6  *  This program is free software; you can redistribute it and/or
7  *  modify it under the terms of the GNU General Public License
8  *  as published by the Free Software Foundation, version 2
9  *  of the License.
10  * 
11  *  This program is distributed in the hope that it will be useful,
12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  *  GNU General Public License for more details.
15  */
16
17 #include <linux/module.h>
18 #include <linux/hash.h>
19 #include <linux/kthread.h>
20 #include <net/tcp.h>
21 #include <scsi/scsi.h>
22
23 #include "iscsi.h"
24 #include "digest.h"
25
26 #ifndef NET_PAGE_CALLBACKS_DEFINED
27 #warning Patch put_page_callback-<kernel-version>.patch not applied on your \
28         kernel. ISCSI-SCST will run in the performance degraded mode. Refer \
29         README file for details.
30 #endif
31
32 #define ISCSI_INIT_WRITE_WAKE           0x1
33 #define ISCSI_INIT_WRITE_REMOVE_HASH    0x2
34
35 static int ctr_major;
36 static char ctr_name[] = "iscsi-scst-ctl";
37 static int iscsi_template_registered;
38
39 #if defined(DEBUG) || defined(TRACING)
40 unsigned long iscsi_trace_flag = ISCSI_DEFAULT_LOG_FLAGS;
41 #endif
42
43 static struct kmem_cache *iscsi_cmnd_cache;
44
45 spinlock_t iscsi_rd_lock = SPIN_LOCK_UNLOCKED;
46 LIST_HEAD(iscsi_rd_list);
47 DECLARE_WAIT_QUEUE_HEAD(iscsi_rd_waitQ);
48
49 spinlock_t iscsi_wr_lock = SPIN_LOCK_UNLOCKED;
50 LIST_HEAD(iscsi_wr_list);
51 DECLARE_WAIT_QUEUE_HEAD(iscsi_wr_waitQ);
52
53 static char dummy_data[1024];
54
55 struct iscsi_thread_t {
56         struct task_struct *thr;
57         struct list_head threads_list_entry;
58 };
59
60 static LIST_HEAD(iscsi_threads_list);
61
62 static void cmnd_remove_hash(struct iscsi_cmnd *cmnd);
63 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status);
64 static void cmnd_prepare_skip_pdu(struct iscsi_cmnd *cmnd);
65 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess);
66 static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd);
67
68 static inline u32 cmnd_write_size(struct iscsi_cmnd *cmnd)
69 {
70         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
71
72         if (hdr->flags & ISCSI_CMD_WRITE)
73                 return be32_to_cpu(hdr->data_length);
74         return 0;
75 }
76
77 static inline u32 cmnd_read_size(struct iscsi_cmnd *cmnd)
78 {
79         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
80
81         if (hdr->flags & ISCSI_CMD_READ) {
82                 struct iscsi_rlength_ahdr *ahdr =
83                         (struct iscsi_rlength_ahdr *)cmnd->pdu.ahs;
84
85                 if (!(hdr->flags & ISCSI_CMD_WRITE))
86                         return be32_to_cpu(hdr->data_length);
87                 if (ahdr && ahdr->ahstype == ISCSI_AHSTYPE_RLENGTH)
88                         return be32_to_cpu(ahdr->read_length);
89         }
90         return 0;
91 }
92
93 static inline void iscsi_restart_cmnd(struct iscsi_cmnd *cmnd)
94 {
95         EXTRACHECKS_BUG_ON(cmnd->data_waiting);
96
97         cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
98         scst_restart_cmd(cmnd->scst_cmd, SCST_PREPROCESS_STATUS_SUCCESS,
99                 SCST_CONTEXT_THREAD);
100 }
101
102 static inline void iscsi_restart_waiting_cmnd(struct iscsi_cmnd *cmnd)
103 {
104         /*
105          * There is no race with conn_abort(), since all functions
106          * called from single read thread
107          */
108         iscsi_extracheck_is_rd_thread(cmnd->conn);
109         cmnd->data_waiting = 0;
110
111         iscsi_restart_cmnd(cmnd);
112 }
113
114 static inline void iscsi_fail_waiting_cmnd(struct iscsi_cmnd *cmnd)
115 {
116         TRACE_DBG("Failing data waiting cmd %p", cmnd);
117
118         /*
119          * There is no race with conn_abort(), since all functions
120          * called from single read thread
121          */
122         iscsi_extracheck_is_rd_thread(cmnd->conn);
123         cmnd->data_waiting = 0;
124
125         req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
126 }
127
128 struct iscsi_cmnd *cmnd_alloc(struct iscsi_conn *conn, struct iscsi_cmnd *parent)
129 {
130         struct iscsi_cmnd *cmnd;
131
132         /* ToDo: __GFP_NOFAIL?? */
133 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,17)
134         cmnd = kmem_cache_alloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
135         memset(cmnd, 0, sizeof(*cmnd));
136 #else
137         cmnd = kmem_cache_zalloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
138 #endif
139
140         atomic_set(&cmnd->ref_cnt, 1);
141         cmnd->scst_state = ISCSI_CMD_STATE_NEW;
142         cmnd->conn = conn;
143         cmnd->parent_req = parent;
144         init_waitqueue_head(&cmnd->scst_waitQ);
145
146         if (parent == NULL) {
147                 conn_get(conn);
148
149 #ifdef NET_PAGE_CALLBACKS_DEFINED
150                 atomic_set(&cmnd->net_ref_cnt, 0);
151 #endif          
152                 cmnd->target = conn->target;
153                 spin_lock_init(&cmnd->rsp_cmd_lock);
154                 INIT_LIST_HEAD(&cmnd->rsp_cmd_list);
155                 INIT_LIST_HEAD(&cmnd->rx_ddigest_cmd_list);
156
157                 spin_lock_bh(&conn->cmd_list_lock);
158                 list_add_tail(&cmnd->cmd_list_entry, &conn->cmd_list);
159                 spin_unlock_bh(&conn->cmd_list_lock);
160         }
161
162         TRACE_DBG("conn %p, parent %p, cmnd %p", conn, parent, cmnd);
163         return cmnd;
164 }
165
166 /* Frees a command. Also frees the additional header. */
167 void cmnd_free(struct iscsi_cmnd *cmnd)
168 {
169         TRACE_DBG("%p", cmnd);
170
171         if (unlikely(cmnd->tmfabort)) {
172                 TRACE_MGMT_DBG("Free aborted cmd %p (scst cmd %p, state %d, "
173                         "parent_req %p)", cmnd, cmnd->scst_cmd, cmnd->scst_state,
174                         cmnd->parent_req);
175         }
176
177         /* Catch users from cmd_list or rsp_cmd_list */
178         EXTRACHECKS_BUG_ON(atomic_read(&cmnd->ref_cnt) != 0);
179
180         kfree(cmnd->pdu.ahs);
181
182         if (unlikely(cmnd->on_write_list)) {
183                 struct iscsi_scsi_cmd_hdr *req = cmnd_hdr(cmnd);
184
185                 PRINT_ERROR("cmnd %p still on some list?, %x, %x, %x, %x, %x, %x, %x",
186                         cmnd, req->opcode, req->scb[0], req->flags, req->itt,
187                         be32_to_cpu(req->data_length),
188                         req->cmd_sn, be32_to_cpu(cmnd->pdu.datasize));
189
190                 if (unlikely(cmnd->parent_req)) {
191                         struct iscsi_scsi_cmd_hdr *req =
192                                         cmnd_hdr(cmnd->parent_req);
193                         PRINT_ERROR("%p %x %u", req, req->opcode, req->scb[0]);
194                 }
195                 sBUG();
196         }
197
198         kmem_cache_free(iscsi_cmnd_cache, cmnd);
199         return;
200 }
201
202 void cmnd_done(struct iscsi_cmnd *cmnd)
203 {
204         TRACE_DBG("%p", cmnd);
205
206         if (unlikely(cmnd->tmfabort)) {
207                 TRACE_MGMT_DBG("Done aborted cmd %p (scst cmd %p, state %d, "
208                         "parent_req %p)", cmnd, cmnd->scst_cmd, cmnd->scst_state,
209                         cmnd->parent_req);
210         }
211
212         EXTRACHECKS_BUG_ON(cmnd->on_rx_digest_list);
213
214         if (cmnd->parent_req == NULL) {
215                 struct iscsi_conn *conn = cmnd->conn;
216                 TRACE_DBG("Deleting req %p from conn %p", cmnd, conn);
217
218                 spin_lock_bh(&conn->cmd_list_lock);
219                 list_del(&cmnd->cmd_list_entry);
220                 spin_unlock_bh(&conn->cmd_list_lock);
221
222                 conn_put(conn);
223
224                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rsp_cmd_list));
225                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rx_ddigest_cmd_list));
226
227                 /* Order between above and below code is important! */
228
229                 if (cmnd->scst_cmd) {
230                         switch(cmnd->scst_state) {
231                         case ISCSI_CMD_STATE_AFTER_PREPROC:
232                         {
233                                 struct scst_cmd *scst_cmd = cmnd->scst_cmd;
234                                 TRACE_DBG("cmd %p AFTER_PREPROC", cmnd);
235                                 cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
236                                 cmnd->scst_cmd = NULL;
237                                 scst_restart_cmd(scst_cmd,
238                                         SCST_PREPROCESS_STATUS_ERROR_FATAL,
239                                         SCST_CONTEXT_THREAD);
240                                 break;
241                         }
242                         case ISCSI_CMD_STATE_PROCESSED:
243                                 TRACE_DBG("cmd %p PROCESSED", cmnd);
244                                 scst_tgt_cmd_done(cmnd->scst_cmd);
245                                 break;
246                         default:
247                                 PRINT_ERROR("Unexpected cmnd scst state %d",
248                                         cmnd->scst_state);
249                                 sBUG();
250                                 break;
251                         }
252                 }
253         } else {
254                 EXTRACHECKS_BUG_ON(cmnd->scst_cmd != NULL);
255                 TRACE_DBG("Deleting rsp %p from parent %p", cmnd,
256                         cmnd->parent_req);
257
258                 spin_lock_bh(&cmnd->parent_req->rsp_cmd_lock);
259                 list_del(&cmnd->rsp_cmd_list_entry);
260                 spin_unlock_bh(&cmnd->parent_req->rsp_cmd_lock);
261
262                 cmnd_put(cmnd->parent_req);
263         }
264
265         /* Order between above and below code is important! */
266
267         if (cmnd->own_sg) {
268                 TRACE_DBG("%s", "own_sg");
269                 scst_free(cmnd->sg, cmnd->sg_cnt);
270 #ifdef DEBUG
271                 cmnd->own_sg = 0;
272                 cmnd->sg = NULL;
273                 cmnd->sg_cnt = -1;
274 #endif
275         }
276
277         if (cmnd->dec_active_cmnds) {
278                 struct iscsi_session *sess = cmnd->conn->session;
279                 TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
280                         "new value %d)", cmnd, sess,
281                         atomic_read(&sess->active_cmds)-1);
282                 atomic_dec(&sess->active_cmds);
283         }
284
285         cmnd_free(cmnd);
286         return;
287 }
288
289 /*
290  * Corresponding conn may also gets destroyed atfer this function, except only
291  * if it's called from the read thread!
292  *
293  * It can't be called in parallel with iscsi_cmnds_init_write()!
294  */
295 void req_cmnd_release_force(struct iscsi_cmnd *req, int flags)
296 {
297         struct iscsi_cmnd *rsp, *t;
298         struct iscsi_conn *conn = req->conn;
299         LIST_HEAD(cmds_list);
300
301         TRACE_ENTRY();
302
303         TRACE_DBG("%p", req);
304
305         sBUG_ON(req == conn->read_cmnd);
306
307         if (flags & ISCSI_FORCE_RELEASE_WRITE) {
308                 spin_lock(&conn->write_list_lock);
309                 list_for_each_entry_safe(rsp, t, &conn->write_list,
310                                                 write_list_entry) {
311                         if (rsp->parent_req != req)
312                                 continue;
313
314                         cmd_del_from_write_list(rsp);
315
316                         list_add_tail(&rsp->write_list_entry, &cmds_list);
317                 }
318                 spin_unlock(&conn->write_list_lock);
319
320                 list_for_each_entry_safe(rsp, t, &cmds_list, write_list_entry) {
321                         list_del(&rsp->write_list_entry);
322                         cmnd_put(rsp);
323                 }
324         }
325
326 again_rsp:
327         spin_lock_bh(&req->rsp_cmd_lock);
328         list_for_each_entry_reverse(rsp, &req->rsp_cmd_list, rsp_cmd_list_entry) {
329                 bool r;
330
331                 if (rsp->force_cleanup_done)
332                         continue;
333
334                 rsp->force_cleanup_done = 1;
335
336                 if (cmnd_get_check(rsp))
337                         continue;
338
339                 spin_unlock_bh(&req->rsp_cmd_lock);
340
341                 spin_lock(&conn->write_list_lock);
342                 r = rsp->on_write_list || rsp->write_processing_started;
343                 spin_unlock(&conn->write_list_lock);
344
345                 cmnd_put(rsp);
346
347                 if (r)
348                         continue;
349
350                 /*
351                  * If both on_write_list and write_processing_started not set,
352                  * we can safely put() cmnd
353                  */
354                 cmnd_put(rsp);
355                 goto again_rsp;
356         }
357         spin_unlock_bh(&req->rsp_cmd_lock);
358
359         req_cmnd_release(req);
360
361         TRACE_EXIT();
362         return;
363 }
364
365 /*
366  * Corresponding conn may also gets destroyed atfer this function, except only
367  * if it's called from the read thread! 
368  */
369 void req_cmnd_release(struct iscsi_cmnd *req)
370 {
371         struct iscsi_cmnd *c, *t;
372
373         TRACE_ENTRY();
374
375         TRACE_DBG("%p", req);
376
377 #ifdef EXTRACHECKS
378         sBUG_ON(req->release_called);
379         req->release_called = 1;
380 #endif
381
382         if (unlikely(req->tmfabort)) {
383                 TRACE_MGMT_DBG("Release aborted req cmd %p (scst cmd %p, "
384                         "state %d)", req, req->scst_cmd, req->scst_state);
385         }
386
387         sBUG_ON(req->parent_req != NULL);
388
389         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
390                                 rx_ddigest_cmd_list_entry) {
391                 cmd_del_from_rx_ddigest_list(c);
392                 cmnd_put(c);
393         }
394
395         if (req->hashed)
396                 cmnd_remove_hash(req);
397
398         if (req->dec_active_cmnds) {
399                 struct iscsi_session *sess = req->conn->session;
400                 TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
401                         "new value %d)", req, sess,
402                         atomic_read(&sess->active_cmds)-1);
403                 atomic_dec(&sess->active_cmds);
404                 req->dec_active_cmnds = 0;
405         }
406
407         cmnd_put(req);
408
409         TRACE_EXIT();
410         return;
411 }
412
413 /*
414  * Corresponding conn may also gets destroyed atfer this function, except only
415  * if it's called from the read thread! 
416  */
417 void rsp_cmnd_release(struct iscsi_cmnd *cmnd)
418 {
419         TRACE_DBG("%p", cmnd);
420
421 #ifdef EXTRACHECKS
422         sBUG_ON(cmnd->release_called);
423         cmnd->release_called = 1;
424 #endif
425
426         sBUG_ON(cmnd->hashed);
427         sBUG_ON(cmnd->parent_req == NULL);
428
429         cmnd_put(cmnd);
430         return;
431 }
432
433 /**
434  * create a new command used as response.
435  *
436  * iscsi_cmnd_create_rsp_cmnd - 
437  * @cmnd: ptr to request command
438  *
439  * @return    ptr to response command or NULL
440  */
441 static struct iscsi_cmnd *iscsi_cmnd_create_rsp_cmnd(struct iscsi_cmnd *parent)
442 {
443         struct iscsi_cmnd *rsp;
444
445         rsp = cmnd_alloc(parent->conn, parent);
446
447         spin_lock_bh(&parent->rsp_cmd_lock);
448         TRACE_DBG("Adding rsp %p to parent %p", rsp, parent);
449         list_add_tail(&rsp->rsp_cmd_list_entry, &parent->rsp_cmd_list);
450         spin_unlock_bh(&parent->rsp_cmd_lock);
451         cmnd_get(parent);
452         return rsp;
453 }
454
455 static inline struct iscsi_cmnd *get_rsp_cmnd(struct iscsi_cmnd *req)
456 {
457         struct iscsi_cmnd *res = NULL;
458
459         /* Currently this lock isn't needed, but just in case.. */
460         spin_lock_bh(&req->rsp_cmd_lock);
461         if (!list_empty(&req->rsp_cmd_list)) {
462                 res = list_entry(req->rsp_cmd_list.prev, struct iscsi_cmnd,
463                         rsp_cmd_list_entry);
464         }
465         spin_unlock_bh(&req->rsp_cmd_lock);
466
467         return res;
468 }
469
470 static void iscsi_cmnds_init_write(struct list_head *send, int flags)
471 {
472         struct iscsi_cmnd *rsp = list_entry(send->next, struct iscsi_cmnd, 
473                                                 write_list_entry);
474         struct iscsi_conn *conn = rsp->conn;
475         struct list_head *pos, *next;
476
477         sBUG_ON(list_empty(send));
478
479         /*
480          * If we don't remove hashed req cmd from the hash list here, before
481          * submitting it for transmittion, we will have a race, when for
482          * some reason cmd's release is delayed after transmittion and
483          * initiator sends cmd with the same ITT => this command will be
484          * erroneously rejected as a duplicate.
485          */
486         if ((flags & ISCSI_INIT_WRITE_REMOVE_HASH) && rsp->parent_req->hashed &&
487             (rsp->parent_req->r2t_length == 0) &&
488             (rsp->parent_req->outstanding_r2t == 0))
489                 cmnd_remove_hash(rsp->parent_req);
490
491         list_for_each_safe(pos, next, send) {
492                 rsp = list_entry(pos, struct iscsi_cmnd, write_list_entry);
493
494                 TRACE_DBG("%p:%x", rsp, cmnd_opcode(rsp));
495
496                 sBUG_ON(conn != rsp->conn);
497
498                 if (!(conn->ddigest_type & DIGEST_NONE) &&
499                     (rsp->pdu.datasize != 0))
500                         digest_tx_data(rsp);
501
502                 list_del(&rsp->write_list_entry);
503
504                 spin_lock(&conn->write_list_lock);
505                 cmd_add_on_write_list(conn, rsp);
506                 spin_unlock(&conn->write_list_lock);
507         }
508
509         if (flags & ISCSI_INIT_WRITE_WAKE)
510                 iscsi_make_conn_wr_active(conn);
511 }
512
513 static void iscsi_cmnd_init_write(struct iscsi_cmnd *rsp, int flags)
514 {
515         LIST_HEAD(head);
516
517         if (unlikely(rsp->on_write_list)) {
518                 PRINT_ERROR("cmd already on write list (%x %x %x %x %u %u "
519                         "%u %u %u %u %u %d %d",
520                         cmnd_itt(rsp), cmnd_ttt(rsp), cmnd_opcode(rsp),
521                         cmnd_scsicode(rsp), rsp->r2t_sn,
522                         rsp->r2t_length, rsp->is_unsolicited_data,
523                         rsp->target_task_tag, rsp->outstanding_r2t,
524                         rsp->hdigest, rsp->ddigest,
525                         list_empty(&rsp->rsp_cmd_list), rsp->hashed);
526                 sBUG();
527         }
528         list_add(&rsp->write_list_entry, &head);
529         iscsi_cmnds_init_write(&head, flags);
530 }
531
532 static void iscsi_set_datasize(struct iscsi_cmnd *cmnd, u32 offset, u32 size)
533 {
534         cmnd->pdu.datasize = size;
535
536         if (cmnd->pdu.datasize & 3) {
537                 int idx = (offset + cmnd->pdu.datasize) >> PAGE_SHIFT;
538                 u8 *p = (u8 *)page_address(sg_page(&cmnd->sg[idx])) + 
539                         ((offset + cmnd->pdu.datasize) & ~PAGE_MASK);
540                 int i = 4 - (cmnd->pdu.datasize & 3);
541                 while (i--) 
542                     *p++ = 0;
543         }
544 }
545
546 static void send_data_rsp(struct iscsi_cmnd *req, u8 status, int send_status)
547 {
548         struct iscsi_cmnd *rsp;
549         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
550         struct iscsi_data_in_hdr *rsp_hdr;
551         u32 pdusize, expsize, scsisize, size, offset, sn;
552         LIST_HEAD(send);
553
554         TRACE_DBG("req %p", req);
555         pdusize = req->conn->session->sess_param.max_xmit_data_length;
556         expsize = cmnd_read_size(req);
557         size = min(expsize, (u32)req->bufflen);
558         offset = 0;
559         sn = 0;
560
561         while (1) {
562                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
563                 TRACE_DBG("rsp %p", rsp);
564                 rsp->sg = req->sg;
565                 rsp->bufflen = req->bufflen;
566                 rsp_hdr = (struct iscsi_data_in_hdr *)&rsp->pdu.bhs;
567
568                 rsp_hdr->opcode = ISCSI_OP_SCSI_DATA_IN;
569                 rsp_hdr->itt = req_hdr->itt;
570                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
571                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
572                 rsp_hdr->data_sn = cpu_to_be32(sn);
573
574                 if (size <= pdusize) {
575                         iscsi_set_datasize(rsp, offset, size);
576                         if (send_status) {
577                                 TRACE_DBG("status %x", status);
578                                 rsp_hdr->flags =
579                                         ISCSI_FLG_FINAL | ISCSI_FLG_STATUS;
580                                 rsp_hdr->cmd_status = status;
581                         }
582                         scsisize = req->bufflen;
583                         if (scsisize < expsize) {
584                                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
585                                 size = expsize - scsisize;
586                         } else if (scsisize > expsize) {
587                                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
588                                 size = scsisize - expsize;
589                         } else
590                                 size = 0;
591                         rsp_hdr->residual_count = cpu_to_be32(size);
592                         list_add_tail(&rsp->write_list_entry, &send);
593                         break;
594                 }
595
596                 iscsi_set_datasize(rsp, offset, pdusize);
597
598                 size -= pdusize;
599                 offset += pdusize;
600                 sn++;
601
602                 list_add_tail(&rsp->write_list_entry, &send);
603         }
604         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_REMOVE_HASH);
605 }
606
607 static struct iscsi_cmnd *create_status_rsp(struct iscsi_cmnd *req, int status,
608         const u8 *sense_buf, int sense_len)
609 {
610         struct iscsi_cmnd *rsp;
611         struct iscsi_scsi_rsp_hdr *rsp_hdr;
612         struct iscsi_sense_data *sense;
613         struct scatterlist *sg;
614
615         rsp = iscsi_cmnd_create_rsp_cmnd(req);
616         TRACE_DBG("%p", rsp);
617
618         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
619         rsp_hdr->opcode = ISCSI_OP_SCSI_RSP;
620         rsp_hdr->flags = ISCSI_FLG_FINAL;
621         rsp_hdr->response = ISCSI_RESPONSE_COMMAND_COMPLETED;
622         rsp_hdr->cmd_status = status;
623         rsp_hdr->itt = cmnd_hdr(req)->itt;
624
625         if (SCST_SENSE_VALID(sense_buf)) {
626                 TRACE_DBG("%s", "SENSE VALID");
627                 /* ToDo: __GFP_NOFAIL ?? */
628                 sg = rsp->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
629                                         &rsp->sg_cnt);
630                 if (sg == NULL) {
631                         /* ToDo(); */
632                 }
633                 rsp->own_sg = 1;
634                 sense = (struct iscsi_sense_data *)page_address(sg_page(&sg[0]));
635                 sense->length = cpu_to_be16(sense_len);
636                 memcpy(sense->data, sense_buf, sense_len);
637                 rsp->pdu.datasize = sizeof(struct iscsi_sense_data) + sense_len;
638                 rsp->bufflen = (rsp->pdu.datasize + 3) & -4;
639                 if (rsp->bufflen - rsp->pdu.datasize) {
640                     int i = rsp->pdu.datasize;
641                     u8 *p = (u8 *)sense + i;
642                     
643                     while (i < rsp->bufflen) {
644                         *p ++ = 0;
645                         i++;
646                     }
647                 }
648         } else {
649                 rsp->pdu.datasize = 0;
650                 rsp->bufflen = 0;
651         }
652
653         return rsp;
654 }
655
656 static struct iscsi_cmnd *create_sense_rsp(struct iscsi_cmnd *req,
657         u8 sense_key, u8 asc, u8 ascq)
658 {
659         u8 sense[14];
660         memset(sense, 0, sizeof(sense));
661         sense[0] = 0xf0;
662         sense[2] = sense_key;
663         sense[7] = 6;   // Additional sense length
664         sense[12] = asc;
665         sense[13] = ascq;
666         return create_status_rsp(req, SAM_STAT_CHECK_CONDITION, sense,
667                 sizeof(sense));
668 }
669
670 static void iscsi_cmnd_reject(struct iscsi_cmnd *req, int reason)
671 {
672         struct iscsi_cmnd *rsp;
673         struct iscsi_reject_hdr *rsp_hdr;
674         struct scatterlist *sg;
675         char *addr;
676
677         TRACE_MGMT_DBG("Reject: req %p, reason %x", req, reason);
678
679         rsp = iscsi_cmnd_create_rsp_cmnd(req);
680         rsp_hdr = (struct iscsi_reject_hdr *)&rsp->pdu.bhs;
681
682         rsp_hdr->opcode = ISCSI_OP_REJECT;
683         rsp_hdr->ffffffff = ISCSI_RESERVED_TAG;
684         rsp_hdr->reason = reason;
685
686         /* ToDo: __GFP_NOFAIL ?? */
687         sg = rsp->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
688                                 &rsp->sg_cnt);
689         if (sg == NULL) {
690                 /* ToDo(); */
691         }
692         rsp->own_sg = 1;
693         addr = page_address(sg_page(&sg[0]));
694         clear_page(addr);
695         memcpy(addr, &req->pdu.bhs, sizeof(struct iscsi_hdr));
696         rsp->bufflen = rsp->pdu.datasize = sizeof(struct iscsi_hdr);
697         cmnd_prepare_skip_pdu(req);
698
699         req->pdu.bhs.opcode = ISCSI_OP_PDU_REJECT;
700 }
701
702 static inline int iscsi_get_allowed_cmds(struct iscsi_session *sess)
703 {
704         int res = max(-1, (int)sess->max_queued_cmnds - 
705                                 atomic_read(&sess->active_cmds)-1);
706         TRACE_DBG("allowed cmds %d (sess %p, active_cmds %d)", res,
707                 sess, atomic_read(&sess->active_cmds));
708         return res;
709 }
710
711 static u32 cmnd_set_sn(struct iscsi_cmnd *cmnd, int set_stat_sn)
712 {
713         struct iscsi_conn *conn = cmnd->conn;
714         struct iscsi_session *sess = conn->session;
715         u32 res;
716
717         spin_lock(&sess->sn_lock);
718
719         if (set_stat_sn)
720                 cmnd->pdu.bhs.sn = cpu_to_be32(conn->stat_sn++);
721         cmnd->pdu.bhs.exp_sn = cpu_to_be32(sess->exp_cmd_sn);
722         cmnd->pdu.bhs.max_sn = cpu_to_be32(sess->exp_cmd_sn +
723                                  iscsi_get_allowed_cmds(sess));
724
725         res = cpu_to_be32(conn->stat_sn);
726
727         spin_unlock(&sess->sn_lock);
728         return res;
729 }
730
731 /* Called under sn_lock */
732 static void __update_stat_sn(struct iscsi_cmnd *cmnd)
733 {
734         struct iscsi_conn *conn = cmnd->conn;
735         u32 exp_stat_sn;
736
737         cmnd->pdu.bhs.exp_sn = exp_stat_sn = be32_to_cpu(cmnd->pdu.bhs.exp_sn);
738         TRACE_DBG("%x,%x", cmnd_opcode(cmnd), exp_stat_sn);
739         if ((int)(exp_stat_sn - conn->exp_stat_sn) > 0 &&
740             (int)(exp_stat_sn - conn->stat_sn) <= 0) {
741                 // free pdu resources
742                 cmnd->conn->exp_stat_sn = exp_stat_sn;
743         }
744 }
745
746 static inline void update_stat_sn(struct iscsi_cmnd *cmnd)
747 {
748         spin_lock(&cmnd->conn->session->sn_lock);
749         __update_stat_sn(cmnd);
750         spin_unlock(&cmnd->conn->session->sn_lock);
751 }
752
753 /* Called under sn_lock */
754 static int check_cmd_sn(struct iscsi_cmnd *cmnd)
755 {
756         struct iscsi_session *session = cmnd->conn->session;
757         u32 cmd_sn;
758
759         cmnd->pdu.bhs.sn = cmd_sn = be32_to_cpu(cmnd->pdu.bhs.sn);
760         TRACE_DBG("%d(%d)", cmd_sn, session->exp_cmd_sn);
761         if (likely((s32)(cmd_sn - session->exp_cmd_sn) >= 0))
762                 return 0;
763         PRINT_ERROR("sequence error (%x,%x)", cmd_sn, session->exp_cmd_sn);
764         return -ISCSI_REASON_PROTOCOL_ERROR;
765 }
766
767 static inline struct iscsi_cmnd *__cmnd_find_hash(struct iscsi_session *session,
768         u32 itt, u32 ttt)
769 {
770         struct list_head *head;
771         struct iscsi_cmnd *cmnd;
772
773         head = &session->cmnd_hash[cmnd_hashfn(itt)];
774
775         list_for_each_entry(cmnd, head, hash_list_entry) {
776                 if (cmnd->pdu.bhs.itt == itt) {
777                         if ((ttt != ISCSI_RESERVED_TAG) && (ttt != cmnd->target_task_tag))
778                                 continue;
779                         return cmnd;
780                 }
781         }
782         return NULL;
783 }
784
785 static struct iscsi_cmnd *cmnd_find_hash(struct iscsi_session *session,
786         u32 itt, u32 ttt)
787 {
788         struct iscsi_cmnd *cmnd;
789
790         spin_lock(&session->cmnd_hash_lock);
791         cmnd = __cmnd_find_hash(session, itt, ttt);
792         spin_unlock(&session->cmnd_hash_lock);
793
794         return cmnd;
795 }
796
797 static struct iscsi_cmnd *cmnd_find_hash_get(struct iscsi_session *session,
798         u32 itt, u32 ttt)
799 {
800         struct iscsi_cmnd *cmnd;
801
802         spin_lock(&session->cmnd_hash_lock);
803         cmnd = __cmnd_find_hash(session, itt, ttt);
804         if (cmnd != NULL) {
805                 if (unlikely(cmnd_get_check(cmnd)))
806                         cmnd = NULL;
807         }
808         spin_unlock(&session->cmnd_hash_lock);
809
810         return cmnd;
811 }
812
813 static int cmnd_insert_hash(struct iscsi_cmnd *cmnd)
814 {
815         struct iscsi_session *session = cmnd->conn->session;
816         struct iscsi_cmnd *tmp;
817         struct list_head *head;
818         int err = 0;
819         u32 itt = cmnd->pdu.bhs.itt;
820
821         TRACE_DBG("%p:%x", cmnd, itt);
822         if (unlikely(itt == ISCSI_RESERVED_TAG)) {
823                 PRINT_ERROR("%s", "ITT is RESERVED_TAG");
824                 PRINT_BUFFER("Incorrect BHS", &cmnd->pdu.bhs,
825                         sizeof(cmnd->pdu.bhs));
826                 err = -ISCSI_REASON_PROTOCOL_ERROR;
827                 goto out;
828         }
829
830         spin_lock(&session->cmnd_hash_lock);
831
832         head = &session->cmnd_hash[cmnd_hashfn(cmnd->pdu.bhs.itt)];
833
834         tmp = __cmnd_find_hash(session, itt, ISCSI_RESERVED_TAG);
835         if (likely(!tmp)) {
836                 list_add_tail(&cmnd->hash_list_entry, head);
837                 cmnd->hashed = 1;
838         } else {
839                 PRINT_ERROR("Task %x in progress, cmnd %p", itt, cmnd);
840                 err = -ISCSI_REASON_TASK_IN_PROGRESS;
841         }
842
843         spin_unlock(&session->cmnd_hash_lock);
844
845         if (likely(!err)) {
846                 spin_lock(&session->sn_lock);
847                 __update_stat_sn(cmnd);
848                 err = check_cmd_sn(cmnd);
849                 spin_unlock(&session->sn_lock);
850         }
851
852 out:
853         return err;
854 }
855
856 static void cmnd_remove_hash(struct iscsi_cmnd *cmnd)
857 {
858         struct iscsi_session *session = cmnd->conn->session;
859         struct iscsi_cmnd *tmp;
860
861         spin_lock(&session->cmnd_hash_lock);
862
863         tmp = __cmnd_find_hash(session, cmnd->pdu.bhs.itt, ISCSI_RESERVED_TAG);
864
865         if (likely(tmp && tmp == cmnd)) {
866                 list_del(&cmnd->hash_list_entry);
867                 cmnd->hashed = 0;
868         } else {
869                 PRINT_ERROR("%p:%x not found", cmnd, cmnd_itt(cmnd));
870         }
871
872         spin_unlock(&session->cmnd_hash_lock);
873 }
874
875 static void cmnd_prepare_skip_pdu(struct iscsi_cmnd *cmnd)
876 {
877         struct iscsi_conn *conn = cmnd->conn;
878         struct scatterlist *sg = cmnd->sg;
879         char *addr;
880         u32 size;
881         int i;
882
883         TRACE_MGMT_DBG("Skipping (%p, %x %x %x %u, %p, scst state %d)", cmnd,
884                 cmnd_itt(cmnd), cmnd_opcode(cmnd), cmnd_hdr(cmnd)->scb[0],
885                 cmnd->pdu.datasize, cmnd->scst_cmd, cmnd->scst_state);
886
887         iscsi_extracheck_is_rd_thread(conn);
888
889         if (!(size = cmnd->pdu.datasize))
890                 return;
891
892         if (sg == NULL) {
893                 /* ToDo: __GFP_NOFAIL ?? */
894                 sg = cmnd->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
895                                         &cmnd->sg_cnt);
896                 if (sg == NULL) {
897                         /* ToDo(); */
898                 }
899                 cmnd->own_sg = 1;
900                 cmnd->bufflen = PAGE_SIZE;
901         }
902
903         addr = page_address(sg_page(&sg[0]));
904         sBUG_ON(addr == NULL);
905         size = (size + 3) & -4;
906         conn->read_size = size;
907         for (i = 0; size > PAGE_SIZE; i++, size -= cmnd->bufflen) {
908                 sBUG_ON(i >= ISCSI_CONN_IOV_MAX);
909                 conn->read_iov[i].iov_base = addr;
910                 conn->read_iov[i].iov_len = cmnd->bufflen;
911         }
912         conn->read_iov[i].iov_base = addr;
913         conn->read_iov[i].iov_len = size;
914         conn->read_msg.msg_iov = conn->read_iov;
915         conn->read_msg.msg_iovlen = ++i;
916 }
917
918 static void cmnd_prepare_skip_pdu_set_resid(struct iscsi_cmnd *req)
919 {
920         struct iscsi_cmnd *rsp;
921         struct iscsi_scsi_rsp_hdr *rsp_hdr;
922         u32 size;
923
924         TRACE_DBG("%p", req);
925
926         rsp = get_rsp_cmnd(req);
927         if (rsp == NULL)
928                 goto skip;
929
930         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
931
932         sBUG_ON(cmnd_opcode(rsp) != ISCSI_OP_SCSI_RSP);
933
934         size = cmnd_write_size(req);
935         if (size) {
936                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
937                 rsp_hdr->residual_count = cpu_to_be32(size);
938         }
939         size = cmnd_read_size(req);
940         if (size) {
941                 if (cmnd_hdr(req)->flags & ISCSI_CMD_WRITE) {
942                         rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_UNDERFLOW;
943                         rsp_hdr->bi_residual_count = cpu_to_be32(size);
944                 } else {
945                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
946                         rsp_hdr->residual_count = cpu_to_be32(size);
947                 }
948         }
949
950 skip:
951         req->pdu.bhs.opcode =
952                 (req->pdu.bhs.opcode & ~ISCSI_OPCODE_MASK) | ISCSI_OP_SCSI_REJECT;
953
954         cmnd_prepare_skip_pdu(req);
955 }
956
957 static int cmnd_prepare_recv_pdu(struct iscsi_conn *conn,
958         struct iscsi_cmnd *cmd, u32 offset, u32 size)
959 {
960         struct scatterlist *sg = cmd->sg;
961         int bufflen = cmd->bufflen;
962         int idx, i;
963         char *addr;
964         int res = 0;
965
966         TRACE_DBG("%p %u,%u", cmd->sg, offset, size);
967
968         iscsi_extracheck_is_rd_thread(conn);
969
970         if (unlikely((offset >= bufflen) ||
971                      (offset + size > bufflen))) {
972                 PRINT_ERROR("Wrong ltn (%u %u %u)", offset, size, bufflen);
973                 mark_conn_closed(conn);
974                 res = -EIO;
975                 goto out;
976         }
977
978         offset += sg[0].offset;
979         idx = offset >> PAGE_SHIFT;
980         offset &= ~PAGE_MASK;
981
982         conn->read_msg.msg_iov = conn->read_iov;
983         conn->read_size = size = (size + 3) & -4;
984
985         i = 0;
986         while (1) {
987                 addr = page_address(sg_page(&sg[idx]));
988                 sBUG_ON(addr == NULL);
989                 conn->read_iov[i].iov_base = addr + offset;
990                 if (offset + size <= PAGE_SIZE) {
991                         TRACE_DBG("idx=%d, offset=%u, size=%d, addr=%p",
992                                 idx, offset, size, addr);
993                         conn->read_iov[i].iov_len = size;
994                         conn->read_msg.msg_iovlen = ++i;
995                         break;
996                 }
997                 conn->read_iov[i].iov_len = PAGE_SIZE - offset;
998                 TRACE_DBG("idx=%d, offset=%u, size=%d, iov_len=%d, addr=%p",
999                         idx, offset, size, conn->read_iov[i].iov_len, addr);
1000                 size -= conn->read_iov[i].iov_len;
1001                 offset = 0;
1002                 if (unlikely(++i >= ISCSI_CONN_IOV_MAX)) {
1003                         PRINT_ERROR("Initiator %s violated negotiated "
1004                                 "parameters by sending too much data (size "
1005                                 "left %d)", conn->session->initiator_name, size);
1006                         mark_conn_closed(conn);
1007                         res = -EINVAL;
1008                         break;
1009                 }
1010                 idx++;
1011         }
1012         TRACE_DBG("msg_iov=%p, msg_iovlen=%d",
1013                 conn->read_msg.msg_iov, conn->read_msg.msg_iovlen);
1014
1015 out:
1016         return res;
1017 }
1018
1019 static void send_r2t(struct iscsi_cmnd *req)
1020 {
1021         struct iscsi_session *session = req->conn->session;
1022         struct iscsi_cmnd *rsp;
1023         struct iscsi_r2t_hdr *rsp_hdr;
1024         u32 offset, burst;
1025         LIST_HEAD(send);
1026
1027         if (unlikely(req->tmfabort)) {
1028                 TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted on R2T "
1029                         "(r2t_length %d, outstanding_r2t %d)", req,
1030                         req->scst_cmd, req->r2t_length, req->outstanding_r2t);
1031                 if (req->outstanding_r2t == 0)
1032                         iscsi_fail_waiting_cmnd(req);
1033                 goto out;
1034         }
1035
1036         /*
1037          * There is no race with data_out_start() and conn_abort(), since
1038          * all functions called from single read thread
1039          */
1040         iscsi_extracheck_is_rd_thread(req->conn);
1041
1042         burst = session->sess_param.max_burst_length;
1043         offset = be32_to_cpu(cmnd_hdr(req)->data_length) - req->r2t_length;
1044
1045         do {
1046                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
1047                 rsp->pdu.bhs.ttt = req->target_task_tag;
1048                 rsp_hdr = (struct iscsi_r2t_hdr *)&rsp->pdu.bhs;
1049                 rsp_hdr->opcode = ISCSI_OP_R2T;
1050                 rsp_hdr->flags = ISCSI_FLG_FINAL;
1051                 rsp_hdr->lun = cmnd_hdr(req)->lun;
1052                 rsp_hdr->itt = cmnd_hdr(req)->itt;
1053                 rsp_hdr->r2t_sn = cpu_to_be32(req->r2t_sn++);
1054                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
1055                 if (req->r2t_length > burst) {
1056                         rsp_hdr->data_length = cpu_to_be32(burst);
1057                         req->r2t_length -= burst;
1058                         offset += burst;
1059                 } else {
1060                         rsp_hdr->data_length = cpu_to_be32(req->r2t_length);
1061                         req->r2t_length = 0;
1062                 }
1063
1064                 TRACE_WRITE("%x %u %u %u %u", cmnd_itt(req),
1065                         be32_to_cpu(rsp_hdr->data_length),
1066                         be32_to_cpu(rsp_hdr->buffer_offset),
1067                         be32_to_cpu(rsp_hdr->r2t_sn), req->outstanding_r2t);
1068
1069                 list_add_tail(&rsp->write_list_entry, &send);
1070
1071                 if (++req->outstanding_r2t >= session->sess_param.max_outstanding_r2t)
1072                         break;
1073
1074         } while(req->r2t_length != 0);
1075
1076         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_WAKE);
1077
1078 out:
1079         return;
1080 }
1081
1082 static int iscsi_pre_exec(struct scst_cmd *scst_cmd)
1083 {
1084         int res = SCST_PREPROCESS_STATUS_SUCCESS;
1085         struct iscsi_cmnd *req = (struct iscsi_cmnd*)
1086                 scst_cmd_get_tgt_priv(scst_cmd);
1087         struct iscsi_cmnd *c, *t;
1088
1089         TRACE_ENTRY();
1090
1091         EXTRACHECKS_BUG_ON(scst_cmd_atomic(scst_cmd));
1092
1093         /* If data digest isn't used this list will be empty */
1094         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
1095                                 rx_ddigest_cmd_list_entry) {
1096                 TRACE_DBG("Checking digest of RX ddigest cmd %p", c);
1097                 if (digest_rx_data(c) != 0) {
1098                         scst_set_cmd_error(scst_cmd,
1099                                 SCST_LOAD_SENSE(iscsi_sense_crc_error));
1100                         res = SCST_PREPROCESS_STATUS_ERROR_SENSE_SET;
1101                         /*
1102                          * The rest of rx_ddigest_cmd_list will be freed 
1103                          * in req_cmnd_release()
1104                          */
1105                         goto out;
1106                 }
1107                 cmd_del_from_rx_ddigest_list(c);
1108                 cmnd_put(c);
1109         }
1110
1111 out:
1112         TRACE_EXIT_RES(res);
1113         return res;
1114 }
1115
1116 static int noop_out_start(struct iscsi_cmnd *cmnd)
1117 {
1118         struct iscsi_conn *conn = cmnd->conn;
1119         u32 size, tmp;
1120         int i, err = 0;
1121
1122         TRACE_DBG("%p", cmnd);
1123
1124         iscsi_extracheck_is_rd_thread(conn);
1125
1126         if (unlikely(cmnd_ttt(cmnd) != cpu_to_be32(ISCSI_RESERVED_TAG))) {
1127                 /*
1128                  * We don't request a NOP-Out by sending a NOP-In.
1129                  * See 10.18.2 in the draft 20.
1130                  */
1131                 PRINT_ERROR("Initiator sent command with not RESERVED tag and "
1132                         "TTT %x", cmnd_itt(cmnd));
1133                 err = -ISCSI_REASON_PROTOCOL_ERROR;
1134                 goto out;
1135         }
1136
1137         if (cmnd_itt(cmnd) == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1138                 if (unlikely(!(cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE)))
1139                         PRINT_ERROR("%s", "Initiator sent RESERVED tag for "
1140                                 "non-immediate command");
1141                 spin_lock(&conn->session->sn_lock);
1142                 __update_stat_sn(cmnd);
1143                 err = check_cmd_sn(cmnd);
1144                 spin_unlock(&conn->session->sn_lock);
1145                 if (unlikely(err))
1146                         goto out;
1147         } else if (unlikely((err = cmnd_insert_hash(cmnd)) < 0)) {
1148                 PRINT_ERROR("Can't insert in hash: ignore this request %x",
1149                         cmnd_itt(cmnd));
1150                 goto out;
1151         }
1152
1153         if ((size = cmnd->pdu.datasize)) {
1154                 size = (size + 3) & -4;
1155                 conn->read_msg.msg_iov = conn->read_iov;
1156                 if (cmnd->pdu.bhs.itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1157                         struct scatterlist *sg;
1158
1159                         /* ToDo: __GFP_NOFAIL ?? */
1160                         cmnd->sg = sg = scst_alloc(size,
1161                                 GFP_KERNEL|__GFP_NOFAIL, &cmnd->sg_cnt);
1162                         if (sg == NULL) {
1163                                 /* ToDo(); */
1164                         }
1165                         if (cmnd->sg_cnt > ISCSI_CONN_IOV_MAX) {
1166                                 /* ToDo(); */
1167                         }
1168                         cmnd->own_sg = 1;
1169                         cmnd->bufflen = size;
1170
1171                         for (i = 0; i < cmnd->sg_cnt; i++) {
1172                                 conn->read_iov[i].iov_base =
1173                                         page_address(sg_page(&sg[i]));
1174                                 tmp = min_t(u32, size, PAGE_SIZE);
1175                                 conn->read_iov[i].iov_len = tmp;
1176                                 conn->read_size += tmp;
1177                                 size -= tmp;
1178                         }
1179                 } else {
1180                         /*
1181                          * There are no problems with the safety from concurrent
1182                          * accesses to dummy_data, since for ISCSI_RESERVED_TAG
1183                          * the data only read and then discarded.
1184                          */
1185                         for (i = 0; i < ISCSI_CONN_IOV_MAX; i++) {
1186                                 conn->read_iov[i].iov_base = dummy_data;
1187                                 tmp = min_t(u32, size, sizeof(dummy_data));
1188                                 conn->read_iov[i].iov_len = tmp;
1189                                 conn->read_size += tmp;
1190                                 size -= tmp;
1191                         }
1192                 }
1193                 sBUG_ON(size == 0);
1194                 conn->read_msg.msg_iovlen = i;
1195                 TRACE_DBG("msg_iov=%p, msg_iovlen=%d", conn->read_msg.msg_iov,
1196                         conn->read_msg.msg_iovlen);
1197         }
1198 out:
1199         return err;
1200 }
1201
1202 static inline u32 get_next_ttt(struct iscsi_conn *conn)
1203 {
1204         u32 ttt;
1205         struct iscsi_session *session = conn->session;
1206
1207         iscsi_extracheck_is_rd_thread(conn);
1208
1209         if (session->next_ttt == ISCSI_RESERVED_TAG)
1210                 session->next_ttt++;
1211         ttt = session->next_ttt++;
1212
1213         return cpu_to_be32(ttt);
1214 }
1215
1216 static int scsi_cmnd_start(struct iscsi_cmnd *req)
1217 {
1218         struct iscsi_conn *conn = req->conn;
1219         struct iscsi_session *session = conn->session;
1220         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1221         struct scst_cmd *scst_cmd;
1222         scst_data_direction dir;
1223         int res = 0;
1224
1225         TRACE_ENTRY();
1226
1227         TRACE_DBG("scsi command: %02x", req_hdr->scb[0]);
1228
1229         TRACE_DBG("Incrementing active_cmds (cmd %p, sess %p, "
1230                 "new value %d)", req, session,
1231                 atomic_read(&session->active_cmds)+1);
1232         atomic_inc(&session->active_cmds);
1233         req->dec_active_cmnds = 1;
1234
1235         scst_cmd = scst_rx_cmd(session->scst_sess,
1236                 (uint8_t*)&req_hdr->lun, sizeof(req_hdr->lun),
1237                 req_hdr->scb, sizeof(req_hdr->scb), SCST_NON_ATOMIC);
1238         if (scst_cmd == NULL) {
1239                 create_status_rsp(req, SAM_STAT_BUSY, NULL, 0);
1240                 cmnd_prepare_skip_pdu_set_resid(req);
1241                 goto out;
1242         }
1243
1244         req->scst_cmd = scst_cmd;
1245         scst_cmd_set_tag(scst_cmd, req_hdr->itt);
1246         scst_cmd_set_tgt_priv(scst_cmd, req);
1247 #ifndef NET_PAGE_CALLBACKS_DEFINED
1248         scst_cmd_set_data_buf_tgt_alloc(scst_cmd);
1249 #endif
1250
1251         if (req_hdr->flags & ISCSI_CMD_READ)
1252                 dir = SCST_DATA_READ;
1253         else if (req_hdr->flags & ISCSI_CMD_WRITE)
1254                 dir = SCST_DATA_WRITE;
1255         else
1256                 dir = SCST_DATA_NONE;
1257         scst_cmd_set_expected(scst_cmd, dir, be32_to_cpu(req_hdr->data_length));
1258
1259         switch(req_hdr->flags & ISCSI_CMD_ATTR_MASK) {
1260         case ISCSI_CMD_SIMPLE:
1261                 scst_cmd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1262                 break;
1263         case ISCSI_CMD_HEAD_OF_QUEUE:
1264                 scst_cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1265                 break;
1266         case ISCSI_CMD_ORDERED:
1267                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1268                 break;
1269         case ISCSI_CMD_ACA:
1270                 scst_cmd->queue_type = SCST_CMD_QUEUE_ACA;
1271                 break;
1272         case ISCSI_CMD_UNTAGGED:
1273                 scst_cmd->queue_type = SCST_CMD_QUEUE_UNTAGGED;
1274                 break;
1275         default:
1276                 PRINT_ERROR("Unknown task code %x, use ORDERED instead",
1277                         req_hdr->flags & ISCSI_CMD_ATTR_MASK);
1278                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1279                 break;
1280         }
1281
1282         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
1283         scst_cmd_set_tgt_sn(scst_cmd, req_hdr->cmd_sn);
1284
1285         TRACE_DBG("START Command (tag %d, queue_type %d)",
1286                 req_hdr->itt, scst_cmd->queue_type);
1287         req->scst_state = ISCSI_CMD_STATE_RX_CMD;
1288         scst_cmd_init_stage1_done(scst_cmd, SCST_CONTEXT_DIRECT, 0);
1289
1290         wait_event(req->scst_waitQ, (req->scst_state != ISCSI_CMD_STATE_RX_CMD));
1291
1292         if (unlikely(req->scst_state != ISCSI_CMD_STATE_AFTER_PREPROC)) {
1293                 TRACE_DBG("req %p is in %x state", req, req->scst_state);
1294                 if (req->scst_state == ISCSI_CMD_STATE_PROCESSED) {
1295                         cmnd_prepare_skip_pdu_set_resid(req);
1296                         goto out;
1297                 }
1298                 if (unlikely(req->tmfabort)) {
1299                         TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
1300                                 req->scst_cmd);
1301                         cmnd_prepare_skip_pdu(req);
1302                         goto out;
1303                 }
1304                 sBUG();
1305         }
1306
1307         dir = scst_cmd_get_data_direction(scst_cmd);
1308         if (dir != SCST_DATA_WRITE) {
1309                 if (unlikely(!(req_hdr->flags & ISCSI_CMD_FINAL) ||
1310                              req->pdu.datasize)) {
1311                         PRINT_ERROR("Unexpected unsolicited data (ITT %x "
1312                                 "CDB %x", cmnd_itt(req), req_hdr->scb[0]);
1313                         create_sense_rsp(req, ABORTED_COMMAND, 0xc, 0xc);
1314                         cmnd_prepare_skip_pdu_set_resid(req);
1315                         goto out;
1316                 }
1317         }
1318
1319         if (dir == SCST_DATA_WRITE) {
1320                 req->is_unsolicited_data = !(req_hdr->flags & ISCSI_CMD_FINAL);
1321                 req->r2t_length = be32_to_cpu(req_hdr->data_length) - req->pdu.datasize;
1322                 if (req->r2t_length > 0)
1323                         req->data_waiting = 1;
1324         }
1325         req->target_task_tag = get_next_ttt(conn);
1326         req->sg = scst_cmd_get_sg(scst_cmd);
1327         req->bufflen = scst_cmd_get_bufflen(scst_cmd);
1328         if (unlikely(req->r2t_length > req->bufflen)) {
1329                 PRINT_ERROR("req->r2t_length %d > req->bufflen %d",
1330                         req->r2t_length, req->bufflen);
1331                 req->r2t_length = req->bufflen;
1332         }
1333
1334         TRACE_DBG("req=%p, dir=%d, is_unsolicited_data=%d, "
1335                 "r2t_length=%d, bufflen=%d", req, dir,
1336                 req->is_unsolicited_data, req->r2t_length, req->bufflen);
1337
1338         if (unlikely(!session->sess_param.immediate_data &&
1339                      req->pdu.datasize)) {
1340                 PRINT_ERROR("Initiator %s violated negotiated paremeters: "
1341                         "forbidden immediate data sent (ITT %x, op  %x)",
1342                         session->initiator_name, cmnd_itt(req), req_hdr->scb[0]);
1343                 res = -EINVAL;
1344                 goto out;
1345         }
1346
1347         if (unlikely(session->sess_param.initial_r2t &&
1348                      !(req_hdr->flags & ISCSI_CMD_FINAL))) {
1349                 PRINT_ERROR("Initiator %s violated negotiated paremeters: "
1350                         "initial R2T is required (ITT %x, op  %x)",
1351                         session->initiator_name, cmnd_itt(req), req_hdr->scb[0]);
1352                 res = -EINVAL;
1353                 goto out;
1354         }
1355
1356         if (req->pdu.datasize) {
1357                 if (unlikely(dir != SCST_DATA_WRITE)) {
1358                         PRINT_ERROR("pdu.datasize(%d) >0, but dir(%x) isn't WRITE",
1359                                 req->pdu.datasize, dir);
1360                         create_sense_rsp(req, ABORTED_COMMAND, 0xc, 0xc);
1361                         cmnd_prepare_skip_pdu_set_resid(req);
1362                 } else
1363                         res = cmnd_prepare_recv_pdu(conn, req, 0, req->pdu.datasize);
1364         }
1365 out:
1366         /* Aborted commands will be freed in cmnd_rx_end() */
1367         TRACE_EXIT_RES(res);
1368         return res;
1369 }
1370
1371 static int data_out_start(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
1372 {
1373         struct iscsi_data_out_hdr *req_hdr = (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1374         struct iscsi_cmnd *req = NULL;
1375         u32 offset = be32_to_cpu(req_hdr->buffer_offset);
1376         int res = 0;
1377
1378         TRACE_ENTRY();
1379
1380         /*
1381          * There is no race with send_r2t() and conn_abort(), since
1382          * all functions called from single read thread
1383          */
1384         iscsi_extracheck_is_rd_thread(cmnd->conn);
1385
1386         update_stat_sn(cmnd);
1387
1388         cmnd->cmd_req = req = cmnd_find_hash(conn->session, req_hdr->itt,
1389                                         req_hdr->ttt);
1390         if (unlikely(req == NULL)) {
1391                 /* It might happen if req was aborted and then freed */
1392                 TRACE(TRACE_MGMT_MINOR, "Unable to find scsi task %x %x",
1393                         cmnd_itt(cmnd), cmnd_ttt(cmnd));
1394                 goto skip_pdu;
1395         }
1396
1397         if (req->is_unsolicited_data) {
1398                 if (unlikely(req->r2t_length < cmnd->pdu.datasize)) {
1399                         PRINT_ERROR("Data size (%d) > R2T length (%d)",
1400                                 cmnd->pdu.datasize, req->r2t_length);
1401                         mark_conn_closed(conn);
1402                         res = -EINVAL;
1403                         goto out;
1404                 }
1405                 req->r2t_length -= cmnd->pdu.datasize;
1406         }
1407
1408         /* Check unsolicited burst data */
1409         if (unlikely((req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) &&
1410                      (req->pdu.bhs.flags & ISCSI_FLG_FINAL))) {
1411                 PRINT_ERROR("Unexpected data from %x %x",
1412                         cmnd_itt(cmnd), cmnd_ttt(cmnd));
1413                 mark_conn_closed(conn);
1414                 res = -EINVAL;
1415                 goto out;
1416         }
1417
1418         TRACE_WRITE("%u %p %p %u %u", req_hdr->ttt, cmnd, req,
1419                 offset, cmnd->pdu.datasize);
1420
1421         res = cmnd_prepare_recv_pdu(conn, req, offset, cmnd->pdu.datasize);
1422
1423 out:
1424         TRACE_EXIT_RES(res);
1425         return res;
1426
1427 skip_pdu:
1428         cmnd->pdu.bhs.opcode = ISCSI_OP_DATA_REJECT;
1429         cmnd_prepare_skip_pdu(cmnd);
1430         goto out;
1431 }
1432
1433 static void data_out_end(struct iscsi_cmnd *cmnd)
1434 {
1435         struct iscsi_data_out_hdr *req_hdr = (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1436         struct iscsi_cmnd *req;
1437
1438         sBUG_ON(cmnd == NULL);
1439         req = cmnd->cmd_req;
1440         sBUG_ON(req == NULL);
1441
1442         TRACE_DBG("cmnd %p, req %p", cmnd, req);
1443
1444         iscsi_extracheck_is_rd_thread(cmnd->conn);
1445
1446         if (!(cmnd->conn->ddigest_type & DIGEST_NONE) &&
1447             !cmnd->ddigest_checked) {
1448                 cmd_add_on_rx_ddigest_list(req, cmnd);
1449                 cmnd_get(cmnd);
1450         }
1451
1452         if (req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1453                 TRACE_DBG("ISCSI_RESERVED_TAG, FINAL %x",
1454                         req_hdr->flags & ISCSI_FLG_FINAL);
1455
1456                 if (req_hdr->flags & ISCSI_FLG_FINAL) {
1457                         req->is_unsolicited_data = 0;
1458                         if (req->pending)
1459                                 goto out_put;
1460                 } else
1461                         goto out_put;
1462         } else {
1463                 TRACE_DBG("FINAL %x, outstanding_r2t %d, r2t_length %d",
1464                         req_hdr->flags & ISCSI_FLG_FINAL,
1465                         req->outstanding_r2t, req->r2t_length);
1466
1467                 if (req_hdr->flags & ISCSI_FLG_FINAL) {
1468                         if (unlikely(req->is_unsolicited_data)) {
1469                                 PRINT_ERROR("Unexpected unsolicited data "
1470                                         "(r2t_length %u, outstanding_r2t %d)",
1471                                         req->r2t_length, req->is_unsolicited_data);
1472                                 mark_conn_closed(req->conn);
1473                                 goto out_put;
1474                         }
1475                         req->outstanding_r2t--;
1476                 } else
1477                         goto out_put;
1478         }
1479
1480         if (req->r2t_length != 0) {
1481                 if (!req->is_unsolicited_data)
1482                         send_r2t(req);
1483         } else
1484                 iscsi_restart_waiting_cmnd(req);
1485
1486 out_put:
1487         cmnd_put(cmnd);
1488         return;
1489 }
1490
1491 static void __cmnd_abort(struct iscsi_cmnd *cmnd)
1492 {
1493         TRACE_MGMT_DBG("Aborting cmd %p, scst_cmd %p (scst state %x, "
1494                 "ref_cnt %d, itt %x, sn %u, op %x, r2t_len %x, CDB op %x, "
1495                 "size to write %u, is_unsolicited_data %d, "
1496                 "outstanding_r2t %d, data_waiting %d, sess->exp_cmd_sn %u, "
1497                 "conn %p, rd_task %p)", cmnd, cmnd->scst_cmd, cmnd->scst_state,
1498                 atomic_read(&cmnd->ref_cnt), cmnd_itt(cmnd), cmnd->pdu.bhs.sn,
1499                 cmnd_opcode(cmnd), cmnd->r2t_length, cmnd_scsicode(cmnd),
1500                 cmnd_write_size(cmnd), cmnd->is_unsolicited_data,
1501                 cmnd->outstanding_r2t, cmnd->data_waiting,
1502                 cmnd->conn->session->exp_cmd_sn, cmnd->conn,
1503                 cmnd->conn->rd_task);
1504
1505 #ifdef NET_PAGE_CALLBACKS_DEFINED
1506         TRACE_MGMT_DBG("net_ref_cnt %d", atomic_read(&cmnd->net_ref_cnt));
1507 #endif
1508
1509         cmnd->tmfabort = 1;
1510
1511         return;
1512 }
1513
1514 /* Must be called from the read thread */
1515 static int cmnd_abort(struct iscsi_cmnd *req)
1516 {
1517         struct iscsi_session *session = req->conn->session;
1518         struct iscsi_task_mgt_hdr *req_hdr =
1519                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1520         struct iscsi_cmnd *cmnd;
1521         int err;
1522
1523         req_hdr->ref_cmd_sn = be32_to_cpu(req_hdr->ref_cmd_sn);
1524
1525         if (after(req_hdr->ref_cmd_sn, req_hdr->cmd_sn)) {
1526                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) > CmdSN(%u)",
1527                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn);
1528                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1529                 goto out;
1530         }
1531
1532         if ((cmnd = cmnd_find_hash_get(session, req_hdr->rtt, ISCSI_RESERVED_TAG))) {
1533                 struct iscsi_conn *conn = cmnd->conn;
1534                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
1535
1536                 if (req_hdr->lun != hdr->lun) {
1537                          PRINT_ERROR("ABORT TASK: LUN mismatch: req LUN "
1538                                 "%Lx, cmd LUN %Lx, rtt %u", req_hdr->lun,
1539                                 hdr->lun, req_hdr->rtt);
1540                         err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1541                         goto out_put;
1542                 }
1543
1544                 if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
1545                         if (req_hdr->ref_cmd_sn != req_hdr->cmd_sn) {
1546                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != TM cmd "
1547                                         "CmdSN(%u) for immediate command %p",
1548                                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn,
1549                                         cmnd);
1550                                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1551                                 goto out_put;
1552                         }
1553                 } else {
1554                         if (req_hdr->ref_cmd_sn != hdr->cmd_sn) {
1555                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != "
1556                                         "CmdSN(%u) for command %p",
1557                                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn,
1558                                         cmnd);
1559                                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1560                                 goto out_put;
1561                         }
1562                 }
1563
1564                 if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
1565                     (req_hdr->cmd_sn == hdr->cmd_sn)) {
1566                         PRINT_ERROR("ABORT TASK: SN mismatch: req SN %x, "
1567                                 "cmd SN %x, rtt %u", req_hdr->cmd_sn,
1568                                 hdr->cmd_sn, req_hdr->rtt);
1569                         err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1570                         goto out_put;
1571                 }
1572
1573                 spin_lock_bh(&conn->cmd_list_lock);
1574                 __cmnd_abort(cmnd);
1575                 spin_unlock_bh(&conn->cmd_list_lock);
1576
1577                 cmnd_put(cmnd);
1578                 err = 0;
1579         } else
1580                 err = ISCSI_RESPONSE_UNKNOWN_TASK;
1581
1582 out:
1583         return err;
1584
1585 out_put:
1586         cmnd_put(cmnd);
1587         goto out;
1588 }
1589
1590 /* Must be called from the read thread */
1591 static int target_abort(struct iscsi_cmnd *req, int all)
1592 {
1593         struct iscsi_target *target = req->conn->session->target;
1594         struct iscsi_task_mgt_hdr *req_hdr =
1595                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1596         struct iscsi_session *session;
1597         struct iscsi_conn *conn;
1598         struct iscsi_cmnd *cmnd;
1599
1600         mutex_lock(&target->target_mutex);
1601
1602         list_for_each_entry(session, &target->session_list, session_list_entry) {
1603                 list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
1604                         spin_lock_bh(&conn->cmd_list_lock);
1605                         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1606                                 if (cmnd == req)
1607                                         continue;
1608                                 if (all)
1609                                         __cmnd_abort(cmnd);
1610                                 else if (req_hdr->lun == cmnd_hdr(cmnd)->lun)
1611                                         __cmnd_abort(cmnd);
1612                         }
1613                         spin_unlock_bh(&conn->cmd_list_lock);
1614                 }
1615         }
1616
1617         mutex_unlock(&target->target_mutex);
1618         return 0;
1619 }
1620
1621 /* Must be called from the read thread */
1622 static void task_set_abort(struct iscsi_cmnd *req)
1623 {
1624         struct iscsi_session *session = req->conn->session;
1625         struct iscsi_task_mgt_hdr *req_hdr =
1626                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1627         struct iscsi_target *target = session->target;
1628         struct iscsi_conn *conn;
1629         struct iscsi_cmnd *cmnd;
1630
1631         mutex_lock(&target->target_mutex);
1632
1633         list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
1634                 spin_lock_bh(&conn->cmd_list_lock);
1635                 list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1636                         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
1637                         if (cmnd == req)
1638                                 continue;
1639                         if (req_hdr->lun != hdr->lun)
1640                                 continue;
1641                         if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
1642                             req_hdr->cmd_sn == hdr->cmd_sn)
1643                                 continue;
1644                         __cmnd_abort(cmnd);
1645                 }
1646                 spin_unlock_bh(&conn->cmd_list_lock);
1647         }
1648
1649         mutex_unlock(&target->target_mutex);
1650         return;
1651 }
1652
1653 /* Must be called from the read thread */
1654 void conn_abort(struct iscsi_conn *conn)
1655 {
1656         struct iscsi_cmnd *cmnd;
1657
1658         TRACE_MGMT_DBG("Aborting conn %p", conn);
1659
1660         iscsi_extracheck_is_rd_thread(conn);
1661
1662         spin_lock_bh(&conn->cmd_list_lock);
1663 again:
1664         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1665                 __cmnd_abort(cmnd);
1666                 if (cmnd->data_waiting) {
1667                         if (!cmnd_get_check(cmnd)) {
1668                                 spin_unlock_bh(&conn->cmd_list_lock);
1669
1670                                 /* ToDo: this is racy for MC/S */
1671                                 TRACE_MGMT_DBG("Restarting data waiting cmd %p",
1672                                         cmnd);
1673                                 iscsi_fail_waiting_cmnd(cmnd);
1674
1675                                 cmnd_put(cmnd);
1676
1677                                 /*
1678                                  * We are in the read thread, so we may not
1679                                  * worry that after cmnd release conn gets
1680                                  * released as well.
1681                                  */
1682                                 spin_lock_bh(&conn->cmd_list_lock);
1683                                 goto again;
1684                         }
1685                 }
1686         }
1687         spin_unlock_bh(&conn->cmd_list_lock);
1688
1689         return;
1690 }
1691
1692 static void execute_task_management(struct iscsi_cmnd *req)
1693 {
1694         struct iscsi_conn *conn = req->conn;
1695         struct iscsi_session *sess = conn->session;
1696         struct iscsi_task_mgt_hdr *req_hdr =
1697                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1698         int err = 0, function = req_hdr->function & ISCSI_FUNCTION_MASK;
1699         struct scst_rx_mgmt_params params;
1700
1701         TRACE((function == ISCSI_FUNCTION_ABORT_TASK) ? TRACE_MGMT_MINOR : TRACE_MGMT,
1702                 "TM fn %d", function);
1703
1704         TRACE_MGMT_DBG("TM req %p, itt %x, rtt %x, sn %u, con %p", req,
1705                 cmnd_itt(req), req_hdr->rtt, req_hdr->cmd_sn, conn);
1706
1707         iscsi_extracheck_is_rd_thread(conn);
1708
1709         spin_lock(&sess->sn_lock);
1710         sess->tm_active = 1;
1711         sess->tm_sn = req_hdr->cmd_sn;
1712         if (sess->tm_rsp != NULL) {
1713                 struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
1714                 TRACE(TRACE_MGMT_MINOR, "Dropping delayed TM rsp %p", tm_rsp);
1715                 sess->tm_rsp = NULL;
1716                 spin_unlock(&sess->sn_lock);
1717                 rsp_cmnd_release(tm_rsp);
1718         } else
1719                 spin_unlock(&sess->sn_lock);
1720
1721         memset(&params, 0, sizeof(params));
1722         params.atomic = SCST_NON_ATOMIC;
1723         params.tgt_priv = req;
1724
1725         if ((function != ISCSI_FUNCTION_ABORT_TASK) &&
1726             (req_hdr->rtt != ISCSI_RESERVED_TAG)) {
1727                 PRINT_ERROR("Invalid RTT %x (TM fn %x)", req_hdr->rtt,
1728                         function);
1729                 err = -1;
1730                 goto reject;
1731         }
1732
1733         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
1734
1735         switch (function) {
1736         case ISCSI_FUNCTION_ABORT_TASK:
1737                 err = cmnd_abort(req);
1738                 if (err == 0) {
1739                         params.fn = SCST_ABORT_TASK;
1740                         params.tag = req_hdr->rtt;
1741                         params.tag_set = 1;
1742                         params.lun = (uint8_t *)&req_hdr->lun;
1743                         params.lun_len = sizeof(req_hdr->lun);
1744                         params.lun_set = 1;
1745                         params.cmd_sn = req_hdr->cmd_sn;
1746                         params.cmd_sn_set = 1;
1747                         err = scst_rx_mgmt_fn(conn->session->scst_sess,
1748                                 &params);
1749                 }
1750                 break;
1751         case ISCSI_FUNCTION_ABORT_TASK_SET:
1752                 task_set_abort(req);
1753                 params.fn = SCST_ABORT_TASK_SET;
1754                 params.lun = (uint8_t *)&req_hdr->lun;
1755                 params.lun_len = sizeof(req_hdr->lun);
1756                 params.lun_set = 1;
1757                 params.cmd_sn = req_hdr->cmd_sn;
1758                 params.cmd_sn_set = 1;
1759                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1760                         &params);
1761                 break;
1762         case ISCSI_FUNCTION_CLEAR_TASK_SET:
1763                 task_set_abort(req);
1764                 params.fn = SCST_CLEAR_TASK_SET;
1765                 params.lun = (uint8_t *)&req_hdr->lun;
1766                 params.lun_len = sizeof(req_hdr->lun);
1767                 params.lun_set = 1;
1768                 params.cmd_sn = req_hdr->cmd_sn;
1769                 params.cmd_sn_set = 1;
1770                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1771                         &params);
1772                 break;
1773         case ISCSI_FUNCTION_CLEAR_ACA:
1774                 params.fn = SCST_CLEAR_ACA;
1775                 params.lun = (uint8_t *)&req_hdr->lun;
1776                 params.lun_len = sizeof(req_hdr->lun);
1777                 params.lun_set = 1;
1778                 params.cmd_sn = req_hdr->cmd_sn;
1779                 params.cmd_sn_set = 1;
1780                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1781                         &params);
1782                 break;
1783         case ISCSI_FUNCTION_TARGET_COLD_RESET:
1784         case ISCSI_FUNCTION_TARGET_WARM_RESET:
1785                 target_abort(req, 1);
1786                 params.fn = SCST_TARGET_RESET;
1787                 params.cmd_sn = req_hdr->cmd_sn;
1788                 params.cmd_sn_set = 1;
1789                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1790                         &params);
1791                 break;
1792         case ISCSI_FUNCTION_LOGICAL_UNIT_RESET:
1793                 target_abort(req, 0);
1794                 params.fn = SCST_LUN_RESET;
1795                 params.lun = (uint8_t *)&req_hdr->lun;
1796                 params.lun_len = sizeof(req_hdr->lun);
1797                 params.lun_set = 1;
1798                 params.cmd_sn = req_hdr->cmd_sn;
1799                 params.cmd_sn_set = 1;
1800                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1801                         &params);
1802                 break;
1803         case ISCSI_FUNCTION_TASK_REASSIGN:
1804                 iscsi_send_task_mgmt_resp(req, 
1805                         ISCSI_RESPONSE_FUNCTION_UNSUPPORTED);
1806                 break;
1807         default:
1808                 iscsi_send_task_mgmt_resp(req,
1809                         ISCSI_RESPONSE_FUNCTION_REJECTED);
1810                 break;
1811         }
1812
1813 reject:
1814         if (err != 0) {
1815                 iscsi_send_task_mgmt_resp(req,
1816                         ISCSI_RESPONSE_FUNCTION_REJECTED);
1817         }
1818 }
1819
1820 static void noop_out_exec(struct iscsi_cmnd *req)
1821 {
1822         struct iscsi_cmnd *rsp;
1823         struct iscsi_nop_in_hdr *rsp_hdr;
1824
1825         TRACE_DBG("%p", req);
1826
1827         if (cmnd_itt(req) != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1828                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
1829
1830                 rsp_hdr = (struct iscsi_nop_in_hdr *)&rsp->pdu.bhs;
1831                 rsp_hdr->opcode = ISCSI_OP_NOOP_IN;
1832                 rsp_hdr->flags = ISCSI_FLG_FINAL;
1833                 rsp_hdr->itt = req->pdu.bhs.itt;
1834                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
1835
1836                 if (req->pdu.datasize)
1837                         sBUG_ON(req->sg == NULL);
1838                 else
1839                         sBUG_ON(req->sg != NULL);
1840
1841                 if (req->sg) {
1842                         rsp->sg = req->sg;
1843                         rsp->bufflen = req->bufflen;
1844                 }
1845
1846                 sBUG_ON(get_pgcnt(req->pdu.datasize, 0) > ISCSI_CONN_IOV_MAX);
1847                 rsp->pdu.datasize = req->pdu.datasize;
1848                 iscsi_cmnd_init_write(rsp,
1849                         ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
1850                 req_cmnd_release(req);
1851         } else
1852                 cmnd_put(req);
1853 }
1854
1855 static void logout_exec(struct iscsi_cmnd *req)
1856 {
1857         struct iscsi_logout_req_hdr *req_hdr;
1858         struct iscsi_cmnd *rsp;
1859         struct iscsi_logout_rsp_hdr *rsp_hdr;
1860
1861         PRINT_INFO("Logout received from initiator %s",
1862                 req->conn->session->initiator_name);
1863         TRACE_DBG("%p", req);
1864
1865         req_hdr = (struct iscsi_logout_req_hdr *)&req->pdu.bhs;
1866         rsp = iscsi_cmnd_create_rsp_cmnd(req);
1867         rsp_hdr = (struct iscsi_logout_rsp_hdr *)&rsp->pdu.bhs;
1868         rsp_hdr->opcode = ISCSI_OP_LOGOUT_RSP;
1869         rsp_hdr->flags = ISCSI_FLG_FINAL;
1870         rsp_hdr->itt = req_hdr->itt;
1871         rsp->should_close_conn = 1;
1872         iscsi_cmnd_init_write(rsp,
1873                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
1874         req_cmnd_release(req);
1875 }
1876
1877 static void iscsi_cmnd_exec(struct iscsi_cmnd *cmnd)
1878 {
1879         TRACE_DBG("%p,%x,%u", cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn);
1880
1881         if (unlikely(cmnd->tmfabort)) {
1882                 TRACE_MGMT_DBG("cmnd %p (scst_cmd %p) aborted", cmnd,
1883                         cmnd->scst_cmd);
1884                 req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
1885                 goto out;
1886         }
1887
1888         iscsi_extracheck_is_rd_thread(cmnd->conn);
1889
1890         switch (cmnd_opcode(cmnd)) {
1891         case ISCSI_OP_SCSI_CMD:
1892                 if (cmnd->r2t_length != 0) {
1893                         if (!cmnd->is_unsolicited_data) {
1894                                 send_r2t(cmnd);
1895                                 break;
1896                         }
1897                 } else
1898                         iscsi_restart_cmnd(cmnd);
1899                 break;
1900         case ISCSI_OP_NOOP_OUT:
1901                 noop_out_exec(cmnd);
1902                 break;
1903         case ISCSI_OP_SCSI_TASK_MGT_MSG:
1904                 execute_task_management(cmnd);
1905                 break;
1906         case ISCSI_OP_LOGOUT_CMD:
1907                 logout_exec(cmnd);
1908                 break;
1909         case ISCSI_OP_SCSI_REJECT:
1910         {
1911                 struct iscsi_cmnd *rsp = get_rsp_cmnd(cmnd);
1912                 TRACE_MGMT_DBG("REJECT cmnd %p (scst_cmd %p), rsp %p", cmnd,
1913                         cmnd->scst_cmd, rsp);
1914                 if (rsp != NULL)
1915                         iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH |
1916                                                          ISCSI_INIT_WRITE_WAKE);
1917                 req_cmnd_release(cmnd);
1918                 break;
1919         }
1920         default:
1921                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
1922                 req_cmnd_release(cmnd);
1923                 break;
1924         }
1925 out:
1926         return;
1927 }
1928
1929 static void __cmnd_send_pdu(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd,
1930         u32 offset, u32 size)
1931 {
1932         TRACE_DBG("%p %u,%u,%u", cmnd, offset, size, cmnd->bufflen);
1933
1934         iscsi_extracheck_is_wr_thread(conn);
1935
1936         sBUG_ON(offset > cmnd->bufflen);
1937         sBUG_ON(offset + size > cmnd->bufflen);
1938
1939         conn->write_offset = offset;
1940         conn->write_size += size;
1941 }
1942
1943 static void cmnd_send_pdu(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
1944 {
1945         u32 size;
1946
1947         if (!cmnd->pdu.datasize)
1948                 return;
1949
1950         size = (cmnd->pdu.datasize + 3) & -4;
1951         sBUG_ON(cmnd->sg == NULL);
1952         sBUG_ON(cmnd->bufflen != size);
1953         __cmnd_send_pdu(conn, cmnd, 0, size);
1954 }
1955
1956 static void set_cork(struct socket *sock, int on)
1957 {
1958         int opt = on;
1959         mm_segment_t oldfs;
1960
1961         oldfs = get_fs();
1962         set_fs(get_ds());
1963         sock->ops->setsockopt(sock, SOL_TCP, TCP_CORK, (void *)&opt, sizeof(opt));
1964         set_fs(oldfs);
1965 }
1966
1967 void cmnd_tx_start(struct iscsi_cmnd *cmnd)
1968 {
1969         struct iscsi_conn *conn = cmnd->conn;
1970
1971         TRACE_DBG("%p:%p:%x", conn, cmnd, cmnd_opcode(cmnd));
1972         iscsi_cmnd_set_length(&cmnd->pdu);
1973
1974         iscsi_extracheck_is_wr_thread(conn);
1975
1976         set_cork(conn->sock, 1);
1977
1978         conn->write_iop = conn->write_iov;
1979         conn->write_iop->iov_base = &cmnd->pdu.bhs;
1980         conn->write_iop->iov_len = sizeof(cmnd->pdu.bhs);
1981         conn->write_iop_used = 1;
1982         conn->write_size = sizeof(cmnd->pdu.bhs);
1983
1984         switch (cmnd_opcode(cmnd)) {
1985         case ISCSI_OP_NOOP_IN:
1986                 cmnd_set_sn(cmnd, 1);
1987                 cmnd_send_pdu(conn, cmnd);
1988                 break;
1989         case ISCSI_OP_SCSI_RSP:
1990                 cmnd_set_sn(cmnd, 1);
1991                 cmnd_send_pdu(conn, cmnd);
1992                 break;
1993         case ISCSI_OP_SCSI_TASK_MGT_RSP:
1994                 cmnd_set_sn(cmnd, 1);
1995                 break;
1996         case ISCSI_OP_TEXT_RSP:
1997                 cmnd_set_sn(cmnd, 1);
1998                 break;
1999         case ISCSI_OP_SCSI_DATA_IN:
2000         {
2001                 struct iscsi_data_in_hdr *rsp = (struct iscsi_data_in_hdr *)&cmnd->pdu.bhs;
2002                 u32 offset = cpu_to_be32(rsp->buffer_offset);
2003
2004                 cmnd_set_sn(cmnd, (rsp->flags & ISCSI_FLG_FINAL) ? 1 : 0);
2005                 __cmnd_send_pdu(conn, cmnd, offset, cmnd->pdu.datasize);
2006                 break;
2007         }
2008         case ISCSI_OP_LOGOUT_RSP:
2009                 cmnd_set_sn(cmnd, 1);
2010                 break;
2011         case ISCSI_OP_R2T:
2012                 cmnd->pdu.bhs.sn = cmnd_set_sn(cmnd, 0);
2013                 break;
2014         case ISCSI_OP_ASYNC_MSG:
2015                 cmnd_set_sn(cmnd, 1);
2016                 break;
2017         case ISCSI_OP_REJECT:
2018                 cmnd_set_sn(cmnd, 1);
2019                 cmnd_send_pdu(conn, cmnd);
2020                 break;
2021         default:
2022                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2023                 break;
2024         }
2025
2026         // move this?
2027         conn->write_size = (conn->write_size + 3) & -4;
2028         iscsi_dump_pdu(&cmnd->pdu);
2029 }
2030
2031 void cmnd_tx_end(struct iscsi_cmnd *cmnd)
2032 {
2033         struct iscsi_conn *conn = cmnd->conn;
2034
2035         TRACE_DBG("%p:%x (should_close_conn %d)", cmnd, cmnd_opcode(cmnd),
2036                 cmnd->should_close_conn);
2037
2038         switch (cmnd_opcode(cmnd)) {
2039         case ISCSI_OP_NOOP_IN:
2040         case ISCSI_OP_SCSI_RSP:
2041         case ISCSI_OP_SCSI_TASK_MGT_RSP:
2042         case ISCSI_OP_TEXT_RSP:
2043         case ISCSI_OP_R2T:
2044         case ISCSI_OP_ASYNC_MSG:
2045         case ISCSI_OP_REJECT:
2046         case ISCSI_OP_SCSI_DATA_IN:
2047         case ISCSI_OP_LOGOUT_RSP:
2048                 break;
2049         default:
2050                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2051                 sBUG();
2052                 break;
2053         }
2054
2055         if (cmnd->should_close_conn) {
2056                 PRINT_INFO("Closing connection at initiator %s request",
2057                         conn->session->initiator_name);
2058                 mark_conn_closed(conn);
2059         }
2060
2061         set_cork(cmnd->conn->sock, 0);
2062 }
2063
2064 /*
2065  * Push the command for execution. This functions reorders the commands.
2066  * Called from the read thread.
2067  */
2068 static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd)
2069 {
2070         struct iscsi_session *session = cmnd->conn->session;
2071         struct list_head *entry;
2072         u32 cmd_sn;
2073
2074         TRACE_DBG("%p:%x %u,%u",
2075                 cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn, session->exp_cmd_sn);
2076
2077         iscsi_extracheck_is_rd_thread(cmnd->conn);
2078
2079         sBUG_ON(cmnd->parent_req != NULL);
2080
2081         if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
2082                 TRACE_DBG("Immediate cmd %p (cmd_sn %u)", cmnd,
2083                         cmnd->pdu.bhs.sn);
2084                 iscsi_cmnd_exec(cmnd);
2085                 goto out;
2086         }
2087
2088         spin_lock(&session->sn_lock);
2089
2090         cmd_sn = cmnd->pdu.bhs.sn;
2091         if (cmd_sn == session->exp_cmd_sn) {
2092                 while (1) {
2093                         session->exp_cmd_sn = ++cmd_sn;
2094
2095                         if (unlikely(session->tm_active)) {
2096                                 if (before(cmd_sn, session->tm_sn)) {
2097                                         struct iscsi_conn *conn = cmnd->conn;
2098
2099                                         spin_unlock(&session->sn_lock);
2100
2101                                         spin_lock_bh(&conn->cmd_list_lock);
2102                                         __cmnd_abort(cmnd);
2103                                         spin_unlock_bh(&conn->cmd_list_lock);
2104
2105                                         spin_lock(&session->sn_lock);
2106                                 }
2107                                 iscsi_check_send_delayed_tm_resp(session);
2108                         }
2109
2110                         spin_unlock(&session->sn_lock);
2111
2112                         iscsi_cmnd_exec(cmnd);
2113
2114                         if (list_empty(&session->pending_list))
2115                                 break;
2116                         cmnd = list_entry(session->pending_list.next, struct iscsi_cmnd,
2117                                                 pending_list_entry);
2118                         if (cmnd->pdu.bhs.sn != cmd_sn)
2119                                 break;
2120
2121                         list_del(&cmnd->pending_list_entry);
2122                         cmnd->pending = 0;
2123
2124                         TRACE_DBG("Processing pending cmd %p (cmd_sn %u)",
2125                                 cmnd, cmd_sn);
2126
2127                         spin_lock(&session->sn_lock);
2128                 }
2129         } else {
2130                 int drop = 0;
2131
2132                 TRACE_DBG("Pending cmd %p (cmd_sn %u, exp_cmd_sn %u)",
2133                         cmnd, cmd_sn, session->exp_cmd_sn);
2134
2135                 /*
2136                  * iSCSI RFC 3720: "The target MUST silently ignore any
2137                  * non-immediate command outside of [from ExpCmdSN to MaxCmdSN
2138                  * inclusive] range". But we won't honor the MaxCmdSN
2139                  * requirement, because, since we adjust MaxCmdSN from the
2140                  * separate write thread, rarery it is possible that initiator
2141                  * can legally send command with CmdSN>MaxSN. But it won't
2142                  * hurt anything, in the worst case it will lead to
2143                  * additional QUEUE FULL status.
2144                  */
2145
2146                 if (unlikely(before(cmd_sn, session->exp_cmd_sn))) {
2147                         PRINT_ERROR("Unexpected cmd_sn (%u,%u)", cmd_sn,
2148                                 session->exp_cmd_sn);
2149                         drop = 1;
2150                 }
2151
2152 #if 0
2153                 if (unlikely(after(cmd_sn, session->exp_cmd_sn +
2154                                         iscsi_get_allowed_cmds(session)))) {
2155                         TRACE_MGMT_DBG("Too large cmd_sn %u (exp_cmd_sn %u, "
2156                                 "max_sn %u)", cmd_sn, session->exp_cmd_sn,
2157                                 iscsi_get_allowed_cmds(session));
2158                 }
2159 #endif
2160
2161                 spin_unlock(&session->sn_lock);
2162
2163                 if (unlikely(drop)) {
2164                         req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
2165                         goto out;
2166                 }
2167
2168                 if (unlikely(cmnd->tmfabort)) {
2169                         struct iscsi_cmnd *tm_clone;
2170
2171                         TRACE_MGMT_DBG("Pending aborted cmnd %p, creating TM "
2172                                 "clone (scst cmd %p, state %d)", cmnd,
2173                                 cmnd->scst_cmd, cmnd->scst_state);
2174
2175                         tm_clone = cmnd_alloc(cmnd->conn, NULL);
2176                         if (tm_clone != NULL) {
2177                                 tm_clone->tmfabort = 1;
2178                                 tm_clone->pdu = cmnd->pdu;
2179
2180                                 TRACE_MGMT_DBG("TM clone %p created", tm_clone);
2181
2182                                 iscsi_cmnd_exec(cmnd);
2183                                 cmnd = tm_clone;
2184                         } else
2185                                 PRINT_ERROR("%s", "Unable to create TM clone");
2186                 }
2187
2188                 list_for_each(entry, &session->pending_list) {
2189                         struct iscsi_cmnd *tmp = list_entry(entry, struct iscsi_cmnd,
2190                                                         pending_list_entry);
2191                         if (before(cmd_sn, tmp->pdu.bhs.sn))
2192                                 break;
2193                 }
2194
2195                 list_add_tail(&cmnd->pending_list_entry, entry);
2196                 cmnd->pending = 1;
2197         }
2198 out:
2199         return;
2200 }
2201
2202 static int check_segment_length(struct iscsi_cmnd *cmnd)
2203 {
2204         struct iscsi_conn *conn = cmnd->conn;
2205         struct iscsi_session *session = conn->session;
2206
2207         if (unlikely(cmnd->pdu.datasize > session->sess_param.max_recv_data_length)) {
2208                 PRINT_ERROR("Initiator %s violated negotiated parameters: "
2209                         "data too long (ITT %x, datasize %u, "
2210                         "max_recv_data_length %u", session->initiator_name,
2211                         cmnd_itt(cmnd), cmnd->pdu.datasize,
2212                         session->sess_param.max_recv_data_length);
2213                 mark_conn_closed(conn);
2214                 return -EINVAL;
2215         }
2216         return 0;
2217 }
2218
2219 int cmnd_rx_start(struct iscsi_cmnd *cmnd)
2220 {
2221         struct iscsi_conn *conn = cmnd->conn;
2222         int res, rc;
2223
2224         iscsi_dump_pdu(&cmnd->pdu);
2225
2226         res = check_segment_length(cmnd);
2227         if (res != 0)
2228                 goto out;
2229
2230         switch (cmnd_opcode(cmnd)) {
2231         case ISCSI_OP_NOOP_OUT:
2232                 rc = noop_out_start(cmnd);
2233                 break;
2234         case ISCSI_OP_SCSI_CMD:
2235                 rc = cmnd_insert_hash(cmnd);
2236                 if (likely(rc == 0)) {
2237                         res = scsi_cmnd_start(cmnd);
2238                         if (unlikely(res != 0))
2239                                 goto out;
2240                 }
2241                 break;
2242         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2243                 rc = cmnd_insert_hash(cmnd);
2244                 break;
2245         case ISCSI_OP_SCSI_DATA_OUT:
2246                 res = data_out_start(conn, cmnd);
2247                 rc = 0; /* to avoid compiler warning */
2248                 if (unlikely(res != 0))
2249                         goto out;
2250                 break;
2251         case ISCSI_OP_LOGOUT_CMD:
2252                 rc = cmnd_insert_hash(cmnd);
2253                 break;
2254         case ISCSI_OP_TEXT_CMD:
2255         case ISCSI_OP_SNACK_CMD:
2256                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2257                 break;
2258         default:
2259                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2260                 break;
2261         }
2262
2263         if (unlikely(rc < 0)) {
2264                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
2265                 PRINT_ERROR("Error %d (iSCSI opcode %x, ITT %x, op %x)", rc,
2266                         cmnd_opcode(cmnd), cmnd_itt(cmnd),
2267                         (cmnd_opcode(cmnd) == ISCSI_OP_SCSI_CMD ?
2268                                 hdr->scb[0] : -1));
2269                 iscsi_cmnd_reject(cmnd, -rc);
2270         }
2271
2272 out:
2273         TRACE_EXIT_RES(res);
2274         return res;
2275 }
2276
2277 void cmnd_rx_end(struct iscsi_cmnd *cmnd)
2278 {
2279         TRACE_ENTRY();
2280
2281         TRACE_DBG("%p:%x", cmnd, cmnd_opcode(cmnd));
2282
2283         switch (cmnd_opcode(cmnd)) {
2284         case ISCSI_OP_SCSI_CMD:
2285         case ISCSI_OP_SCSI_REJECT:
2286         case ISCSI_OP_NOOP_OUT:
2287         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2288         case ISCSI_OP_LOGOUT_CMD:
2289                 iscsi_session_push_cmnd(cmnd);
2290                 break;
2291         case ISCSI_OP_SCSI_DATA_OUT:
2292                 data_out_end(cmnd);
2293                 break;
2294         case ISCSI_OP_PDU_REJECT:
2295         {
2296                 struct iscsi_cmnd *rsp = get_rsp_cmnd(cmnd);
2297                 if (rsp != NULL)
2298                         iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH |
2299                                                         ISCSI_INIT_WRITE_WAKE);
2300                 req_cmnd_release(cmnd);
2301                 break;
2302         }
2303         case ISCSI_OP_DATA_REJECT:
2304                 req_cmnd_release(cmnd);
2305                 break;
2306         default:
2307                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2308                 req_cmnd_release(cmnd);
2309                 break;
2310         }
2311
2312         TRACE_EXIT();
2313         return;
2314 }
2315
2316 #ifndef NET_PAGE_CALLBACKS_DEFINED
2317 static int iscsi_alloc_data_buf(struct scst_cmd *cmd)
2318 {
2319         if (scst_cmd_get_data_direction(cmd) == SCST_DATA_READ) {
2320                 /*
2321                  * sock->ops->sendpage() is async zero copy operation,
2322                  * so we must be sure not to free and reuse
2323                  * the command's buffer before the sending was completed
2324                  * by the network layers. It is possible only if we
2325                  * don't use SGV cache.
2326                  */
2327                 scst_cmd_set_no_sgv(cmd);
2328         }
2329         return 1;
2330 }
2331 #endif
2332
2333 static inline void iscsi_set_state_wake_up(struct iscsi_cmnd *req,
2334         int new_state)
2335 {
2336         /*
2337          * We use wait_event() to wait for the state change, but it checks its
2338          * condition without any protection, so without cmnd_get() it is
2339          * possible that req will die "immediately" after the state assignment
2340          * and wake_up() will operate on dead data.
2341          */
2342         cmnd_get_ordered(req);
2343         req->scst_state = new_state;
2344         wake_up(&req->scst_waitQ);
2345         cmnd_put(req);
2346         return;
2347 }
2348
2349 static void iscsi_preprocessing_done(struct scst_cmd *scst_cmd)
2350 {
2351         struct iscsi_cmnd *req = (struct iscsi_cmnd*)
2352                                 scst_cmd_get_tgt_priv(scst_cmd);
2353
2354         TRACE_DBG("req %p", req);
2355
2356         EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_RX_CMD);
2357
2358         iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_AFTER_PREPROC);
2359         return;
2360 }
2361
2362 /* 
2363  * No locks.
2364  *
2365  * IMPORTANT! Connection conn must be protected by additional conn_get()
2366  * upon entrance in this function, because otherwise it could be destroyed
2367  * inside as a result of iscsi_send(), which releases sent commands.
2368  */
2369 static void iscsi_try_local_processing(struct iscsi_conn *conn)
2370 {
2371         int local;
2372
2373         TRACE_ENTRY();
2374
2375         spin_lock_bh(&iscsi_wr_lock);
2376         switch(conn->wr_state) {
2377         case ISCSI_CONN_WR_STATE_IN_LIST:
2378                 list_del(&conn->wr_list_entry);
2379                 /* go through */
2380         case ISCSI_CONN_WR_STATE_IDLE:
2381 #ifdef EXTRACHECKS
2382                 conn->wr_task = current;
2383 #endif
2384                 conn->wr_state = ISCSI_CONN_WR_STATE_PROCESSING;
2385                 conn->wr_space_ready = 0;
2386                 local = 1;
2387                 break;
2388         default:
2389                 local = 0;
2390                 break;
2391         }
2392         spin_unlock_bh(&iscsi_wr_lock);
2393
2394         if (local) {
2395                 int rc = 1;
2396                 while(test_write_ready(conn)) {
2397                         rc = iscsi_send(conn);
2398                         if (rc <= 0) {
2399                                 break;
2400                         }
2401                 }
2402
2403                 spin_lock_bh(&iscsi_wr_lock);
2404 #ifdef EXTRACHECKS
2405                 conn->wr_task = NULL;
2406 #endif
2407                 if ((rc <= 0) || test_write_ready(conn)) {
2408                         list_add_tail(&conn->wr_list_entry, &iscsi_wr_list);
2409                         conn->wr_state = ISCSI_CONN_WR_STATE_IN_LIST;
2410                         wake_up(&iscsi_wr_waitQ);
2411                 } else
2412                         conn->wr_state = ISCSI_CONN_WR_STATE_IDLE;
2413                 spin_unlock_bh(&iscsi_wr_lock);
2414         }
2415
2416         TRACE_EXIT();
2417         return;
2418 }
2419
2420 static int iscsi_xmit_response(struct scst_cmd *scst_cmd)
2421 {
2422         int resp_flags = scst_cmd_get_tgt_resp_flags(scst_cmd);
2423         struct iscsi_cmnd *req = (struct iscsi_cmnd*)
2424                                         scst_cmd_get_tgt_priv(scst_cmd);
2425         struct iscsi_conn *conn = req->conn;
2426         int status = scst_cmd_get_status(scst_cmd);
2427         u8 *sense = scst_cmd_get_sense_buffer(scst_cmd);
2428         int sense_len = scst_cmd_get_sense_buffer_len(scst_cmd);
2429         int old_state = req->scst_state;
2430
2431         scst_cmd_set_tgt_priv(scst_cmd, NULL);
2432
2433         req->tmfabort |= scst_cmd_aborted(scst_cmd) ? 1 : 0;
2434         if (unlikely(req->tmfabort)) {
2435                 TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
2436                         req->scst_cmd);
2437                 if (old_state == ISCSI_CMD_STATE_RESTARTED) {
2438                         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
2439                         req_cmnd_release_force(req, ISCSI_FORCE_RELEASE_WRITE);
2440                 } else
2441                         iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_PROCESSED);
2442                 goto out;
2443         }
2444
2445         if (unlikely(old_state != ISCSI_CMD_STATE_RESTARTED)) {
2446                 TRACE_DBG("req %p on %d state", req, old_state);
2447                 create_status_rsp(req, status, sense, sense_len);
2448                 switch(old_state) {
2449                 case ISCSI_CMD_STATE_RX_CMD:
2450                 case ISCSI_CMD_STATE_AFTER_PREPROC:
2451                         break;
2452                 default:
2453                         sBUG();
2454                 }
2455                 iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_PROCESSED);
2456                 goto out;
2457         }
2458
2459         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
2460
2461         req->bufflen = scst_cmd_get_resp_data_len(scst_cmd);
2462         req->sg = scst_cmd_get_sg(scst_cmd);
2463
2464         TRACE_DBG("req %p, resp_flags=%x, req->bufflen=%d, req->sg=%p", req,
2465                 resp_flags, req->bufflen, req->sg);
2466
2467         if (unlikely((req->bufflen != 0) &&
2468                      !(resp_flags & SCST_TSC_FLAG_STATUS))) {
2469                 PRINT_ERROR("%s", "Sending DATA without STATUS is unsupported");
2470                 scst_set_cmd_error(scst_cmd,
2471                         SCST_LOAD_SENSE(scst_sense_hardw_error));
2472                 resp_flags = scst_cmd_get_tgt_resp_flags(scst_cmd);
2473                 sBUG();
2474         }
2475
2476         if (req->bufflen != 0) {
2477                 /* 
2478                  * Check above makes sure that SCST_TSC_FLAG_STATUS is set,
2479                  * so status is valid here, but in future that could change.
2480                  * ToDo
2481                  */
2482                 if (status != SAM_STAT_CHECK_CONDITION) {
2483                         send_data_rsp(req, status,
2484                                 resp_flags & SCST_TSC_FLAG_STATUS);
2485                 } else {
2486                         struct iscsi_cmnd *rsp;
2487                         struct iscsi_scsi_rsp_hdr *rsp_hdr;
2488                         int resid;
2489                         send_data_rsp(req, 0, 0);
2490                         if (resp_flags & SCST_TSC_FLAG_STATUS) {
2491                                 rsp = create_status_rsp(req, status, sense,
2492                                         sense_len);
2493                                 rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
2494                                 resid = cmnd_read_size(req) - req->bufflen;
2495                                 if (resid > 0) {
2496                                         rsp_hdr->flags |=
2497                                                 ISCSI_FLG_RESIDUAL_UNDERFLOW;
2498                                         rsp_hdr->residual_count = cpu_to_be32(resid);
2499                                 } else if (resid < 0) {
2500                                         rsp_hdr->flags |=
2501                                                 ISCSI_FLG_RESIDUAL_OVERFLOW;
2502                                         rsp_hdr->residual_count = cpu_to_be32(-resid);
2503                                 }
2504                                 iscsi_cmnd_init_write(rsp,
2505                                         ISCSI_INIT_WRITE_REMOVE_HASH);
2506                         }
2507                 }
2508         } else if (resp_flags & SCST_TSC_FLAG_STATUS) {
2509                 struct iscsi_cmnd *rsp;
2510                 struct iscsi_scsi_rsp_hdr *rsp_hdr;
2511                 u32 resid;
2512                 rsp = create_status_rsp(req, status, sense, sense_len);
2513                 rsp_hdr = (struct iscsi_scsi_rsp_hdr *) &rsp->pdu.bhs;
2514                 resid = cmnd_read_size(req);
2515                 if (resid != 0) {
2516                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
2517                         rsp_hdr->residual_count = cpu_to_be32(resid);
2518                 }
2519                 iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH);
2520         }
2521 #ifdef EXTRACHECKS
2522         else
2523                 sBUG();
2524 #endif
2525
2526         conn_get_ordered(conn);
2527         req_cmnd_release(req);
2528         iscsi_try_local_processing(conn);
2529         conn_put(conn);
2530
2531 out:
2532         return SCST_TGT_RES_SUCCESS;
2533 }
2534
2535 /* Called under sn_lock */
2536 static bool iscsi_is_delay_tm_resp(struct iscsi_cmnd *rsp)
2537 {
2538         bool res = 0;
2539         struct iscsi_task_mgt_hdr *req_hdr =
2540                 (struct iscsi_task_mgt_hdr *)&rsp->parent_req->pdu.bhs;
2541         int function = req_hdr->function & ISCSI_FUNCTION_MASK;
2542         struct iscsi_session *sess = rsp->conn->session;
2543
2544         TRACE_ENTRY();
2545
2546         /* This should be checked for immediate TM commands as well */
2547
2548         switch(function) {
2549         default:
2550                 if (before(sess->exp_cmd_sn, req_hdr->cmd_sn))
2551                         res = 1;
2552                 break;
2553         }
2554
2555         TRACE_EXIT_RES(res);
2556         return res;
2557 }
2558
2559 /* Called under sn_lock, but might drop it inside, then reaquire */
2560 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess)
2561 {
2562         struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
2563
2564         TRACE_ENTRY();
2565
2566         if (tm_rsp == NULL)
2567                 goto out;
2568
2569         if (iscsi_is_delay_tm_resp(tm_rsp))
2570                 goto out;
2571
2572         TRACE(TRACE_MGMT_MINOR, "Sending delayed rsp %p", tm_rsp);
2573
2574         sess->tm_rsp = NULL;
2575         sess->tm_active = 0;
2576
2577         spin_unlock(&sess->sn_lock);
2578
2579         iscsi_cmnd_init_write(tm_rsp,
2580                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2581
2582         spin_lock(&sess->sn_lock);
2583
2584 out:
2585         TRACE_EXIT();
2586         return;
2587 }
2588
2589 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status)
2590 {
2591         struct iscsi_cmnd *rsp;
2592         struct iscsi_task_mgt_hdr *req_hdr =
2593                                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2594         struct iscsi_task_rsp_hdr *rsp_hdr;
2595         struct iscsi_session *sess = req->conn->session;
2596         int fn = req_hdr->function & ISCSI_FUNCTION_MASK;
2597
2598         TRACE_ENTRY();
2599
2600         TRACE_MGMT_DBG("TM req %p finished", req);
2601         TRACE((req_hdr->function == ISCSI_FUNCTION_ABORT_TASK) ?
2602                          TRACE_MGMT_MINOR : TRACE_MGMT,
2603                 "TM fn %d finished, status %d", fn, status);
2604
2605         rsp = iscsi_cmnd_create_rsp_cmnd(req);
2606         rsp_hdr = (struct iscsi_task_rsp_hdr *)&rsp->pdu.bhs;
2607
2608         rsp_hdr->opcode = ISCSI_OP_SCSI_TASK_MGT_RSP;
2609         rsp_hdr->flags = ISCSI_FLG_FINAL;
2610         rsp_hdr->itt = req_hdr->itt;
2611         rsp_hdr->response = status;
2612
2613         if (fn == ISCSI_FUNCTION_TARGET_COLD_RESET)
2614                 rsp->should_close_conn = 1;
2615
2616         sBUG_ON(sess->tm_rsp != NULL);
2617
2618         spin_lock(&sess->sn_lock);
2619         if (iscsi_is_delay_tm_resp(rsp)) {
2620                 TRACE(TRACE_MGMT_MINOR, "Delaying TM fn %x response %p "
2621                         "(req %p), because not all affected commands received "
2622                         "(TM cmd sn %u, exp sn %u)",
2623                         req_hdr->function & ISCSI_FUNCTION_MASK, rsp, req,
2624                         req_hdr->cmd_sn, sess->exp_cmd_sn);
2625                 sess->tm_rsp = rsp;
2626                 spin_unlock(&sess->sn_lock);
2627                 goto out_release;
2628         }
2629         sess->tm_active = 0;
2630         spin_unlock(&sess->sn_lock);
2631
2632         iscsi_cmnd_init_write(rsp,
2633                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2634
2635 out_release:
2636         req_cmnd_release(req);
2637
2638         TRACE_EXIT();
2639         return;
2640 }
2641
2642 static inline int iscsi_get_mgmt_response(int status)
2643 {
2644         switch(status) {
2645         case SCST_MGMT_STATUS_SUCCESS:
2646                 return ISCSI_RESPONSE_FUNCTION_COMPLETE;
2647
2648         case SCST_MGMT_STATUS_TASK_NOT_EXIST:
2649                 return ISCSI_RESPONSE_UNKNOWN_TASK;
2650
2651         case SCST_MGMT_STATUS_LUN_NOT_EXIST:
2652                 return ISCSI_RESPONSE_UNKNOWN_LUN;
2653
2654         case SCST_MGMT_STATUS_FN_NOT_SUPPORTED:
2655                 return ISCSI_RESPONSE_FUNCTION_UNSUPPORTED;
2656
2657         case SCST_MGMT_STATUS_REJECTED:
2658         case SCST_MGMT_STATUS_FAILED:
2659         default:
2660                 return ISCSI_RESPONSE_FUNCTION_REJECTED;
2661         }
2662 }
2663
2664 static void iscsi_task_mgmt_fn_done(struct scst_mgmt_cmd *scst_mcmd)
2665 {
2666         struct iscsi_cmnd *req = (struct iscsi_cmnd*)
2667                                 scst_mgmt_cmd_get_tgt_priv(scst_mcmd);
2668         int status = iscsi_get_mgmt_response(scst_mgmt_cmd_get_status(scst_mcmd));
2669
2670         TRACE_MGMT_DBG("req %p, scst_mcmd %p, scst status %d", req, scst_mcmd, 
2671                 scst_mgmt_cmd_get_status(scst_mcmd));
2672
2673         iscsi_send_task_mgmt_resp(req, status);
2674
2675         scst_mgmt_cmd_set_tgt_priv(scst_mcmd, NULL);
2676 }
2677
2678 static int iscsi_target_detect(struct scst_tgt_template *templ)
2679 {
2680         /* Nothing to do */
2681         return 0;
2682 }
2683
2684 static int iscsi_target_release(struct scst_tgt *scst_tgt)
2685 {
2686         /* Nothing to do */
2687         return 0;
2688 }
2689
2690 struct scst_tgt_template iscsi_template = {
2691         .name = "iscsi",
2692         .sg_tablesize = ISCSI_CONN_IOV_MAX,
2693         .threads_num = 0,
2694         .no_clustering = 1,
2695         .xmit_response_atomic = 0,
2696         .preprocessing_done_atomic = 1,
2697         .detect = iscsi_target_detect,
2698         .release = iscsi_target_release,
2699         .xmit_response = iscsi_xmit_response,
2700 #ifndef NET_PAGE_CALLBACKS_DEFINED
2701         .alloc_data_buf = iscsi_alloc_data_buf,
2702 #endif
2703         .preprocessing_done = iscsi_preprocessing_done,
2704         .pre_exec = iscsi_pre_exec,
2705         .task_mgmt_fn_done = iscsi_task_mgmt_fn_done,
2706 };
2707
2708 static __init int iscsi_run_threads(int count, char *name, int (*fn)(void *))
2709 {
2710         int res = 0;
2711         int i;
2712         struct iscsi_thread_t *thr;
2713
2714         for (i = 0; i < count; i++) {
2715                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
2716                 if (!thr) {
2717                         res = -ENOMEM;
2718                         PRINT_ERROR("Failed to allocate thr %d", res);
2719                         goto out;
2720                 }
2721                 thr->thr = kthread_run(fn, NULL, "%s%d", name, i);
2722                 if (IS_ERR(thr->thr)) {
2723                         res = PTR_ERR(thr->thr);
2724                         PRINT_ERROR("kthread_create() failed: %d", res);
2725                         kfree(thr);
2726                         goto out;
2727                 }
2728                 list_add(&thr->threads_list_entry, &iscsi_threads_list);
2729         }
2730
2731 out:
2732         return res;
2733 }
2734
2735 static void iscsi_stop_threads(void)
2736 {
2737         struct iscsi_thread_t *t, *tmp;
2738
2739         list_for_each_entry_safe(t, tmp, &iscsi_threads_list,
2740                                 threads_list_entry) {
2741                 int rc = kthread_stop(t->thr);
2742                 if (rc < 0) {
2743                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
2744                 }
2745                 list_del(&t->threads_list_entry);
2746                 kfree(t);
2747         }
2748 }
2749
2750 static int __init iscsi_init(void)
2751 {
2752         int err;
2753         int num;
2754
2755         PRINT_INFO("iSCSI SCST Target - version %s", ISCSI_VERSION_STRING);
2756
2757 #ifdef NET_PAGE_CALLBACKS_DEFINED
2758         err = net_set_get_put_page_callbacks(iscsi_get_page_callback,
2759                         iscsi_put_page_callback);
2760         if (err != 0) {
2761                 PRINT_INFO("Unable to set page callbackes: %d", err);
2762                 goto out;
2763         }
2764 #else
2765         PRINT_INFO("%s", "Patch put_page_callback-<kernel-version>.patch "
2766                 "not applied on your kernel. Running in the performance "
2767                 "degraded mode. Refer README file for details");
2768 #endif
2769
2770         BUILD_BUG_ON(MAX_DATA_SEG_LEN != (ISCSI_CONN_IOV_MAX<<PAGE_SHIFT));
2771
2772         if ((ctr_major = register_chrdev(0, ctr_name, &ctr_fops)) < 0) {
2773                 PRINT_ERROR("failed to register the control device %d", ctr_major);
2774                 err = ctr_major;
2775                 goto out_callb;
2776         }
2777
2778         if ((err = event_init()) < 0)
2779                 goto out_reg;
2780
2781         iscsi_cmnd_cache = KMEM_CACHE(iscsi_cmnd, SCST_SLAB_FLAGS);
2782         if (!iscsi_cmnd_cache) {
2783                 err = -ENOMEM;
2784                 goto out_event;
2785         }
2786
2787         err = scst_register_target_template(&iscsi_template);
2788         if (err < 0)
2789                 goto out_kmem;
2790
2791         iscsi_template_registered = 1;
2792
2793         if ((err = iscsi_procfs_init()) < 0)
2794                 goto out_reg_tmpl;
2795
2796         num = max(num_online_cpus(), 2);
2797
2798         err = iscsi_run_threads(num, "iscsird", istrd);
2799         if (err != 0)
2800                 goto out_thr;
2801
2802         err = iscsi_run_threads(num, "iscsiwr", istwr);
2803         if (err != 0)
2804                 goto out_thr;
2805
2806 out:
2807         return err;
2808
2809 out_thr:
2810         iscsi_procfs_exit();
2811         iscsi_stop_threads();
2812
2813 out_reg_tmpl:
2814         scst_unregister_target_template(&iscsi_template);
2815
2816 out_kmem:
2817         kmem_cache_destroy(iscsi_cmnd_cache);
2818
2819 out_event:
2820         event_exit();
2821
2822 out_reg:
2823         unregister_chrdev(ctr_major, ctr_name);
2824
2825 out_callb:
2826 #ifdef NET_PAGE_CALLBACKS_DEFINED
2827         net_set_get_put_page_callbacks(NULL, NULL);
2828 #endif  
2829         goto out;
2830 }
2831
2832 static void __exit iscsi_exit(void)
2833 {
2834         iscsi_stop_threads();
2835
2836         unregister_chrdev(ctr_major, ctr_name);
2837
2838         iscsi_procfs_exit();
2839         event_exit();
2840
2841         kmem_cache_destroy(iscsi_cmnd_cache);
2842
2843         scst_unregister_target_template(&iscsi_template);
2844
2845 #ifdef NET_PAGE_CALLBACKS_DEFINED
2846         net_set_get_put_page_callbacks(NULL, NULL);
2847 #endif
2848 }
2849
2850 module_init(iscsi_init);
2851 module_exit(iscsi_exit);
2852
2853 MODULE_LICENSE("GPL");