- Important iSCSI stability on connection close fixes.
[mirror/scst/.git] / iscsi-scst / kernel / iscsi.c
1 /*
2  *  Copyright (C) 2002-2003 Ardis Technolgies <roman@ardistech.com>
3  *  Copyright (C) 2007 Vladislav Bolkhovitin
4  *  Copyright (C) 2007 CMS Distribution Limited
5  * 
6  *  This program is free software; you can redistribute it and/or
7  *  modify it under the terms of the GNU General Public License
8  *  as published by the Free Software Foundation, version 2
9  *  of the License.
10  * 
11  *  This program is distributed in the hope that it will be useful,
12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  *  GNU General Public License for more details.
15  */
16
17 #include <linux/module.h>
18 #include <linux/hash.h>
19 #include <linux/kthread.h>
20 #include <net/tcp.h>
21 #include <scsi/scsi.h>
22
23 #include "iscsi.h"
24 #include "digest.h"
25
26 #ifndef NET_PAGE_CALLBACKS_DEFINED
27 #warning Patch put_page_callback-<kernel-version>.patch not applied on your \
28         kernel. ISCSI-SCST will run in the performance degraded mode. Refer \
29         README file for details.
30 #endif
31
32 #define ISCSI_INIT_WRITE_WAKE           0x1
33 #define ISCSI_INIT_WRITE_REMOVE_HASH    0x2
34
35 static int ctr_major;
36 static char ctr_name[] = "iscsi-scst-ctl";
37 static int iscsi_template_registered;
38
39 #if defined(DEBUG) || defined(TRACING)
40 unsigned long iscsi_trace_flag = ISCSI_DEFAULT_LOG_FLAGS;
41 #endif
42
43 static struct kmem_cache *iscsi_cmnd_cache;
44
45 spinlock_t iscsi_rd_lock = SPIN_LOCK_UNLOCKED;
46 LIST_HEAD(iscsi_rd_list);
47 DECLARE_WAIT_QUEUE_HEAD(iscsi_rd_waitQ);
48
49 spinlock_t iscsi_wr_lock = SPIN_LOCK_UNLOCKED;
50 LIST_HEAD(iscsi_wr_list);
51 DECLARE_WAIT_QUEUE_HEAD(iscsi_wr_waitQ);
52
53 static char dummy_data[1024];
54
55 struct iscsi_thread_t {
56         struct task_struct *thr;
57         struct list_head threads_list_entry;
58 };
59
60 static LIST_HEAD(iscsi_threads_list);
61
62 static void cmnd_remove_hash(struct iscsi_cmnd *cmnd);
63 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status);
64 static void cmnd_prepare_skip_pdu(struct iscsi_cmnd *cmnd);
65 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess);
66 static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd);
67
68 static inline u32 cmnd_write_size(struct iscsi_cmnd *cmnd)
69 {
70         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
71
72         if (hdr->flags & ISCSI_CMD_WRITE)
73                 return be32_to_cpu(hdr->data_length);
74         return 0;
75 }
76
77 static inline u32 cmnd_read_size(struct iscsi_cmnd *cmnd)
78 {
79         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
80
81         if (hdr->flags & ISCSI_CMD_READ) {
82                 struct iscsi_rlength_ahdr *ahdr =
83                         (struct iscsi_rlength_ahdr *)cmnd->pdu.ahs;
84
85                 if (!(hdr->flags & ISCSI_CMD_WRITE))
86                         return be32_to_cpu(hdr->data_length);
87                 if (ahdr && ahdr->ahstype == ISCSI_AHSTYPE_RLENGTH)
88                         return be32_to_cpu(ahdr->read_length);
89         }
90         return 0;
91 }
92
93 static inline void iscsi_restart_cmnd(struct iscsi_cmnd *cmnd)
94 {
95         cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
96         scst_restart_cmd(cmnd->scst_cmd, SCST_PREPROCESS_STATUS_SUCCESS,
97                 SCST_CONTEXT_THREAD);
98 }
99
100 static inline void iscsi_restart_waiting_cmnd(struct iscsi_cmnd *cmnd)
101 {
102         /*
103          * There is no race with conn_abort(), since all functions
104          * called from single read thread
105          */
106         iscsi_extracheck_is_rd_thread(cmnd->conn);
107         cmnd->data_waiting = 0;
108
109         iscsi_restart_cmnd(cmnd);
110 }
111
112 static inline void iscsi_fail_waiting_cmnd(struct iscsi_cmnd *cmnd)
113 {
114         TRACE_DBG("Failing data waiting cmd %p", cmnd);
115
116         /*
117          * There is no race with conn_abort(), since all functions
118          * called from single read thread
119          */
120         iscsi_extracheck_is_rd_thread(cmnd->conn);
121         cmnd->data_waiting = 0;
122
123         req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
124 }
125
126 struct iscsi_cmnd *cmnd_alloc(struct iscsi_conn *conn, struct iscsi_cmnd *parent)
127 {
128         struct iscsi_cmnd *cmnd;
129
130         /* ToDo: __GFP_NOFAIL?? */
131 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,17)
132         cmnd = kmem_cache_alloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
133         memset(cmnd, 0, sizeof(*cmnd));
134 #else
135         cmnd = kmem_cache_zalloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
136 #endif
137
138         atomic_set(&cmnd->ref_cnt, 1);
139         cmnd->scst_state = ISCSI_CMD_STATE_NEW;
140         cmnd->conn = conn;
141         cmnd->parent_req = parent;
142         init_waitqueue_head(&cmnd->scst_waitQ);
143
144         if (parent == NULL) {
145                 conn_get(conn);
146
147 #ifdef NET_PAGE_CALLBACKS_DEFINED
148                 atomic_set(&cmnd->net_ref_cnt, 0);
149 #endif          
150                 cmnd->target = conn->target;
151                 spin_lock_init(&cmnd->rsp_cmd_lock);
152                 INIT_LIST_HEAD(&cmnd->rsp_cmd_list);
153                 INIT_LIST_HEAD(&cmnd->rx_ddigest_cmd_list);
154
155                 spin_lock_bh(&conn->cmd_list_lock);
156                 list_add_tail(&cmnd->cmd_list_entry, &conn->cmd_list);
157                 spin_unlock_bh(&conn->cmd_list_lock);
158         }
159
160         TRACE_DBG("conn %p, parent %p, cmnd %p", conn, parent, cmnd);
161         return cmnd;
162 }
163
164 /* Frees a command. Also frees the additional header. */
165 void cmnd_free(struct iscsi_cmnd *cmnd)
166 {
167         TRACE_DBG("%p", cmnd);
168
169         /* Catch users from cmd_list or rsp_cmd_list */
170         EXTRACHECKS_BUG_ON(atomic_read(&cmnd->ref_cnt) != 0);
171
172         kfree(cmnd->pdu.ahs);
173
174         if (unlikely(cmnd->on_write_list)) {
175                 struct iscsi_scsi_cmd_hdr *req = cmnd_hdr(cmnd);
176
177                 PRINT_ERROR("cmnd %p still on some list?, %x, %x, %x, %x, %x, %x, %x",
178                         cmnd, req->opcode, req->scb[0], req->flags, req->itt,
179                         be32_to_cpu(req->data_length),
180                         req->cmd_sn, be32_to_cpu(cmnd->pdu.datasize));
181
182                 if (unlikely(cmnd->parent_req)) {
183                         struct iscsi_scsi_cmd_hdr *req =
184                                         cmnd_hdr(cmnd->parent_req);
185                         PRINT_ERROR("%p %x %u", req, req->opcode, req->scb[0]);
186                 }
187                 sBUG();
188         }
189
190         kmem_cache_free(iscsi_cmnd_cache, cmnd);
191         return;
192 }
193
194 void cmnd_done(struct iscsi_cmnd *cmnd)
195 {
196         TRACE_DBG("%p", cmnd);
197
198         if (unlikely(cmnd->tmfabort)) {
199                 TRACE_MGMT_DBG("Done aborted cmd %p (scst cmd %p, state %d, "
200                         "parent_req %p)", cmnd, cmnd->scst_cmd, cmnd->scst_state,
201                         cmnd->parent_req);
202         }
203
204         EXTRACHECKS_BUG_ON(cmnd->on_rx_digest_list);
205
206         if (cmnd->parent_req == NULL) {
207                 struct iscsi_conn *conn = cmnd->conn;
208                 TRACE_DBG("Deleting req %p from conn %p", cmnd, conn);
209
210                 spin_lock_bh(&conn->cmd_list_lock);
211                 list_del(&cmnd->cmd_list_entry);
212                 spin_unlock_bh(&conn->cmd_list_lock);
213
214                 conn_put(conn);
215
216                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rsp_cmd_list));
217                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rx_ddigest_cmd_list));
218
219                 /* Order between above and below code is important! */
220
221                 if (cmnd->scst_cmd) {
222                         switch(cmnd->scst_state) {
223                         case ISCSI_CMD_STATE_AFTER_PREPROC:
224                                 TRACE_DBG("cmd %p AFTER_PREPROC", cmnd);
225                                 cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
226                                 scst_restart_cmd(cmnd->scst_cmd,
227                                         SCST_PREPROCESS_STATUS_ERROR_FATAL,
228                                         SCST_CONTEXT_DIRECT);
229                                 break;
230                         case ISCSI_CMD_STATE_PROCESSED:
231                                 TRACE_DBG("cmd %p PROCESSED", cmnd);
232                                 scst_tgt_cmd_done(cmnd->scst_cmd);
233                                 break;
234                         default:
235                                 PRINT_ERROR("Unexpected cmnd scst state %d",
236                                         cmnd->scst_state);
237                                 sBUG();
238                                 break;
239                         }
240                 }
241         } else {
242                 EXTRACHECKS_BUG_ON(cmnd->scst_cmd != NULL);
243                 TRACE_DBG("Deleting rsp %p from parent %p", cmnd,
244                         cmnd->parent_req);
245
246                 spin_lock_bh(&cmnd->parent_req->rsp_cmd_lock);
247                 list_del(&cmnd->rsp_cmd_list_entry);
248                 spin_unlock_bh(&cmnd->parent_req->rsp_cmd_lock);
249
250                 cmnd_put(cmnd->parent_req);
251         }
252
253         /* Order between above and below code is important! */
254
255         if (cmnd->own_sg) {
256                 TRACE_DBG("%s", "own_sg");
257                 scst_free(cmnd->sg, cmnd->sg_cnt);
258 #ifdef DEBUG
259                 cmnd->own_sg = 0;
260                 cmnd->sg = NULL;
261                 cmnd->sg_cnt = -1;
262 #endif
263         }
264
265         if (cmnd->dec_active_cmnds) {
266                 struct iscsi_session *sess = cmnd->conn->session;
267                 TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
268                         "new value %d)", cmnd, sess,
269                         atomic_read(&sess->active_cmds)-1);
270                 atomic_dec(&sess->active_cmds);
271         }
272
273         cmnd_free(cmnd);
274         return;
275 }
276
277 /*
278  * Corresponding conn may also gets destroyed atfer this function, except only
279  * if it's called from the read thread!
280  *
281  * It can't be called in parallel with iscsi_cmnds_init_write()!
282  */
283 void req_cmnd_release_force(struct iscsi_cmnd *req, int flags)
284 {
285         struct iscsi_cmnd *rsp, *t;
286         struct iscsi_conn *conn = req->conn;
287         LIST_HEAD(cmds_list);
288
289         TRACE_ENTRY();
290
291         TRACE_DBG("%p", req);
292
293         sBUG_ON(req == conn->read_cmnd);
294
295         if (flags & ISCSI_FORCE_RELEASE_WRITE) {
296                 spin_lock(&conn->write_list_lock);
297                 list_for_each_entry_safe(rsp, t, &conn->write_list,
298                                                 write_list_entry) {
299                         if (rsp->parent_req != req)
300                                 continue;
301
302                         cmd_del_from_write_list(rsp);
303
304                         list_add_tail(&rsp->write_list_entry, &cmds_list);
305                 }
306                 spin_unlock(&conn->write_list_lock);
307
308                 list_for_each_entry_safe(rsp, t, &cmds_list, write_list_entry) {
309                         list_del(&rsp->write_list_entry);
310                         cmnd_put(rsp);
311                 }
312         }
313
314 again_rsp:
315         spin_lock_bh(&req->rsp_cmd_lock);
316         list_for_each_entry_reverse(rsp, &req->rsp_cmd_list, rsp_cmd_list_entry) {
317                 bool r;
318
319                 if (rsp->force_cleanup_done)
320                         continue;
321
322                 rsp->force_cleanup_done = 1;
323
324                 if (cmnd_get_check(rsp))
325                         continue;
326
327                 spin_unlock_bh(&req->rsp_cmd_lock);
328
329                 spin_lock(&conn->write_list_lock);
330                 r = rsp->on_write_list || rsp->write_processing_started;
331                 spin_unlock(&conn->write_list_lock);
332
333                 cmnd_put(rsp);
334
335                 if (r)
336                         continue;
337
338                 /*
339                  * If both on_write_list and write_processing_started not set,
340                  * we can safely put() cmnd
341                  */
342                 cmnd_put(rsp);
343                 goto again_rsp;
344         }
345         spin_unlock_bh(&req->rsp_cmd_lock);
346
347         req_cmnd_release(req);
348
349         TRACE_EXIT();
350         return;
351 }
352
353 /*
354  * Corresponding conn may also gets destroyed atfer this function, except only
355  * if it's called from the read thread! 
356  */
357 void req_cmnd_release(struct iscsi_cmnd *req)
358 {
359         struct iscsi_cmnd *c, *t;
360
361         TRACE_ENTRY();
362
363         TRACE_DBG("%p", req);
364
365 #ifdef EXTRACHECKS
366         sBUG_ON(req->release_called);
367         req->release_called = 1;
368 #endif
369
370         if (unlikely(req->tmfabort)) {
371                 TRACE_MGMT_DBG("Release aborted req cmd %p (scst cmd %p, "
372                         "state %d)", req, req->scst_cmd, req->scst_state);
373         }
374
375         sBUG_ON(req->parent_req != NULL);
376
377         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
378                                 rx_ddigest_cmd_list_entry) {
379                 cmd_del_from_rx_ddigest_list(c);
380                 cmnd_put(c);
381         }
382
383         if (req->hashed)
384                 cmnd_remove_hash(req);
385
386         if (req->dec_active_cmnds) {
387                 struct iscsi_session *sess = req->conn->session;
388                 TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
389                         "new value %d)", req, sess,
390                         atomic_read(&sess->active_cmds)-1);
391                 atomic_dec(&sess->active_cmds);
392                 req->dec_active_cmnds = 0;
393         }
394
395         cmnd_put(req);
396
397         TRACE_EXIT();
398         return;
399 }
400
401 /*
402  * Corresponding conn may also gets destroyed atfer this function, except only
403  * if it's called from the read thread! 
404  */
405 void rsp_cmnd_release(struct iscsi_cmnd *cmnd)
406 {
407         TRACE_DBG("%p", cmnd);
408
409 #ifdef EXTRACHECKS
410         sBUG_ON(cmnd->release_called);
411         cmnd->release_called = 1;
412 #endif
413
414         sBUG_ON(cmnd->hashed);
415         sBUG_ON(cmnd->parent_req == NULL);
416
417         cmnd_put(cmnd);
418         return;
419 }
420
421 /**
422  * create a new command used as response.
423  *
424  * iscsi_cmnd_create_rsp_cmnd - 
425  * @cmnd: ptr to request command
426  *
427  * @return    ptr to response command or NULL
428  */
429 static struct iscsi_cmnd *iscsi_cmnd_create_rsp_cmnd(struct iscsi_cmnd *parent)
430 {
431         struct iscsi_cmnd *rsp;
432
433         rsp = cmnd_alloc(parent->conn, parent);
434
435         spin_lock_bh(&parent->rsp_cmd_lock);
436         TRACE_DBG("Adding rsp %p to parent %p", rsp, parent);
437         list_add_tail(&rsp->rsp_cmd_list_entry, &parent->rsp_cmd_list);
438         spin_unlock_bh(&parent->rsp_cmd_lock);
439         cmnd_get(parent);
440         return rsp;
441 }
442
443 static inline struct iscsi_cmnd *get_rsp_cmnd(struct iscsi_cmnd *req)
444 {
445         struct iscsi_cmnd *res;
446
447         /* Currently this lock isn't needed, but just in case.. */
448         spin_lock_bh(&req->rsp_cmd_lock);
449         res = list_entry(req->rsp_cmd_list.prev, struct iscsi_cmnd,
450                 rsp_cmd_list_entry);
451         spin_unlock_bh(&req->rsp_cmd_lock);
452
453         return res;
454 }
455
456 static void iscsi_cmnds_init_write(struct list_head *send, int flags)
457 {
458         struct iscsi_cmnd *rsp = list_entry(send->next, struct iscsi_cmnd, 
459                                                 write_list_entry);
460         struct iscsi_conn *conn = rsp->conn;
461         struct list_head *pos, *next;
462
463         /*
464          * If we don't remove hashed req cmd from the hash list here, before
465          * submitting it for transmittion, we will have a race, when for
466          * some reason cmd's release is delayed after transmittion and
467          * initiator sends cmd with the same ITT => this command will be
468          * erroneously rejected as a duplicate.
469          */
470         if ((flags & ISCSI_INIT_WRITE_REMOVE_HASH) && rsp->parent_req->hashed &&
471             (rsp->parent_req->r2t_length == 0) &&
472             (rsp->parent_req->outstanding_r2t == 0))
473                 cmnd_remove_hash(rsp->parent_req);
474
475         list_for_each_safe(pos, next, send) {
476                 rsp = list_entry(pos, struct iscsi_cmnd, write_list_entry);
477
478                 TRACE_DBG("%p:%x", rsp, cmnd_opcode(rsp));
479
480                 sBUG_ON(conn != rsp->conn);
481
482                 if (!(conn->ddigest_type & DIGEST_NONE) &&
483                     (rsp->pdu.datasize != 0))
484                         digest_tx_data(rsp);
485
486                 list_del(&rsp->write_list_entry);
487
488                 spin_lock(&conn->write_list_lock);
489                 cmd_add_on_write_list(conn, rsp);
490                 spin_unlock(&conn->write_list_lock);
491         }
492
493         if (flags & ISCSI_INIT_WRITE_WAKE)
494                 iscsi_make_conn_wr_active(conn);
495 }
496
497 static void iscsi_cmnd_init_write(struct iscsi_cmnd *rsp, int flags)
498 {
499         LIST_HEAD(head);
500
501         if (unlikely(rsp->on_write_list)) {
502                 PRINT_ERROR("cmd already on write list (%x %x %x %x %u %u "
503                         "%u %u %u %u %u %d %d",
504                         cmnd_itt(rsp), cmnd_ttt(rsp), cmnd_opcode(rsp),
505                         cmnd_scsicode(rsp), rsp->r2t_sn,
506                         rsp->r2t_length, rsp->is_unsolicited_data,
507                         rsp->target_task_tag, rsp->outstanding_r2t,
508                         rsp->hdigest, rsp->ddigest,
509                         list_empty(&rsp->rsp_cmd_list), rsp->hashed);
510                 sBUG();
511         }
512         list_add(&rsp->write_list_entry, &head);
513         iscsi_cmnds_init_write(&head, flags);
514 }
515
516 static void iscsi_set_datasize(struct iscsi_cmnd *cmnd, u32 offset, u32 size)
517 {
518         cmnd->pdu.datasize = size;
519
520         if (cmnd->pdu.datasize & 3) {
521                 int idx = (offset + cmnd->pdu.datasize) >> PAGE_SHIFT;
522                 u8 *p = (u8 *)page_address(cmnd->sg[idx].page) + 
523                         ((offset + cmnd->pdu.datasize) & ~PAGE_MASK);
524                 int i = 4 - (cmnd->pdu.datasize & 3);
525                 while (i--) 
526                     *p++ = 0;
527         }
528 }
529
530 static void send_data_rsp(struct iscsi_cmnd *req, u8 status, int send_status)
531 {
532         struct iscsi_cmnd *rsp;
533         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
534         struct iscsi_data_in_hdr *rsp_hdr;
535         u32 pdusize, expsize, scsisize, size, offset, sn;
536         LIST_HEAD(send);
537
538         TRACE_DBG("req %p", req);
539         pdusize = req->conn->session->sess_param.max_xmit_data_length;
540         expsize = cmnd_read_size(req);
541         size = min(expsize, (u32)req->bufflen);
542         offset = 0;
543         sn = 0;
544
545         while (1) {
546                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
547                 TRACE_DBG("rsp %p", rsp);
548                 rsp->sg = req->sg;
549                 rsp->bufflen = req->bufflen;
550                 rsp_hdr = (struct iscsi_data_in_hdr *)&rsp->pdu.bhs;
551
552                 rsp_hdr->opcode = ISCSI_OP_SCSI_DATA_IN;
553                 rsp_hdr->itt = req_hdr->itt;
554                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
555                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
556                 rsp_hdr->data_sn = cpu_to_be32(sn);
557
558                 if (size <= pdusize) {
559                         iscsi_set_datasize(rsp, offset, size);
560                         if (send_status) {
561                                 TRACE_DBG("status %x", status);
562                                 rsp_hdr->flags =
563                                         ISCSI_FLG_FINAL | ISCSI_FLG_STATUS;
564                                 rsp_hdr->cmd_status = status;
565                         }
566                         scsisize = req->bufflen;
567                         if (scsisize < expsize) {
568                                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
569                                 size = expsize - scsisize;
570                         } else if (scsisize > expsize) {
571                                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
572                                 size = scsisize - expsize;
573                         } else
574                                 size = 0;
575                         rsp_hdr->residual_count = cpu_to_be32(size);
576                         list_add_tail(&rsp->write_list_entry, &send);
577                         break;
578                 }
579
580                 iscsi_set_datasize(rsp, offset, pdusize);
581
582                 size -= pdusize;
583                 offset += pdusize;
584                 sn++;
585
586                 list_add_tail(&rsp->write_list_entry, &send);
587         }
588         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_REMOVE_HASH);
589 }
590
591 static struct iscsi_cmnd *create_status_rsp(struct iscsi_cmnd *req, int status,
592         const u8 *sense_buf, int sense_len)
593 {
594         struct iscsi_cmnd *rsp;
595         struct iscsi_scsi_rsp_hdr *rsp_hdr;
596         struct iscsi_sense_data *sense;
597         struct scatterlist *sg;
598
599         rsp = iscsi_cmnd_create_rsp_cmnd(req);
600         TRACE_DBG("%p", rsp);
601
602         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
603         rsp_hdr->opcode = ISCSI_OP_SCSI_RSP;
604         rsp_hdr->flags = ISCSI_FLG_FINAL;
605         rsp_hdr->response = ISCSI_RESPONSE_COMMAND_COMPLETED;
606         rsp_hdr->cmd_status = status;
607         rsp_hdr->itt = cmnd_hdr(req)->itt;
608
609         if (status == SAM_STAT_CHECK_CONDITION) {
610                 TRACE_DBG("%s", "CHECK_CONDITION");
611                 /* ToDo: __GFP_NOFAIL ?? */
612                 sg = rsp->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
613                                         &rsp->sg_cnt);
614                 if (sg == NULL) {
615                         /* ToDo(); */
616                 }
617                 rsp->own_sg = 1;
618                 sense = (struct iscsi_sense_data *)page_address(sg[0].page);
619                 sense->length = cpu_to_be16(sense_len);
620                 memcpy(sense->data, sense_buf, sense_len);
621                 rsp->pdu.datasize = sizeof(struct iscsi_sense_data) + sense_len;
622                 rsp->bufflen = (rsp->pdu.datasize + 3) & -4;
623                 if (rsp->bufflen - rsp->pdu.datasize) {
624                     int i = rsp->pdu.datasize;
625                     u8 *p = (u8 *)sense + i;
626                     
627                     while (i < rsp->bufflen) {
628                         *p ++ = 0;
629                         i++;
630                     }
631                 }
632         } else {
633                 rsp->pdu.datasize = 0;
634                 rsp->bufflen = 0;
635         }
636
637         return rsp;
638 }
639
640 static struct iscsi_cmnd *create_sense_rsp(struct iscsi_cmnd *req,
641         u8 sense_key, u8 asc, u8 ascq)
642 {
643         u8 sense[14];
644         memset(sense, 0, sizeof(sense));
645         sense[0] = 0xf0;
646         sense[2] = sense_key;
647         sense[7] = 6;   // Additional sense length
648         sense[12] = asc;
649         sense[13] = ascq;
650         return create_status_rsp(req, SAM_STAT_CHECK_CONDITION, sense,
651                 sizeof(sense));
652 }
653
654 static void iscsi_cmnd_reject(struct iscsi_cmnd *req, int reason)
655 {
656         struct iscsi_cmnd *rsp;
657         struct iscsi_reject_hdr *rsp_hdr;
658         struct scatterlist *sg;
659         char *addr;
660
661         TRACE_MGMT_DBG("Reject: req %p, reason %x", req, reason);
662
663         rsp = iscsi_cmnd_create_rsp_cmnd(req);
664         rsp_hdr = (struct iscsi_reject_hdr *)&rsp->pdu.bhs;
665
666         rsp_hdr->opcode = ISCSI_OP_REJECT;
667         rsp_hdr->ffffffff = ISCSI_RESERVED_TAG;
668         rsp_hdr->reason = reason;
669
670         /* ToDo: __GFP_NOFAIL ?? */
671         sg = rsp->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
672                                 &rsp->sg_cnt);
673         if (sg == NULL) {
674                 /* ToDo(); */
675         }
676         rsp->own_sg = 1;
677         addr = page_address(sg[0].page);
678         clear_page(addr);
679         memcpy(addr, &req->pdu.bhs, sizeof(struct iscsi_hdr));
680         rsp->bufflen = rsp->pdu.datasize = sizeof(struct iscsi_hdr);
681         cmnd_prepare_skip_pdu(req);
682
683         req->pdu.bhs.opcode = ISCSI_OP_PDU_REJECT;
684 }
685
686 static inline int iscsi_get_allowed_cmds(struct iscsi_session *sess)
687 {
688         int res = max(-1, (int)sess->max_queued_cmnds - 
689                                 atomic_read(&sess->active_cmds)-1);
690         TRACE_DBG("allowed cmds %d (sess %p, active_cmds %d)", res,
691                 sess, atomic_read(&sess->active_cmds));
692         return res;
693 }
694
695 static u32 cmnd_set_sn(struct iscsi_cmnd *cmnd, int set_stat_sn)
696 {
697         struct iscsi_conn *conn = cmnd->conn;
698         struct iscsi_session *sess = conn->session;
699         u32 res;
700
701         spin_lock(&sess->sn_lock);
702
703         if (set_stat_sn)
704                 cmnd->pdu.bhs.sn = cpu_to_be32(conn->stat_sn++);
705         cmnd->pdu.bhs.exp_sn = cpu_to_be32(sess->exp_cmd_sn);
706         cmnd->pdu.bhs.max_sn = cpu_to_be32(sess->exp_cmd_sn +
707                                  iscsi_get_allowed_cmds(sess));
708
709         res = cpu_to_be32(conn->stat_sn);
710
711         spin_unlock(&sess->sn_lock);
712         return res;
713 }
714
715 /* Called under sn_lock */
716 static void __update_stat_sn(struct iscsi_cmnd *cmnd)
717 {
718         struct iscsi_conn *conn = cmnd->conn;
719         u32 exp_stat_sn;
720
721         cmnd->pdu.bhs.exp_sn = exp_stat_sn = be32_to_cpu(cmnd->pdu.bhs.exp_sn);
722         TRACE_DBG("%x,%x", cmnd_opcode(cmnd), exp_stat_sn);
723         if ((int)(exp_stat_sn - conn->exp_stat_sn) > 0 &&
724             (int)(exp_stat_sn - conn->stat_sn) <= 0) {
725                 // free pdu resources
726                 cmnd->conn->exp_stat_sn = exp_stat_sn;
727         }
728 }
729
730 static inline void update_stat_sn(struct iscsi_cmnd *cmnd)
731 {
732         spin_lock(&cmnd->conn->session->sn_lock);
733         __update_stat_sn(cmnd);
734         spin_unlock(&cmnd->conn->session->sn_lock);
735 }
736
737 /* Called under sn_lock */
738 static int check_cmd_sn(struct iscsi_cmnd *cmnd)
739 {
740         struct iscsi_session *session = cmnd->conn->session;
741         u32 cmd_sn;
742
743         cmnd->pdu.bhs.sn = cmd_sn = be32_to_cpu(cmnd->pdu.bhs.sn);
744         TRACE_DBG("%d(%d)", cmd_sn, session->exp_cmd_sn);
745         if (likely((s32)(cmd_sn - session->exp_cmd_sn) >= 0))
746                 return 0;
747         PRINT_ERROR("sequence error (%x,%x)", cmd_sn, session->exp_cmd_sn);
748         return -ISCSI_REASON_PROTOCOL_ERROR;
749 }
750
751 static inline struct iscsi_cmnd *__cmnd_find_hash(struct iscsi_session *session,
752         u32 itt, u32 ttt)
753 {
754         struct list_head *head;
755         struct iscsi_cmnd *cmnd;
756
757         head = &session->cmnd_hash[cmnd_hashfn(itt)];
758
759         list_for_each_entry(cmnd, head, hash_list_entry) {
760                 if (cmnd->pdu.bhs.itt == itt) {
761                         if ((ttt != ISCSI_RESERVED_TAG) && (ttt != cmnd->target_task_tag))
762                                 continue;
763                         return cmnd;
764                 }
765         }
766         return NULL;
767 }
768
769 static struct iscsi_cmnd *cmnd_find_hash(struct iscsi_session *session,
770         u32 itt, u32 ttt)
771 {
772         struct iscsi_cmnd *cmnd;
773
774         spin_lock(&session->cmnd_hash_lock);
775         cmnd = __cmnd_find_hash(session, itt, ttt);
776         spin_unlock(&session->cmnd_hash_lock);
777
778         return cmnd;
779 }
780
781 static struct iscsi_cmnd *cmnd_find_hash_get(struct iscsi_session *session,
782         u32 itt, u32 ttt)
783 {
784         struct iscsi_cmnd *cmnd;
785
786         spin_lock(&session->cmnd_hash_lock);
787         cmnd = __cmnd_find_hash(session, itt, ttt);
788         if (cmnd != NULL) {
789                 if (unlikely(cmnd_get_check(cmnd)))
790                         cmnd = NULL;
791         }
792         spin_unlock(&session->cmnd_hash_lock);
793
794         return cmnd;
795 }
796
797 static int cmnd_insert_hash(struct iscsi_cmnd *cmnd)
798 {
799         struct iscsi_session *session = cmnd->conn->session;
800         struct iscsi_cmnd *tmp;
801         struct list_head *head;
802         int err = 0;
803         u32 itt = cmnd->pdu.bhs.itt;
804
805         TRACE_DBG("%p:%x", cmnd, itt);
806         if (unlikely(itt == ISCSI_RESERVED_TAG)) {
807                 PRINT_ERROR("%s", "ITT is RESERVED_TAG");
808                 err = -ISCSI_REASON_PROTOCOL_ERROR;
809                 goto out;
810         }
811
812         spin_lock(&session->cmnd_hash_lock);
813
814         head = &session->cmnd_hash[cmnd_hashfn(cmnd->pdu.bhs.itt)];
815
816         tmp = __cmnd_find_hash(session, itt, ISCSI_RESERVED_TAG);
817         if (likely(!tmp)) {
818                 list_add_tail(&cmnd->hash_list_entry, head);
819                 cmnd->hashed = 1;
820         } else {
821                 PRINT_ERROR("Task %x in progress, cmnd %p", itt, cmnd);
822                 err = -ISCSI_REASON_TASK_IN_PROGRESS;
823         }
824
825         spin_unlock(&session->cmnd_hash_lock);
826
827         if (likely(!err)) {
828                 spin_lock(&session->sn_lock);
829                 __update_stat_sn(cmnd);
830                 err = check_cmd_sn(cmnd);
831                 spin_unlock(&session->sn_lock);
832         }
833
834 out:
835         return err;
836 }
837
838 static void cmnd_remove_hash(struct iscsi_cmnd *cmnd)
839 {
840         struct iscsi_session *session = cmnd->conn->session;
841         struct iscsi_cmnd *tmp;
842
843         spin_lock(&session->cmnd_hash_lock);
844
845         tmp = __cmnd_find_hash(session, cmnd->pdu.bhs.itt, ISCSI_RESERVED_TAG);
846
847         if (likely(tmp && tmp == cmnd)) {
848                 list_del(&cmnd->hash_list_entry);
849                 cmnd->hashed = 0;
850         } else {
851                 PRINT_ERROR("%p:%x not found", cmnd, cmnd_itt(cmnd));
852         }
853
854         spin_unlock(&session->cmnd_hash_lock);
855 }
856
857 static void cmnd_prepare_skip_pdu(struct iscsi_cmnd *cmnd)
858 {
859         struct iscsi_conn *conn = cmnd->conn;
860         struct scatterlist *sg = cmnd->sg;
861         char *addr;
862         u32 size;
863         int i;
864
865         TRACE_MGMT_DBG("Skipping (%p, %x %x %x %u, %p, scst state %d)", cmnd,
866                 cmnd_itt(cmnd), cmnd_opcode(cmnd), cmnd_hdr(cmnd)->scb[0],
867                 cmnd->pdu.datasize, cmnd->scst_cmd, cmnd->scst_state);
868
869         iscsi_extracheck_is_rd_thread(conn);
870
871         if (!(size = cmnd->pdu.datasize))
872                 return;
873
874         if (sg == NULL) {
875                 /* ToDo: __GFP_NOFAIL ?? */
876                 sg = cmnd->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
877                                         &cmnd->sg_cnt);
878                 if (sg == NULL) {
879                         /* ToDo(); */
880                 }
881                 cmnd->own_sg = 1;
882                 cmnd->bufflen = PAGE_SIZE;
883         }
884
885         addr = page_address(sg[0].page);
886         sBUG_ON(addr == NULL);
887         size = (size + 3) & -4;
888         conn->read_size = size;
889         for (i = 0; size > PAGE_SIZE; i++, size -= cmnd->bufflen) {
890                 sBUG_ON(i >= ISCSI_CONN_IOV_MAX);
891                 conn->read_iov[i].iov_base = addr;
892                 conn->read_iov[i].iov_len = cmnd->bufflen;
893         }
894         conn->read_iov[i].iov_base = addr;
895         conn->read_iov[i].iov_len = size;
896         conn->read_msg.msg_iov = conn->read_iov;
897         conn->read_msg.msg_iovlen = ++i;
898 }
899
900 static void cmnd_prepare_skip_pdu_set_resid(struct iscsi_cmnd *req)
901 {
902         struct iscsi_cmnd *rsp;
903         struct iscsi_scsi_rsp_hdr *rsp_hdr;
904         u32 size;
905
906         TRACE_DBG("%p", req);
907
908         rsp = get_rsp_cmnd(req);
909         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
910         if (unlikely(cmnd_opcode(rsp) != ISCSI_OP_SCSI_RSP)) {
911                 PRINT_ERROR("Unexpected response command %u", cmnd_opcode(rsp));
912                 return;
913         }
914
915         size = cmnd_write_size(req);
916         if (size) {
917                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
918                 rsp_hdr->residual_count = cpu_to_be32(size);
919         }
920         size = cmnd_read_size(req);
921         if (size) {
922                 if (cmnd_hdr(req)->flags & ISCSI_CMD_WRITE) {
923                         rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_UNDERFLOW;
924                         rsp_hdr->bi_residual_count = cpu_to_be32(size);
925                 } else {
926                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
927                         rsp_hdr->residual_count = cpu_to_be32(size);
928                 }
929         }
930         req->pdu.bhs.opcode =
931                 (req->pdu.bhs.opcode & ~ISCSI_OPCODE_MASK) | ISCSI_OP_SCSI_REJECT;
932
933         cmnd_prepare_skip_pdu(req);
934 }
935
936 static int cmnd_prepare_recv_pdu(struct iscsi_conn *conn,
937         struct iscsi_cmnd *cmd, u32 offset, u32 size)
938 {
939         struct scatterlist *sg = cmd->sg;
940         int bufflen = cmd->bufflen;
941         int idx, i;
942         char *addr;
943         int res = 0;
944
945         TRACE_DBG("%p %u,%u", cmd->sg, offset, size);
946
947         iscsi_extracheck_is_rd_thread(conn);
948
949         if (unlikely((offset >= bufflen) ||
950                      (offset + size > bufflen))) {
951                 PRINT_ERROR("Wrong ltn (%u %u %u)", offset, size, bufflen);
952                 mark_conn_closed(conn);
953                 res = -EIO;
954                 goto out;
955         }
956
957         offset += sg[0].offset;
958         idx = offset >> PAGE_SHIFT;
959         offset &= ~PAGE_MASK;
960
961         conn->read_msg.msg_iov = conn->read_iov;
962         conn->read_size = size = (size + 3) & -4;
963
964         i = 0;
965         while (1) {
966                 sBUG_ON(sg[idx].page == NULL);
967                 addr = page_address(sg[idx].page);
968                 sBUG_ON(addr == NULL);
969                 conn->read_iov[i].iov_base = addr + offset;
970                 if (offset + size <= PAGE_SIZE) {
971                         TRACE_DBG("idx=%d, offset=%u, size=%d, addr=%p",
972                                 idx, offset, size, addr);
973                         conn->read_iov[i].iov_len = size;
974                         conn->read_msg.msg_iovlen = ++i;
975                         break;
976                 }
977                 conn->read_iov[i].iov_len = PAGE_SIZE - offset;
978                 TRACE_DBG("idx=%d, offset=%u, size=%d, iov_len=%d, addr=%p",
979                         idx, offset, size, conn->read_iov[i].iov_len, addr);
980                 size -= conn->read_iov[i].iov_len;
981                 offset = 0;
982                 if (unlikely(++i >= ISCSI_CONN_IOV_MAX)) {
983                         PRINT_ERROR("Initiator %s violated negotiated "
984                                 "parameters by sending too much data (size "
985                                 "left %d)", conn->session->initiator_name, size);
986                         mark_conn_closed(conn);
987                         res = -EINVAL;
988                         break;
989                 }
990                 idx++;
991         }
992         TRACE_DBG("msg_iov=%p, msg_iovlen=%d",
993                 conn->read_msg.msg_iov, conn->read_msg.msg_iovlen);
994
995 out:
996         return res;
997 }
998
999 static void send_r2t(struct iscsi_cmnd *req)
1000 {
1001         struct iscsi_session *session = req->conn->session;
1002         struct iscsi_cmnd *rsp;
1003         struct iscsi_r2t_hdr *rsp_hdr;
1004         u32 offset, burst;
1005         LIST_HEAD(send);
1006
1007         if (unlikely(req->tmfabort)) {
1008                 TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted on R2T "
1009                         "(r2t_length %d, outstanding_r2t %d)", req,
1010                         req->scst_cmd, req->r2t_length, req->outstanding_r2t);
1011                 if (req->outstanding_r2t == 0)
1012                         iscsi_fail_waiting_cmnd(req);
1013                 goto out;
1014         }
1015
1016         /*
1017          * There is no race with data_out_start() and conn_abort(), since
1018          * all functions called from single read thread
1019          */
1020         iscsi_extracheck_is_rd_thread(req->conn);
1021
1022         burst = session->sess_param.max_burst_length;
1023         offset = be32_to_cpu(cmnd_hdr(req)->data_length) - req->r2t_length;
1024
1025         do {
1026                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
1027                 rsp->pdu.bhs.ttt = req->target_task_tag;
1028                 rsp_hdr = (struct iscsi_r2t_hdr *)&rsp->pdu.bhs;
1029                 rsp_hdr->opcode = ISCSI_OP_R2T;
1030                 rsp_hdr->flags = ISCSI_FLG_FINAL;
1031                 rsp_hdr->lun = cmnd_hdr(req)->lun;
1032                 rsp_hdr->itt = cmnd_hdr(req)->itt;
1033                 rsp_hdr->r2t_sn = cpu_to_be32(req->r2t_sn++);
1034                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
1035                 if (req->r2t_length > burst) {
1036                         rsp_hdr->data_length = cpu_to_be32(burst);
1037                         req->r2t_length -= burst;
1038                         offset += burst;
1039                 } else {
1040                         rsp_hdr->data_length = cpu_to_be32(req->r2t_length);
1041                         req->r2t_length = 0;
1042                 }
1043
1044                 TRACE_WRITE("%x %u %u %u %u", cmnd_itt(req),
1045                         be32_to_cpu(rsp_hdr->data_length),
1046                         be32_to_cpu(rsp_hdr->buffer_offset),
1047                         be32_to_cpu(rsp_hdr->r2t_sn), req->outstanding_r2t);
1048
1049                 list_add_tail(&rsp->write_list_entry, &send);
1050
1051                 if (++req->outstanding_r2t >= session->sess_param.max_outstanding_r2t)
1052                         break;
1053
1054         } while(req->r2t_length != 0);
1055
1056         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_WAKE);
1057
1058 out:
1059         return;
1060 }
1061
1062 static int iscsi_pre_exec(struct scst_cmd *scst_cmd)
1063 {
1064         int res = SCST_PREPROCESS_STATUS_SUCCESS;
1065         struct iscsi_cmnd *req = (struct iscsi_cmnd*)
1066                 scst_cmd_get_tgt_priv(scst_cmd);
1067         struct iscsi_cmnd *c, *t;
1068
1069         TRACE_ENTRY();
1070
1071         EXTRACHECKS_BUG_ON(scst_cmd_atomic(scst_cmd));
1072
1073         /* If data digest isn't used this list will be empty */
1074         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
1075                                 rx_ddigest_cmd_list_entry) {
1076                 TRACE_DBG("Checking digest of RX ddigest cmd %p", c);
1077                 if (digest_rx_data(c) != 0) {
1078                         scst_set_cmd_error(scst_cmd,
1079                                 SCST_LOAD_SENSE(iscsi_sense_crc_error));
1080                         res = SCST_PREPROCESS_STATUS_ERROR_SENSE_SET;
1081                         /*
1082                          * The rest of rx_ddigest_cmd_list will be freed 
1083                          * in req_cmnd_release()
1084                          */
1085                         goto out;
1086                 }
1087                 cmd_del_from_rx_ddigest_list(c);
1088                 cmnd_put(c);
1089         }
1090
1091 out:
1092         TRACE_EXIT_RES(res);
1093         return res;
1094 }
1095
1096 static int noop_out_start(struct iscsi_cmnd *cmnd)
1097 {
1098         struct iscsi_conn *conn = cmnd->conn;
1099         u32 size, tmp;
1100         int i, err = 0;
1101
1102         TRACE_DBG("%p", cmnd);
1103
1104         iscsi_extracheck_is_rd_thread(conn);
1105
1106         if (unlikely(cmnd_ttt(cmnd) != cpu_to_be32(ISCSI_RESERVED_TAG))) {
1107                 /*
1108                  * We don't request a NOP-Out by sending a NOP-In.
1109                  * See 10.18.2 in the draft 20.
1110                  */
1111                 PRINT_ERROR("Initiator sent command with not RESERVED tag and "
1112                         "TTT %x", cmnd_itt(cmnd));
1113                 err = -ISCSI_REASON_PROTOCOL_ERROR;
1114                 goto out;
1115         }
1116
1117         if (cmnd_itt(cmnd) == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1118                 if (unlikely(!(cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE)))
1119                         PRINT_ERROR("%s", "Initiator sent RESERVED tag for "
1120                                 "non-immediate command");
1121                 spin_lock(&conn->session->sn_lock);
1122                 __update_stat_sn(cmnd);
1123                 err = check_cmd_sn(cmnd);
1124                 spin_unlock(&conn->session->sn_lock);
1125                 if (unlikely(err))
1126                         goto out;
1127         } else if (unlikely((err = cmnd_insert_hash(cmnd)) < 0)) {
1128                 PRINT_ERROR("Can't insert in hash: ignore this request %x",
1129                         cmnd_itt(cmnd));
1130                 goto out;
1131         }
1132
1133         if ((size = cmnd->pdu.datasize)) {
1134                 size = (size + 3) & -4;
1135                 conn->read_msg.msg_iov = conn->read_iov;
1136                 if (cmnd->pdu.bhs.itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1137                         struct scatterlist *sg;
1138
1139                         /* ToDo: __GFP_NOFAIL ?? */
1140                         cmnd->sg = sg = scst_alloc(size,
1141                                 GFP_KERNEL|__GFP_NOFAIL, &cmnd->sg_cnt);
1142                         if (sg == NULL) {
1143                                 /* ToDo(); */
1144                         }
1145                         if (cmnd->sg_cnt > ISCSI_CONN_IOV_MAX) {
1146                                 /* ToDo(); */
1147                         }
1148                         cmnd->own_sg = 1;
1149                         cmnd->bufflen = size;
1150
1151                         for (i = 0; i < cmnd->sg_cnt; i++) {
1152                                 conn->read_iov[i].iov_base =
1153                                         page_address(sg[i].page);
1154                                 tmp = min_t(u32, size, PAGE_SIZE);
1155                                 conn->read_iov[i].iov_len = tmp;
1156                                 conn->read_size += tmp;
1157                                 size -= tmp;
1158                         }
1159                 } else {
1160                         /*
1161                          * There are no problems with the safety from concurrent
1162                          * accesses to dummy_data, since for ISCSI_RESERVED_TAG
1163                          * the data only read and then discarded.
1164                          */
1165                         for (i = 0; i < ISCSI_CONN_IOV_MAX; i++) {
1166                                 conn->read_iov[i].iov_base = dummy_data;
1167                                 tmp = min_t(u32, size, sizeof(dummy_data));
1168                                 conn->read_iov[i].iov_len = tmp;
1169                                 conn->read_size += tmp;
1170                                 size -= tmp;
1171                         }
1172                 }
1173                 sBUG_ON(size == 0);
1174                 conn->read_msg.msg_iovlen = i;
1175                 TRACE_DBG("msg_iov=%p, msg_iovlen=%d", conn->read_msg.msg_iov,
1176                         conn->read_msg.msg_iovlen);
1177         }
1178 out:
1179         return err;
1180 }
1181
1182 static inline u32 get_next_ttt(struct iscsi_conn *conn)
1183 {
1184         u32 ttt;
1185         struct iscsi_session *session = conn->session;
1186
1187         iscsi_extracheck_is_rd_thread(conn);
1188
1189         if (session->next_ttt == ISCSI_RESERVED_TAG)
1190                 session->next_ttt++;
1191         ttt = session->next_ttt++;
1192
1193         return cpu_to_be32(ttt);
1194 }
1195
1196 static int scsi_cmnd_start(struct iscsi_cmnd *req)
1197 {
1198         struct iscsi_conn *conn = req->conn;
1199         struct iscsi_session *session = conn->session;
1200         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1201         struct scst_cmd *scst_cmd;
1202         scst_data_direction dir;
1203         int res = 0;
1204
1205         TRACE_ENTRY();
1206
1207         TRACE_DBG("scsi command: %02x", req_hdr->scb[0]);
1208
1209         TRACE_DBG("Incrementing active_cmds (cmd %p, sess %p, "
1210                 "new value %d)", req, session,
1211                 atomic_read(&session->active_cmds)+1);
1212         atomic_inc(&session->active_cmds);
1213         req->dec_active_cmnds = 1;
1214
1215         scst_cmd = scst_rx_cmd(session->scst_sess,
1216                 (uint8_t*)&req_hdr->lun, sizeof(req_hdr->lun),
1217                 req_hdr->scb, sizeof(req_hdr->scb), SCST_NON_ATOMIC);
1218         if (scst_cmd == NULL) {
1219                 create_status_rsp(req, SAM_STAT_BUSY, NULL, 0);
1220                 cmnd_prepare_skip_pdu_set_resid(req);
1221                 goto out;
1222         }
1223
1224         req->scst_cmd = scst_cmd;
1225         scst_cmd_set_tag(scst_cmd, req_hdr->itt);
1226         scst_cmd_set_tgt_priv(scst_cmd, req);
1227 #ifndef NET_PAGE_CALLBACKS_DEFINED
1228         scst_cmd_set_data_buf_tgt_alloc(scst_cmd);
1229 #endif
1230
1231         if (req_hdr->flags & ISCSI_CMD_READ)
1232                 dir = SCST_DATA_READ;
1233         else if (req_hdr->flags & ISCSI_CMD_WRITE)
1234                 dir = SCST_DATA_WRITE;
1235         else
1236                 dir = SCST_DATA_NONE;
1237         scst_cmd_set_expected(scst_cmd, dir, be32_to_cpu(req_hdr->data_length));
1238
1239         switch(req_hdr->flags & ISCSI_CMD_ATTR_MASK) {
1240         case ISCSI_CMD_SIMPLE:
1241                 scst_cmd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1242                 break;
1243         case ISCSI_CMD_HEAD_OF_QUEUE:
1244                 scst_cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1245                 break;
1246         case ISCSI_CMD_ORDERED:
1247                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1248                 break;
1249         case ISCSI_CMD_ACA:
1250                 scst_cmd->queue_type = SCST_CMD_QUEUE_ACA;
1251                 break;
1252         case ISCSI_CMD_UNTAGGED:
1253                 scst_cmd->queue_type = SCST_CMD_QUEUE_UNTAGGED;
1254                 break;
1255         default:
1256                 PRINT_ERROR("Unknown task code %x, use ORDERED instead",
1257                         req_hdr->flags & ISCSI_CMD_ATTR_MASK);
1258                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1259                 break;
1260         }
1261
1262         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
1263         scst_cmd_set_tgt_sn(scst_cmd, req_hdr->cmd_sn);
1264
1265         TRACE_DBG("START Command (tag %d, queue_type %d)",
1266                 req_hdr->itt, scst_cmd->queue_type);
1267         req->scst_state = ISCSI_CMD_STATE_RX_CMD;
1268         scst_cmd_init_stage1_done(scst_cmd, SCST_CONTEXT_DIRECT, 0);
1269
1270         wait_event(req->scst_waitQ, (req->scst_state != ISCSI_CMD_STATE_RX_CMD));
1271
1272         if (unlikely(req->scst_state != ISCSI_CMD_STATE_AFTER_PREPROC)) {
1273                 TRACE_DBG("req %p is in %x state", req, req->scst_state);
1274                 if (req->scst_state == ISCSI_CMD_STATE_PROCESSED) {
1275                         /* Response is already prepared */
1276                         cmnd_prepare_skip_pdu_set_resid(req);
1277                         goto out;
1278                 }
1279                 if (unlikely(req->tmfabort)) {
1280                         TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
1281                                 req->scst_cmd);
1282                         cmnd_prepare_skip_pdu(req);
1283                         goto out;
1284                 }
1285                 sBUG();
1286         }
1287
1288         dir = scst_cmd_get_data_direction(scst_cmd);
1289         if (dir != SCST_DATA_WRITE) {
1290                 if (unlikely(!(req_hdr->flags & ISCSI_CMD_FINAL) ||
1291                              req->pdu.datasize)) {
1292                         PRINT_ERROR("Unexpected unsolicited data (ITT %x "
1293                                 "CDB %x", cmnd_itt(req), req_hdr->scb[0]);
1294                         create_sense_rsp(req, ABORTED_COMMAND, 0xc, 0xc);
1295                         cmnd_prepare_skip_pdu_set_resid(req);
1296                         goto out;
1297                 }
1298         }
1299
1300         if (dir == SCST_DATA_WRITE) {
1301                 req->is_unsolicited_data = !(req_hdr->flags & ISCSI_CMD_FINAL);
1302                 req->r2t_length = be32_to_cpu(req_hdr->data_length) - req->pdu.datasize;
1303                 req->data_waiting = 1;
1304         }
1305         req->target_task_tag = get_next_ttt(conn);
1306         req->sg = scst_cmd_get_sg(scst_cmd);
1307         req->bufflen = scst_cmd_get_bufflen(scst_cmd);
1308         if (unlikely(req->r2t_length > req->bufflen)) {
1309                 PRINT_ERROR("req->r2t_length %d > req->bufflen %d",
1310                         req->r2t_length, req->bufflen);
1311                 req->r2t_length = req->bufflen;
1312         }
1313
1314         TRACE_DBG("req=%p, dir=%d, is_unsolicited_data=%d, "
1315                 "r2t_length=%d, bufflen=%d", req, dir,
1316                 req->is_unsolicited_data, req->r2t_length, req->bufflen);
1317
1318         if (unlikely(!session->sess_param.immediate_data &&
1319                      req->pdu.datasize)) {
1320                 PRINT_ERROR("Initiator %s violated negotiated paremeters: "
1321                         "forbidden immediate data sent (ITT %x, op  %x)",
1322                         session->initiator_name, cmnd_itt(req), req_hdr->scb[0]);
1323                 res = -EINVAL;
1324                 goto out;
1325         }
1326
1327         if (unlikely(session->sess_param.initial_r2t &&
1328                      !(req_hdr->flags & ISCSI_CMD_FINAL))) {
1329                 PRINT_ERROR("Initiator %s violated negotiated paremeters: "
1330                         "initial R2T is required (ITT %x, op  %x)",
1331                         session->initiator_name, cmnd_itt(req), req_hdr->scb[0]);
1332                 res = -EINVAL;
1333                 goto out;
1334         }
1335
1336         if (req->pdu.datasize) {
1337                 if (unlikely(dir != SCST_DATA_WRITE)) {
1338                         PRINT_ERROR("pdu.datasize(%d) >0, but dir(%x) isn't WRITE",
1339                                 req->pdu.datasize, dir);
1340                         create_sense_rsp(req, ABORTED_COMMAND, 0xc, 0xc);
1341                         cmnd_prepare_skip_pdu_set_resid(req);
1342                 } else
1343                         res = cmnd_prepare_recv_pdu(conn, req, 0, req->pdu.datasize);
1344         }
1345 out:
1346         /* Aborted commands will be freed in cmnd_rx_end() */
1347         TRACE_EXIT_RES(res);
1348         return res;
1349 }
1350
1351 static int data_out_start(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
1352 {
1353         struct iscsi_data_out_hdr *req_hdr = (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1354         struct iscsi_cmnd *req = NULL;
1355         u32 offset = be32_to_cpu(req_hdr->buffer_offset);
1356         int res = 0;
1357
1358         TRACE_ENTRY();
1359
1360         /*
1361          * There is no race with send_r2t() and conn_abort(), since
1362          * all functions called from single read thread
1363          */
1364         iscsi_extracheck_is_rd_thread(cmnd->conn);
1365
1366         update_stat_sn(cmnd);
1367
1368         cmnd->cmd_req = req = cmnd_find_hash(conn->session, req_hdr->itt,
1369                                         req_hdr->ttt);
1370         if (unlikely(req == NULL)) {
1371                 /* It might happen if req was aborted and then freed */
1372                 TRACE(TRACE_MGMT_MINOR, "Unable to find scsi task %x %x",
1373                         cmnd_itt(cmnd), cmnd_ttt(cmnd));
1374                 goto skip_pdu;
1375         }
1376
1377         if (req->is_unsolicited_data) {
1378                 if (unlikely(req->r2t_length < cmnd->pdu.datasize)) {
1379                         PRINT_ERROR("Data size (%d) > R2T length (%d)",
1380                                 cmnd->pdu.datasize, req->r2t_length);
1381                         mark_conn_closed(conn);
1382                         res = -EINVAL;
1383                         goto out;
1384                 }
1385                 req->r2t_length -= cmnd->pdu.datasize;
1386         }
1387
1388         /* Check unsolicited burst data */
1389         if (unlikely((req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) &&
1390                      (req->pdu.bhs.flags & ISCSI_FLG_FINAL))) {
1391                 PRINT_ERROR("Unexpected data from %x %x",
1392                         cmnd_itt(cmnd), cmnd_ttt(cmnd));
1393                 mark_conn_closed(conn);
1394                 res = -EINVAL;
1395                 goto out;
1396         }
1397
1398         TRACE_WRITE("%u %p %p %u %u", req_hdr->ttt, cmnd, req,
1399                 offset, cmnd->pdu.datasize);
1400
1401         res = cmnd_prepare_recv_pdu(conn, req, offset, cmnd->pdu.datasize);
1402
1403 out:
1404         TRACE_EXIT_RES(res);
1405         return res;
1406
1407 skip_pdu:
1408         cmnd->pdu.bhs.opcode = ISCSI_OP_DATA_REJECT;
1409         cmnd_prepare_skip_pdu(cmnd);
1410         goto out;
1411 }
1412
1413 static void data_out_end(struct iscsi_cmnd *cmnd)
1414 {
1415         struct iscsi_data_out_hdr *req_hdr = (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1416         struct iscsi_cmnd *req;
1417
1418         sBUG_ON(cmnd == NULL);
1419         req = cmnd->cmd_req;
1420         sBUG_ON(req == NULL);
1421
1422         TRACE_DBG("cmnd %p, req %p", cmnd, req);
1423
1424         iscsi_extracheck_is_rd_thread(cmnd->conn);
1425
1426         if (!(cmnd->conn->ddigest_type & DIGEST_NONE)) {
1427                 cmd_add_on_rx_ddigest_list(req, cmnd);
1428                 cmnd_get(cmnd);
1429         }
1430
1431         if (req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1432                 TRACE_DBG("ISCSI_RESERVED_TAG, FINAL %x",
1433                         req_hdr->flags & ISCSI_FLG_FINAL);
1434
1435                 if (req_hdr->flags & ISCSI_FLG_FINAL) {
1436                         req->is_unsolicited_data = 0;
1437                         if (req->pending)
1438                                 goto out_put;
1439                 } else
1440                         goto out_put;
1441         } else {
1442                 TRACE_DBG("FINAL %x, outstanding_r2t %d, r2t_length %d",
1443                         req_hdr->flags & ISCSI_FLG_FINAL,
1444                         req->outstanding_r2t, req->r2t_length);
1445
1446                 if (req_hdr->flags & ISCSI_FLG_FINAL) {
1447                         if (unlikely(req->is_unsolicited_data)) {
1448                                 PRINT_ERROR("Unexpected unsolicited data "
1449                                         "(r2t_length %u, outstanding_r2t %d)",
1450                                         req->r2t_length, req->is_unsolicited_data);
1451                                 mark_conn_closed(req->conn);
1452                                 goto out_put;
1453                         }
1454                         req->outstanding_r2t--;
1455                 } else
1456                         goto out_put;
1457         }
1458
1459         if (req->r2t_length != 0) {
1460                 if (!req->is_unsolicited_data)
1461                         send_r2t(req);
1462         } else
1463                 iscsi_restart_waiting_cmnd(req);
1464
1465 out_put:
1466         cmnd_put(cmnd);
1467         return;
1468 }
1469
1470 static void __cmnd_abort(struct iscsi_cmnd *cmnd)
1471 {
1472         TRACE_MGMT_DBG("Aborting cmd %p, scst_cmd %p (scst state %x, "
1473                 "ref_cnt %d, itt %x, sn %u, op %x, r2t_len %x, CDB op %x, "
1474                 "size to write %u, is_unsolicited_data %d, "
1475                 "outstanding_r2t %d, data_waiting %d, sess->exp_cmd_sn %u, "
1476                 "conn %p, rd_task %p)", cmnd, cmnd->scst_cmd, cmnd->scst_state,
1477                 atomic_read(&cmnd->ref_cnt), cmnd_itt(cmnd), cmnd->pdu.bhs.sn,
1478                 cmnd_opcode(cmnd), cmnd->r2t_length, cmnd_scsicode(cmnd),
1479                 cmnd_write_size(cmnd), cmnd->is_unsolicited_data,
1480                 cmnd->outstanding_r2t, cmnd->data_waiting,
1481                 cmnd->conn->session->exp_cmd_sn, cmnd->conn,
1482                 cmnd->conn->rd_task);
1483
1484 #ifdef NET_PAGE_CALLBACKS_DEFINED
1485         TRACE_MGMT_DBG("net_ref_cnt %d", atomic_read(&cmnd->net_ref_cnt));
1486 #endif
1487
1488         cmnd->tmfabort = 1;
1489
1490         return;
1491 }
1492
1493 /* Must be called from the read thread */
1494 static int cmnd_abort(struct iscsi_cmnd *req)
1495 {
1496         struct iscsi_session *session = req->conn->session;
1497         struct iscsi_task_mgt_hdr *req_hdr =
1498                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1499         struct iscsi_cmnd *cmnd;
1500         int err;
1501
1502         req_hdr->ref_cmd_sn = be32_to_cpu(req_hdr->ref_cmd_sn);
1503
1504         if (after(req_hdr->ref_cmd_sn, req_hdr->cmd_sn)) {
1505                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) > CmdSN(%u)",
1506                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn);
1507                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1508                 goto out;
1509         }
1510
1511         if ((cmnd = cmnd_find_hash_get(session, req_hdr->rtt, ISCSI_RESERVED_TAG))) {
1512                 struct iscsi_conn *conn = cmnd->conn;
1513                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
1514
1515                 if (req_hdr->lun != hdr->lun) {
1516                          PRINT_ERROR("ABORT TASK: LUN mismatch: req LUN "
1517                                 "%Lx, cmd LUN %Lx, rtt %u", req_hdr->lun,
1518                                 hdr->lun, req_hdr->rtt);
1519                         err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1520                         goto out_put;
1521                 }
1522
1523                 if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
1524                         if (req_hdr->ref_cmd_sn != req_hdr->cmd_sn) {
1525                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != TM cmd "
1526                                         "CmdSN(%u) for immediate command %p",
1527                                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn,
1528                                         cmnd);
1529                                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1530                                 goto out_put;
1531                         }
1532                 } else {
1533                         if (req_hdr->ref_cmd_sn != hdr->cmd_sn) {
1534                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != "
1535                                         "CmdSN(%u) for command %p",
1536                                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn,
1537                                         cmnd);
1538                                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1539                                 goto out_put;
1540                         }
1541                 }
1542
1543                 if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
1544                     (req_hdr->cmd_sn == hdr->cmd_sn)) {
1545                         PRINT_ERROR("ABORT TASK: SN mismatch: req SN %x, "
1546                                 "cmd SN %x, rtt %u", req_hdr->cmd_sn,
1547                                 hdr->cmd_sn, req_hdr->rtt);
1548                         err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1549                         goto out_put;
1550                 }
1551
1552                 spin_lock_bh(&conn->cmd_list_lock);
1553                 __cmnd_abort(cmnd);
1554                 spin_unlock_bh(&conn->cmd_list_lock);
1555
1556                 cmnd_put(cmnd);
1557                 err = 0;
1558         } else
1559                 err = ISCSI_RESPONSE_UNKNOWN_TASK;
1560
1561 out:
1562         return err;
1563
1564 out_put:
1565         cmnd_put(cmnd);
1566         goto out;
1567 }
1568
1569 /* Must be called from the read thread */
1570 static int target_abort(struct iscsi_cmnd *req, int all)
1571 {
1572         struct iscsi_target *target = req->conn->session->target;
1573         struct iscsi_task_mgt_hdr *req_hdr =
1574                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1575         struct iscsi_session *session;
1576         struct iscsi_conn *conn;
1577         struct iscsi_cmnd *cmnd;
1578
1579         mutex_lock(&target->target_mutex);
1580
1581         list_for_each_entry(session, &target->session_list, session_list_entry) {
1582                 list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
1583                         spin_lock_bh(&conn->cmd_list_lock);
1584                         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1585                                 if (cmnd == req)
1586                                         continue;
1587                                 if (all)
1588                                         __cmnd_abort(cmnd);
1589                                 else if (req_hdr->lun == cmnd_hdr(cmnd)->lun)
1590                                         __cmnd_abort(cmnd);
1591                         }
1592                         spin_unlock_bh(&conn->cmd_list_lock);
1593                 }
1594         }
1595
1596         mutex_unlock(&target->target_mutex);
1597         return 0;
1598 }
1599
1600 /* Must be called from the read thread */
1601 static void task_set_abort(struct iscsi_cmnd *req)
1602 {
1603         struct iscsi_session *session = req->conn->session;
1604         struct iscsi_task_mgt_hdr *req_hdr =
1605                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1606         struct iscsi_target *target = session->target;
1607         struct iscsi_conn *conn;
1608         struct iscsi_cmnd *cmnd;
1609
1610         mutex_lock(&target->target_mutex);
1611
1612         list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
1613                 spin_lock_bh(&conn->cmd_list_lock);
1614                 list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1615                         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
1616                         if (cmnd == req)
1617                                 continue;
1618                         if (req_hdr->lun != hdr->lun)
1619                                 continue;
1620                         if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
1621                             req_hdr->cmd_sn == hdr->cmd_sn)
1622                                 continue;
1623                         __cmnd_abort(cmnd);
1624                 }
1625                 spin_unlock_bh(&conn->cmd_list_lock);
1626         }
1627
1628         mutex_unlock(&target->target_mutex);
1629         return;
1630 }
1631
1632 /* Must be called from the read thread */
1633 void conn_abort(struct iscsi_conn *conn)
1634 {
1635         struct iscsi_cmnd *cmnd;
1636
1637         TRACE_MGMT_DBG("Aborting conn %p", conn);
1638
1639         iscsi_extracheck_is_rd_thread(conn);
1640
1641         spin_lock_bh(&conn->cmd_list_lock);
1642 again:
1643         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1644                 __cmnd_abort(cmnd);
1645                 if (cmnd->data_waiting) {
1646                         if (!cmnd_get_check(cmnd)) {
1647                                 spin_unlock_bh(&conn->cmd_list_lock);
1648
1649                                 /* ToDo: this is racy for MC/S */
1650                                 TRACE_MGMT_DBG("Restarting data waiting cmd %p",
1651                                         cmnd);
1652                                 iscsi_fail_waiting_cmnd(cmnd);
1653
1654                                 cmnd_put(cmnd);
1655
1656                                 /*
1657                                  * We are in the read thread, so we may not
1658                                  * worry that after cmnd release conn gets
1659                                  * released as well.
1660                                  */
1661                                 spin_lock_bh(&conn->cmd_list_lock);
1662                                 goto again;
1663                         }
1664                 }
1665         }
1666         spin_unlock_bh(&conn->cmd_list_lock);
1667
1668         return;
1669 }
1670
1671 static void execute_task_management(struct iscsi_cmnd *req)
1672 {
1673         struct iscsi_conn *conn = req->conn;
1674         struct iscsi_session *sess = conn->session;
1675         struct iscsi_task_mgt_hdr *req_hdr =
1676                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1677         int err = 0, function = req_hdr->function & ISCSI_FUNCTION_MASK;
1678         struct scst_rx_mgmt_params params;
1679
1680         TRACE((function == ISCSI_FUNCTION_ABORT_TASK) ? TRACE_MGMT_MINOR : TRACE_MGMT,
1681                 "TM fn %d", function);
1682
1683         TRACE_MGMT_DBG("TM req %p, itt %x, rtt %x, sn %u, con %p", req,
1684                 cmnd_itt(req), req_hdr->rtt, req_hdr->cmd_sn, conn);
1685
1686         iscsi_extracheck_is_rd_thread(conn);
1687
1688         spin_lock(&sess->sn_lock);
1689         sess->tm_active = 1;
1690         sess->tm_sn = req_hdr->cmd_sn;
1691         if (sess->tm_rsp != NULL) {
1692                 struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
1693                 TRACE(TRACE_MGMT_MINOR, "Dropping delayed TM rsp %p", tm_rsp);
1694                 sess->tm_rsp = NULL;
1695                 spin_unlock(&sess->sn_lock);
1696                 rsp_cmnd_release(tm_rsp);
1697         } else
1698                 spin_unlock(&sess->sn_lock);
1699
1700         memset(&params, 0, sizeof(params));
1701         params.atomic = SCST_NON_ATOMIC;
1702         params.tgt_priv = req;
1703
1704         if ((function != ISCSI_FUNCTION_ABORT_TASK) &&
1705             (req_hdr->rtt != ISCSI_RESERVED_TAG)) {
1706                 PRINT_ERROR("Invalid RTT %x (TM fn %x)", req_hdr->rtt,
1707                         function);
1708                 err = -1;
1709                 goto reject;
1710         }
1711
1712         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
1713
1714         switch (function) {
1715         case ISCSI_FUNCTION_ABORT_TASK:
1716                 err = cmnd_abort(req);
1717                 if (err == 0) {
1718                         params.fn = SCST_ABORT_TASK;
1719                         params.tag = req_hdr->rtt;
1720                         params.tag_set = 1;
1721                         params.lun = (uint8_t *)&req_hdr->lun;
1722                         params.lun_len = sizeof(req_hdr->lun);
1723                         params.lun_set = 1;
1724                         params.cmd_sn = req_hdr->cmd_sn;
1725                         params.cmd_sn_set = 1;
1726                         err = scst_rx_mgmt_fn(conn->session->scst_sess,
1727                                 &params);
1728                 }
1729                 break;
1730         case ISCSI_FUNCTION_ABORT_TASK_SET:
1731                 task_set_abort(req);
1732                 params.fn = SCST_ABORT_TASK_SET;
1733                 params.lun = (uint8_t *)&req_hdr->lun;
1734                 params.lun_len = sizeof(req_hdr->lun);
1735                 params.lun_set = 1;
1736                 params.cmd_sn = req_hdr->cmd_sn;
1737                 params.cmd_sn_set = 1;
1738                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1739                         &params);
1740                 break;
1741         case ISCSI_FUNCTION_CLEAR_TASK_SET:
1742                 task_set_abort(req);
1743                 params.fn = SCST_CLEAR_TASK_SET;
1744                 params.lun = (uint8_t *)&req_hdr->lun;
1745                 params.lun_len = sizeof(req_hdr->lun);
1746                 params.lun_set = 1;
1747                 params.cmd_sn = req_hdr->cmd_sn;
1748                 params.cmd_sn_set = 1;
1749                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1750                         &params);
1751                 break;
1752         case ISCSI_FUNCTION_CLEAR_ACA:
1753                 params.fn = SCST_CLEAR_ACA;
1754                 params.lun = (uint8_t *)&req_hdr->lun;
1755                 params.lun_len = sizeof(req_hdr->lun);
1756                 params.lun_set = 1;
1757                 params.cmd_sn = req_hdr->cmd_sn;
1758                 params.cmd_sn_set = 1;
1759                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1760                         &params);
1761                 break;
1762         case ISCSI_FUNCTION_TARGET_COLD_RESET:
1763         case ISCSI_FUNCTION_TARGET_WARM_RESET:
1764                 target_abort(req, 1);
1765                 params.fn = SCST_TARGET_RESET;
1766                 params.cmd_sn = req_hdr->cmd_sn;
1767                 params.cmd_sn_set = 1;
1768                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1769                         &params);
1770                 break;
1771         case ISCSI_FUNCTION_LOGICAL_UNIT_RESET:
1772                 target_abort(req, 0);
1773                 params.fn = SCST_LUN_RESET;
1774                 params.lun = (uint8_t *)&req_hdr->lun;
1775                 params.lun_len = sizeof(req_hdr->lun);
1776                 params.lun_set = 1;
1777                 params.cmd_sn = req_hdr->cmd_sn;
1778                 params.cmd_sn_set = 1;
1779                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1780                         &params);
1781                 break;
1782         case ISCSI_FUNCTION_TASK_REASSIGN:
1783                 iscsi_send_task_mgmt_resp(req, 
1784                         ISCSI_RESPONSE_FUNCTION_UNSUPPORTED);
1785                 break;
1786         default:
1787                 iscsi_send_task_mgmt_resp(req,
1788                         ISCSI_RESPONSE_FUNCTION_REJECTED);
1789                 break;
1790         }
1791
1792 reject:
1793         if (err != 0) {
1794                 iscsi_send_task_mgmt_resp(req,
1795                         ISCSI_RESPONSE_FUNCTION_REJECTED);
1796         }
1797 }
1798
1799 static void noop_out_exec(struct iscsi_cmnd *req)
1800 {
1801         struct iscsi_cmnd *rsp;
1802         struct iscsi_nop_in_hdr *rsp_hdr;
1803
1804         TRACE_DBG("%p", req);
1805
1806         if (cmnd_itt(req) != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1807                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
1808
1809                 rsp_hdr = (struct iscsi_nop_in_hdr *)&rsp->pdu.bhs;
1810                 rsp_hdr->opcode = ISCSI_OP_NOOP_IN;
1811                 rsp_hdr->flags = ISCSI_FLG_FINAL;
1812                 rsp_hdr->itt = req->pdu.bhs.itt;
1813                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
1814
1815                 if (req->pdu.datasize)
1816                         sBUG_ON(req->sg == NULL);
1817                 else
1818                         sBUG_ON(req->sg != NULL);
1819
1820                 if (req->sg) {
1821                         rsp->sg = req->sg;
1822                         rsp->bufflen = req->bufflen;
1823                 }
1824
1825                 sBUG_ON(get_pgcnt(req->pdu.datasize, 0) > ISCSI_CONN_IOV_MAX);
1826                 rsp->pdu.datasize = req->pdu.datasize;
1827                 iscsi_cmnd_init_write(rsp,
1828                         ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
1829                 req_cmnd_release(req);
1830         } else
1831                 cmnd_put(req);
1832 }
1833
1834 static void logout_exec(struct iscsi_cmnd *req)
1835 {
1836         struct iscsi_logout_req_hdr *req_hdr;
1837         struct iscsi_cmnd *rsp;
1838         struct iscsi_logout_rsp_hdr *rsp_hdr;
1839
1840         PRINT_INFO("Logout received from initiator %s",
1841                 req->conn->session->initiator_name);
1842         TRACE_DBG("%p", req);
1843
1844         req_hdr = (struct iscsi_logout_req_hdr *)&req->pdu.bhs;
1845         rsp = iscsi_cmnd_create_rsp_cmnd(req);
1846         rsp_hdr = (struct iscsi_logout_rsp_hdr *)&rsp->pdu.bhs;
1847         rsp_hdr->opcode = ISCSI_OP_LOGOUT_RSP;
1848         rsp_hdr->flags = ISCSI_FLG_FINAL;
1849         rsp_hdr->itt = req_hdr->itt;
1850         rsp->should_close_conn = 1;
1851         iscsi_cmnd_init_write(rsp,
1852                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
1853         req_cmnd_release(req);
1854 }
1855
1856 static void iscsi_cmnd_exec(struct iscsi_cmnd *cmnd)
1857 {
1858         TRACE_DBG("%p,%x,%u", cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn);
1859
1860         if (unlikely(cmnd->tmfabort)) {
1861                 TRACE_MGMT_DBG("cmnd %p (scst_cmd %p) aborted", cmnd,
1862                         cmnd->scst_cmd);
1863                 req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
1864                 goto out;
1865         }
1866
1867         iscsi_extracheck_is_rd_thread(cmnd->conn);
1868
1869         switch (cmnd_opcode(cmnd)) {
1870         case ISCSI_OP_SCSI_CMD:
1871                 if (cmnd->r2t_length != 0) {
1872                         if (!cmnd->is_unsolicited_data) {
1873                                 send_r2t(cmnd);
1874                                 break;
1875                         }
1876                 } else
1877                         iscsi_restart_cmnd(cmnd);
1878                 break;
1879         case ISCSI_OP_NOOP_OUT:
1880                 noop_out_exec(cmnd);
1881                 break;
1882         case ISCSI_OP_SCSI_TASK_MGT_MSG:
1883                 execute_task_management(cmnd);
1884                 break;
1885         case ISCSI_OP_LOGOUT_CMD:
1886                 logout_exec(cmnd);
1887                 break;
1888         case ISCSI_OP_SCSI_REJECT:
1889                 TRACE_MGMT_DBG("REJECT cmnd %p (scst_cmd %p)", cmnd,
1890                         cmnd->scst_cmd);
1891                 iscsi_cmnd_init_write(get_rsp_cmnd(cmnd),
1892                         ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
1893                 req_cmnd_release(cmnd);
1894                 break;
1895         default:
1896                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
1897                 req_cmnd_release(cmnd);
1898                 break;
1899         }
1900 out:
1901         return;
1902 }
1903
1904 static void __cmnd_send_pdu(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd,
1905         u32 offset, u32 size)
1906 {
1907         TRACE_DBG("%p %u,%u,%u", cmnd, offset, size, cmnd->bufflen);
1908
1909         iscsi_extracheck_is_wr_thread(conn);
1910
1911         sBUG_ON(offset > cmnd->bufflen);
1912         sBUG_ON(offset + size > cmnd->bufflen);
1913
1914         conn->write_offset = offset;
1915         conn->write_size += size;
1916 }
1917
1918 static void cmnd_send_pdu(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
1919 {
1920         u32 size;
1921
1922         if (!cmnd->pdu.datasize)
1923                 return;
1924
1925         size = (cmnd->pdu.datasize + 3) & -4;
1926         sBUG_ON(cmnd->sg == NULL);
1927         sBUG_ON(cmnd->bufflen != size);
1928         __cmnd_send_pdu(conn, cmnd, 0, size);
1929 }
1930
1931 static void set_cork(struct socket *sock, int on)
1932 {
1933         int opt = on;
1934         mm_segment_t oldfs;
1935
1936         oldfs = get_fs();
1937         set_fs(get_ds());
1938         sock->ops->setsockopt(sock, SOL_TCP, TCP_CORK, (void *)&opt, sizeof(opt));
1939         set_fs(oldfs);
1940 }
1941
1942 void cmnd_tx_start(struct iscsi_cmnd *cmnd)
1943 {
1944         struct iscsi_conn *conn = cmnd->conn;
1945
1946         TRACE_DBG("%p:%p:%x", conn, cmnd, cmnd_opcode(cmnd));
1947         iscsi_cmnd_set_length(&cmnd->pdu);
1948
1949         iscsi_extracheck_is_wr_thread(conn);
1950
1951         set_cork(conn->sock, 1);
1952
1953         conn->write_iop = conn->write_iov;
1954         conn->write_iop->iov_base = &cmnd->pdu.bhs;
1955         conn->write_iop->iov_len = sizeof(cmnd->pdu.bhs);
1956         conn->write_iop_used = 1;
1957         conn->write_size = sizeof(cmnd->pdu.bhs);
1958
1959         switch (cmnd_opcode(cmnd)) {
1960         case ISCSI_OP_NOOP_IN:
1961                 cmnd_set_sn(cmnd, 1);
1962                 cmnd_send_pdu(conn, cmnd);
1963                 break;
1964         case ISCSI_OP_SCSI_RSP:
1965                 cmnd_set_sn(cmnd, 1);
1966                 cmnd_send_pdu(conn, cmnd);
1967                 break;
1968         case ISCSI_OP_SCSI_TASK_MGT_RSP:
1969                 cmnd_set_sn(cmnd, 1);
1970                 break;
1971         case ISCSI_OP_TEXT_RSP:
1972                 cmnd_set_sn(cmnd, 1);
1973                 break;
1974         case ISCSI_OP_SCSI_DATA_IN:
1975         {
1976                 struct iscsi_data_in_hdr *rsp = (struct iscsi_data_in_hdr *)&cmnd->pdu.bhs;
1977                 u32 offset = cpu_to_be32(rsp->buffer_offset);
1978
1979                 cmnd_set_sn(cmnd, (rsp->flags & ISCSI_FLG_FINAL) ? 1 : 0);
1980                 __cmnd_send_pdu(conn, cmnd, offset, cmnd->pdu.datasize);
1981                 break;
1982         }
1983         case ISCSI_OP_LOGOUT_RSP:
1984                 cmnd_set_sn(cmnd, 1);
1985                 break;
1986         case ISCSI_OP_R2T:
1987                 cmnd->pdu.bhs.sn = cmnd_set_sn(cmnd, 0);
1988                 break;
1989         case ISCSI_OP_ASYNC_MSG:
1990                 cmnd_set_sn(cmnd, 1);
1991                 break;
1992         case ISCSI_OP_REJECT:
1993                 cmnd_set_sn(cmnd, 1);
1994                 cmnd_send_pdu(conn, cmnd);
1995                 break;
1996         default:
1997                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
1998                 break;
1999         }
2000
2001         // move this?
2002         conn->write_size = (conn->write_size + 3) & -4;
2003         iscsi_dump_pdu(&cmnd->pdu);
2004 }
2005
2006 void cmnd_tx_end(struct iscsi_cmnd *cmnd)
2007 {
2008         struct iscsi_conn *conn = cmnd->conn;
2009
2010         TRACE_DBG("%p:%x (should_close_conn %d)", cmnd, cmnd_opcode(cmnd),
2011                 cmnd->should_close_conn);
2012
2013         switch (cmnd_opcode(cmnd)) {
2014         case ISCSI_OP_NOOP_IN:
2015         case ISCSI_OP_SCSI_RSP:
2016         case ISCSI_OP_SCSI_TASK_MGT_RSP:
2017         case ISCSI_OP_TEXT_RSP:
2018         case ISCSI_OP_R2T:
2019         case ISCSI_OP_ASYNC_MSG:
2020         case ISCSI_OP_REJECT:
2021         case ISCSI_OP_SCSI_DATA_IN:
2022         case ISCSI_OP_LOGOUT_RSP:
2023                 break;
2024         default:
2025                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2026                 sBUG();
2027                 break;
2028         }
2029
2030         if (cmnd->should_close_conn) {
2031                 PRINT_INFO("Closing connection at initiator %s request",
2032                         conn->session->initiator_name);
2033                 mark_conn_closed(conn);
2034         }
2035
2036         set_cork(cmnd->conn->sock, 0);
2037 }
2038
2039 /*
2040  * Push the command for execution. This functions reorders the commands.
2041  * Called from the read thread.
2042  */
2043 static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd)
2044 {
2045         struct iscsi_session *session = cmnd->conn->session;
2046         struct list_head *entry;
2047         u32 cmd_sn;
2048
2049         TRACE_DBG("%p:%x %u,%u",
2050                 cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn, session->exp_cmd_sn);
2051
2052         iscsi_extracheck_is_rd_thread(cmnd->conn);
2053
2054         sBUG_ON(cmnd->parent_req != NULL);
2055
2056         if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
2057                 TRACE_DBG("Immediate cmd %p (cmd_sn %u)", cmnd,
2058                         cmnd->pdu.bhs.sn);
2059                 iscsi_cmnd_exec(cmnd);
2060                 goto out;
2061         }
2062
2063         spin_lock(&session->sn_lock);
2064
2065         cmd_sn = cmnd->pdu.bhs.sn;
2066         if (cmd_sn == session->exp_cmd_sn) {
2067                 while (1) {
2068                         session->exp_cmd_sn = ++cmd_sn;
2069
2070                         if (unlikely(session->tm_active)) {
2071                                 if (before(cmd_sn, session->tm_sn)) {
2072                                         struct iscsi_conn *conn = cmnd->conn;
2073
2074                                         spin_unlock(&session->sn_lock);
2075
2076                                         spin_lock_bh(&conn->cmd_list_lock);
2077                                         __cmnd_abort(cmnd);
2078                                         spin_unlock_bh(&conn->cmd_list_lock);
2079
2080                                         spin_lock(&session->sn_lock);
2081                                 }
2082                                 iscsi_check_send_delayed_tm_resp(session);
2083                         }
2084
2085                         spin_unlock(&session->sn_lock);
2086
2087                         iscsi_cmnd_exec(cmnd);
2088
2089                         if (list_empty(&session->pending_list))
2090                                 break;
2091                         cmnd = list_entry(session->pending_list.next, struct iscsi_cmnd,
2092                                                 pending_list_entry);
2093                         if (cmnd->pdu.bhs.sn != cmd_sn)
2094                                 break;
2095
2096                         list_del(&cmnd->pending_list_entry);
2097                         cmnd->pending = 0;
2098
2099                         TRACE_DBG("Processing pending cmd %p (cmd_sn %u)",
2100                                 cmnd, cmd_sn);
2101
2102                         spin_lock(&session->sn_lock);
2103                 }
2104         } else {
2105                 int drop = 0;
2106
2107                 TRACE_DBG("Pending cmd %p (cmd_sn %u, exp_cmd_sn %u)",
2108                         cmnd, cmd_sn, session->exp_cmd_sn);
2109
2110                 /*
2111                  * iSCSI RFC 3720: "The target MUST silently ignore any
2112                  * non-immediate command outside of [from ExpCmdSN to MaxCmdSN
2113                  * inclusive] range". But we won't honor the MaxCmdSN
2114                  * requirement, because, since we adjust MaxCmdSN from the
2115                  * separate write thread, rarery it is possible that initiator
2116                  * can legally send command with CmdSN>MaxSN. But it won't
2117                  * hurt anything, in the worst case it will lead to
2118                  * additional QUEUE FULL status.
2119                  */
2120
2121                 if (unlikely(before(cmd_sn, session->exp_cmd_sn))) {
2122                         PRINT_ERROR("Unexpected cmd_sn (%u,%u)", cmd_sn,
2123                                 session->exp_cmd_sn);
2124                         drop = 1;
2125                 }
2126
2127 #if 0
2128                 if (unlikely(after(cmd_sn, session->exp_cmd_sn +
2129                                         iscsi_get_allowed_cmds(session)))) {
2130                         TRACE_MGMT_DBG("Too large cmd_sn %u (exp_cmd_sn %u, "
2131                                 "max_sn %u)", cmd_sn, session->exp_cmd_sn,
2132                                 iscsi_get_allowed_cmds(session));
2133                 }
2134 #endif
2135
2136                 spin_unlock(&session->sn_lock);
2137
2138                 if (unlikely(drop)) {
2139                         req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
2140                         goto out;
2141                 }
2142
2143                 if (unlikely(cmnd->tmfabort)) {
2144                         struct iscsi_cmnd *tm_clone;
2145
2146                         TRACE_MGMT_DBG("Pending aborted cmnd %p, creating TM "
2147                                 "clone (scst cmd %p, state %d)", cmnd,
2148                                 cmnd->scst_cmd, cmnd->scst_state);
2149
2150                         tm_clone = cmnd_alloc(cmnd->conn, NULL);
2151                         if (tm_clone != NULL) {
2152                                 tm_clone->tmfabort = 1;
2153                                 tm_clone->pdu = cmnd->pdu;
2154
2155                                 TRACE_MGMT_DBG("TM clone %p created", tm_clone);
2156
2157                                 iscsi_cmnd_exec(cmnd);
2158                                 cmnd = tm_clone;
2159                         } else
2160                                 PRINT_ERROR("%s", "Unable to create TM clone");
2161                 }
2162
2163                 list_for_each(entry, &session->pending_list) {
2164                         struct iscsi_cmnd *tmp = list_entry(entry, struct iscsi_cmnd,
2165                                                         pending_list_entry);
2166                         if (before(cmd_sn, tmp->pdu.bhs.sn))
2167                                 break;
2168                 }
2169
2170                 list_add_tail(&cmnd->pending_list_entry, entry);
2171                 cmnd->pending = 1;
2172         }
2173 out:
2174         return;
2175 }
2176
2177 static int check_segment_length(struct iscsi_cmnd *cmnd)
2178 {
2179         struct iscsi_conn *conn = cmnd->conn;
2180         struct iscsi_session *session = conn->session;
2181
2182         if (unlikely(cmnd->pdu.datasize > session->sess_param.max_recv_data_length)) {
2183                 PRINT_ERROR("Initiator %s violated negotiated parameters: "
2184                         "data too long (ITT %x, datasize %u, "
2185                         "max_recv_data_length %u", session->initiator_name,
2186                         cmnd_itt(cmnd), cmnd->pdu.datasize,
2187                         session->sess_param.max_recv_data_length);
2188                 mark_conn_closed(conn);
2189                 return -EINVAL;
2190         }
2191         return 0;
2192 }
2193
2194 int cmnd_rx_start(struct iscsi_cmnd *cmnd)
2195 {
2196         struct iscsi_conn *conn = cmnd->conn;
2197         int res, rc;
2198
2199         iscsi_dump_pdu(&cmnd->pdu);
2200
2201         res = check_segment_length(cmnd);
2202         if (res != 0)
2203                 goto out;
2204
2205         switch (cmnd_opcode(cmnd)) {
2206         case ISCSI_OP_NOOP_OUT:
2207                 rc = noop_out_start(cmnd);
2208                 break;
2209         case ISCSI_OP_SCSI_CMD:
2210                 rc = cmnd_insert_hash(cmnd);
2211                 if (likely(rc == 0)) {
2212                         res = scsi_cmnd_start(cmnd);
2213                         if (unlikely(res != 0))
2214                                 goto out;
2215                 }
2216                 break;
2217         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2218                 rc = cmnd_insert_hash(cmnd);
2219                 break;
2220         case ISCSI_OP_SCSI_DATA_OUT:
2221                 res = data_out_start(conn, cmnd);
2222                 rc = 0; /* to avoid compiler warning */
2223                 if (unlikely(res != 0))
2224                         goto out;
2225                 break;
2226         case ISCSI_OP_LOGOUT_CMD:
2227                 rc = cmnd_insert_hash(cmnd);
2228                 break;
2229         case ISCSI_OP_TEXT_CMD:
2230         case ISCSI_OP_SNACK_CMD:
2231                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2232                 break;
2233         default:
2234                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2235                 break;
2236         }
2237
2238         if (unlikely(rc < 0)) {
2239                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
2240                 PRINT_ERROR("Error %d (iSCSI opcode %x, ITT %x, op %x)", rc,
2241                         cmnd_opcode(cmnd), cmnd_itt(cmnd),
2242                         (cmnd_opcode(cmnd) == ISCSI_OP_SCSI_CMD ?
2243                                 hdr->scb[0] : -1));
2244                 iscsi_cmnd_reject(cmnd, -rc);
2245         }
2246
2247 out:
2248         TRACE_EXIT_RES(res);
2249         return res;
2250 }
2251
2252 void cmnd_rx_end(struct iscsi_cmnd *cmnd)
2253 {
2254         TRACE_ENTRY();
2255
2256         TRACE_DBG("%p:%x", cmnd, cmnd_opcode(cmnd));
2257
2258         switch (cmnd_opcode(cmnd)) {
2259         case ISCSI_OP_SCSI_CMD:
2260         case ISCSI_OP_SCSI_REJECT:
2261         case ISCSI_OP_NOOP_OUT:
2262         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2263         case ISCSI_OP_LOGOUT_CMD:
2264                 iscsi_session_push_cmnd(cmnd);
2265                 break;
2266         case ISCSI_OP_SCSI_DATA_OUT:
2267                 data_out_end(cmnd);
2268                 break;
2269         case ISCSI_OP_PDU_REJECT:
2270                 iscsi_cmnd_init_write(get_rsp_cmnd(cmnd),
2271                         ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2272                 req_cmnd_release(cmnd);
2273                 break;
2274         case ISCSI_OP_DATA_REJECT:
2275                 req_cmnd_release(cmnd);
2276                 break;
2277         default:
2278                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2279                 req_cmnd_release(cmnd);
2280                 break;
2281         }
2282
2283         TRACE_EXIT();
2284         return;
2285 }
2286
2287 #ifndef NET_PAGE_CALLBACKS_DEFINED
2288 static int iscsi_alloc_data_buf(struct scst_cmd *cmd)
2289 {
2290         if (scst_cmd_get_data_direction(cmd) == SCST_DATA_READ) {
2291                 /*
2292                  * sock->ops->sendpage() is async zero copy operation,
2293                  * so we must be sure not to free and reuse
2294                  * the command's buffer before the sending was completed
2295                  * by the network layers. It is possible only if we
2296                  * don't use SGV cache.
2297                  */
2298                 scst_cmd_set_no_sgv(cmd);
2299         }
2300         return 1;
2301 }
2302 #endif
2303
2304 static inline void iscsi_set_state_wake_up(struct iscsi_cmnd *req,
2305         int new_state)
2306 {
2307         /*
2308          * We use wait_event() to wait for the state change, but it checks its
2309          * condition without any protection, so without cmnd_get() it is
2310          * possible that req will die "immediately" after the state assignment
2311          * and wake_up() will operate on dead data.
2312          */
2313         cmnd_get_ordered(req);
2314         req->scst_state = new_state;
2315         wake_up(&req->scst_waitQ);
2316         cmnd_put(req);
2317         return;
2318 }
2319
2320 static void iscsi_preprocessing_done(struct scst_cmd *scst_cmd)
2321 {
2322         struct iscsi_cmnd *req = (struct iscsi_cmnd*)
2323                                 scst_cmd_get_tgt_priv(scst_cmd);
2324
2325         TRACE_DBG("req %p", req);
2326
2327         EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_RX_CMD);
2328
2329         iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_AFTER_PREPROC);
2330         return;
2331 }
2332
2333 /* 
2334  * No locks.
2335  *
2336  * IMPORTANT! Connection conn must be protected by additional conn_get()
2337  * upon entrance in this function, because otherwise it could be destroyed
2338  * inside as a result of iscsi_send(), which releases sent commands.
2339  */
2340 static void iscsi_try_local_processing(struct iscsi_conn *conn)
2341 {
2342         int local;
2343
2344         TRACE_ENTRY();
2345
2346         spin_lock_bh(&iscsi_wr_lock);
2347         switch(conn->wr_state) {
2348         case ISCSI_CONN_WR_STATE_IN_LIST:
2349                 list_del(&conn->wr_list_entry);
2350                 /* go through */
2351         case ISCSI_CONN_WR_STATE_IDLE:
2352 #ifdef EXTRACHECKS
2353                 conn->wr_task = current;
2354 #endif
2355                 conn->wr_state = ISCSI_CONN_WR_STATE_PROCESSING;
2356                 conn->wr_space_ready = 0;
2357                 local = 1;
2358                 break;
2359         default:
2360                 local = 0;
2361                 break;
2362         }
2363         spin_unlock_bh(&iscsi_wr_lock);
2364
2365         if (local) {
2366                 int rc = 1;
2367                 while(test_write_ready(conn)) {
2368                         rc = iscsi_send(conn);
2369                         if (rc <= 0) {
2370                                 break;
2371                         }
2372                 }
2373
2374                 spin_lock_bh(&iscsi_wr_lock);
2375 #ifdef EXTRACHECKS
2376                 conn->wr_task = NULL;
2377 #endif
2378                 if ((rc <= 0) || test_write_ready(conn)) {
2379                         list_add_tail(&conn->wr_list_entry, &iscsi_wr_list);
2380                         conn->wr_state = ISCSI_CONN_WR_STATE_IN_LIST;
2381                         wake_up(&iscsi_wr_waitQ);
2382                 } else
2383                         conn->wr_state = ISCSI_CONN_WR_STATE_IDLE;
2384                 spin_unlock_bh(&iscsi_wr_lock);
2385         }
2386
2387         TRACE_EXIT();
2388         return;
2389 }
2390
2391 static int iscsi_xmit_response(struct scst_cmd *scst_cmd)
2392 {
2393         int resp_flags = scst_cmd_get_tgt_resp_flags(scst_cmd);
2394         struct iscsi_cmnd *req = (struct iscsi_cmnd*)
2395                                         scst_cmd_get_tgt_priv(scst_cmd);
2396         struct iscsi_conn *conn = req->conn;
2397         int status = scst_cmd_get_status(scst_cmd);
2398         u8 *sense = scst_cmd_get_sense_buffer(scst_cmd);
2399         int sense_len = scst_cmd_get_sense_buffer_len(scst_cmd);
2400         int old_state = req->scst_state;
2401
2402         scst_cmd_set_tgt_priv(scst_cmd, NULL);
2403
2404         req->tmfabort |= scst_cmd_aborted(scst_cmd) ? 1 : 0;
2405         if (unlikely(req->tmfabort)) {
2406                 TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
2407                         req->scst_cmd);
2408                 if (old_state == ISCSI_CMD_STATE_RESTARTED) {
2409                         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
2410                         req_cmnd_release_force(req, ISCSI_FORCE_RELEASE_WRITE);
2411                 } else
2412                         iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_PROCESSED);
2413                 goto out;
2414         }
2415
2416         if (unlikely(old_state != ISCSI_CMD_STATE_RESTARTED)) {
2417                 TRACE_DBG("req %p on %d state", req, old_state);
2418                 create_status_rsp(req, status, sense, sense_len);
2419                 switch(old_state) {
2420                 case ISCSI_CMD_STATE_RX_CMD:
2421                 case ISCSI_CMD_STATE_AFTER_PREPROC:
2422                         break;
2423                 default:
2424                         sBUG();
2425                 }
2426                 iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_PROCESSED);
2427                 goto out;
2428         }
2429
2430         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
2431
2432         req->bufflen = scst_cmd_get_resp_data_len(scst_cmd);
2433         req->sg = scst_cmd_get_sg(scst_cmd);
2434
2435         TRACE_DBG("req %p, resp_flags=%x, req->bufflen=%d, req->sg=%p", req,
2436                 resp_flags, req->bufflen, req->sg);
2437
2438         if (unlikely((req->bufflen != 0) &&
2439                      !(resp_flags & SCST_TSC_FLAG_STATUS))) {
2440                 PRINT_ERROR("%s", "Sending DATA without STATUS is unsupported");
2441                 scst_set_cmd_error(scst_cmd,
2442                         SCST_LOAD_SENSE(scst_sense_hardw_error));
2443                 resp_flags = scst_cmd_get_tgt_resp_flags(scst_cmd);
2444                 sBUG();
2445         }
2446
2447         if (req->bufflen != 0) {
2448                 /* 
2449                  * Check above makes sure that SCST_TSC_FLAG_STATUS is set,
2450                  * so status is valid here, but in future that could change.
2451                  * ToDo
2452                  */
2453                 if (status != SAM_STAT_CHECK_CONDITION) {
2454                         send_data_rsp(req, status,
2455                                 resp_flags & SCST_TSC_FLAG_STATUS);
2456                 } else {
2457                         struct iscsi_cmnd *rsp;
2458                         struct iscsi_scsi_rsp_hdr *rsp_hdr;
2459                         int resid;
2460                         send_data_rsp(req, 0, 0);
2461                         if (resp_flags & SCST_TSC_FLAG_STATUS) {
2462                                 rsp = create_status_rsp(req, status, sense,
2463                                         sense_len);
2464                                 rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
2465                                 resid = cmnd_read_size(req) - req->bufflen;
2466                                 if (resid > 0) {
2467                                         rsp_hdr->flags |=
2468                                                 ISCSI_FLG_RESIDUAL_UNDERFLOW;
2469                                         rsp_hdr->residual_count = cpu_to_be32(resid);
2470                                 } else if (resid < 0) {
2471                                         rsp_hdr->flags |=
2472                                                 ISCSI_FLG_RESIDUAL_OVERFLOW;
2473                                         rsp_hdr->residual_count = cpu_to_be32(-resid);
2474                                 }
2475                                 iscsi_cmnd_init_write(rsp,
2476                                         ISCSI_INIT_WRITE_REMOVE_HASH);
2477                         }
2478                 }
2479         } else if (resp_flags & SCST_TSC_FLAG_STATUS) {
2480                 struct iscsi_cmnd *rsp;
2481                 struct iscsi_scsi_rsp_hdr *rsp_hdr;
2482                 u32 resid;
2483                 rsp = create_status_rsp(req, status, sense, sense_len);
2484                 rsp_hdr = (struct iscsi_scsi_rsp_hdr *) &rsp->pdu.bhs;
2485                 resid = cmnd_read_size(req);
2486                 if (resid != 0) {
2487                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
2488                         rsp_hdr->residual_count = cpu_to_be32(resid);
2489                 }
2490                 iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH);
2491         }
2492 #ifdef EXTRACHECKS
2493         else
2494                 sBUG();
2495 #endif
2496
2497         conn_get_ordered(conn);
2498         req_cmnd_release(req);
2499         iscsi_try_local_processing(conn);
2500         conn_put(conn);
2501
2502 out:
2503         return SCST_TGT_RES_SUCCESS;
2504 }
2505
2506 /* Called under sn_lock */
2507 static bool iscsi_is_delay_tm_resp(struct iscsi_cmnd *rsp)
2508 {
2509         bool res = 0;
2510         struct iscsi_task_mgt_hdr *req_hdr =
2511                 (struct iscsi_task_mgt_hdr *)&rsp->parent_req->pdu.bhs;
2512         int function = req_hdr->function & ISCSI_FUNCTION_MASK;
2513         struct iscsi_session *sess = rsp->conn->session;
2514
2515         TRACE_ENTRY();
2516
2517         /* This should be checked for immediate TM commands as well */
2518
2519         switch(function) {
2520         default:
2521                 if (before(sess->exp_cmd_sn, req_hdr->cmd_sn))
2522                         res = 1;
2523                 break;
2524         }
2525
2526         TRACE_EXIT_RES(res);
2527         return res;
2528 }
2529
2530 /* Called under sn_lock, but might drop it inside, then reaquire */
2531 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess)
2532 {
2533         struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
2534
2535         TRACE_ENTRY();
2536
2537         if (tm_rsp == NULL)
2538                 goto out;
2539
2540         if (iscsi_is_delay_tm_resp(tm_rsp))
2541                 goto out;
2542
2543         TRACE(TRACE_MGMT_MINOR, "Sending delayed rsp %p", tm_rsp);
2544
2545         sess->tm_rsp = NULL;
2546         sess->tm_active = 0;
2547
2548         spin_unlock(&sess->sn_lock);
2549
2550         iscsi_cmnd_init_write(tm_rsp,
2551                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2552
2553         spin_lock(&sess->sn_lock);
2554
2555 out:
2556         TRACE_EXIT();
2557         return;
2558 }
2559
2560 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status)
2561 {
2562         struct iscsi_cmnd *rsp;
2563         struct iscsi_task_mgt_hdr *req_hdr =
2564                                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2565         struct iscsi_task_rsp_hdr *rsp_hdr;
2566         struct iscsi_session *sess = req->conn->session;
2567         int fn = req_hdr->function & ISCSI_FUNCTION_MASK;
2568
2569         TRACE_ENTRY();
2570
2571         TRACE_MGMT_DBG("TM req %p finished", req);
2572         TRACE((req_hdr->function == ISCSI_FUNCTION_ABORT_TASK) ?
2573                          TRACE_MGMT_MINOR : TRACE_MGMT,
2574                 "TM fn %d finished, status %d", fn, status);
2575
2576         rsp = iscsi_cmnd_create_rsp_cmnd(req);
2577         rsp_hdr = (struct iscsi_task_rsp_hdr *)&rsp->pdu.bhs;
2578
2579         rsp_hdr->opcode = ISCSI_OP_SCSI_TASK_MGT_RSP;
2580         rsp_hdr->flags = ISCSI_FLG_FINAL;
2581         rsp_hdr->itt = req_hdr->itt;
2582         rsp_hdr->response = status;
2583
2584         if (fn == ISCSI_FUNCTION_TARGET_COLD_RESET)
2585                 rsp->should_close_conn = 1;
2586
2587         sBUG_ON(sess->tm_rsp != NULL);
2588
2589         spin_lock(&sess->sn_lock);
2590         if (iscsi_is_delay_tm_resp(rsp)) {
2591                 TRACE(TRACE_MGMT_MINOR, "Delaying TM fn %x response %p "
2592                         "(req %p), because not all affected commands received "
2593                         "(TM cmd sn %u, exp sn %u)",
2594                         req_hdr->function & ISCSI_FUNCTION_MASK, rsp, req,
2595                         req_hdr->cmd_sn, sess->exp_cmd_sn);
2596                 sess->tm_rsp = rsp;
2597                 spin_unlock(&sess->sn_lock);
2598                 goto out_release;
2599         }
2600         sess->tm_active = 0;
2601         spin_unlock(&sess->sn_lock);
2602
2603         iscsi_cmnd_init_write(rsp,
2604                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2605
2606 out_release:
2607         req_cmnd_release(req);
2608
2609         TRACE_EXIT();
2610         return;
2611 }
2612
2613 static inline int iscsi_get_mgmt_response(int status)
2614 {
2615         switch(status) {
2616         case SCST_MGMT_STATUS_SUCCESS:
2617                 return ISCSI_RESPONSE_FUNCTION_COMPLETE;
2618
2619         case SCST_MGMT_STATUS_TASK_NOT_EXIST:
2620                 return ISCSI_RESPONSE_UNKNOWN_TASK;
2621
2622         case SCST_MGMT_STATUS_LUN_NOT_EXIST:
2623                 return ISCSI_RESPONSE_UNKNOWN_LUN;
2624
2625         case SCST_MGMT_STATUS_FN_NOT_SUPPORTED:
2626                 return ISCSI_RESPONSE_FUNCTION_UNSUPPORTED;
2627
2628         case SCST_MGMT_STATUS_REJECTED:
2629         case SCST_MGMT_STATUS_FAILED:
2630         default:
2631                 return ISCSI_RESPONSE_FUNCTION_REJECTED;
2632         }
2633 }
2634
2635 static void iscsi_task_mgmt_fn_done(struct scst_mgmt_cmd *scst_mcmd)
2636 {
2637         struct iscsi_cmnd *req = (struct iscsi_cmnd*)
2638                                 scst_mgmt_cmd_get_tgt_priv(scst_mcmd);
2639         int status = iscsi_get_mgmt_response(scst_mgmt_cmd_get_status(scst_mcmd));
2640
2641         TRACE_MGMT_DBG("req %p, scst_mcmd %p, scst status %d", req, scst_mcmd, 
2642                 scst_mgmt_cmd_get_status(scst_mcmd));
2643
2644         iscsi_send_task_mgmt_resp(req, status);
2645
2646         scst_mgmt_cmd_set_tgt_priv(scst_mcmd, NULL);
2647 }
2648
2649 static int iscsi_target_detect(struct scst_tgt_template *templ)
2650 {
2651         /* Nothing to do */
2652         return 0;
2653 }
2654
2655 static int iscsi_target_release(struct scst_tgt *scst_tgt)
2656 {
2657         /* Nothing to do */
2658         return 0;
2659 }
2660
2661 struct scst_tgt_template iscsi_template = {
2662         .name = "iscsi",
2663         .sg_tablesize = ISCSI_CONN_IOV_MAX,
2664         .threads_num = 0,
2665         .no_clustering = 1,
2666         .xmit_response_atomic = 0,
2667         .preprocessing_done_atomic = 1,
2668         .detect = iscsi_target_detect,
2669         .release = iscsi_target_release,
2670         .xmit_response = iscsi_xmit_response,
2671 #ifndef NET_PAGE_CALLBACKS_DEFINED
2672         .alloc_data_buf = iscsi_alloc_data_buf,
2673 #endif
2674         .preprocessing_done = iscsi_preprocessing_done,
2675         .pre_exec = iscsi_pre_exec,
2676         .task_mgmt_fn_done = iscsi_task_mgmt_fn_done,
2677 };
2678
2679 static __init int iscsi_run_threads(int count, char *name, int (*fn)(void *))
2680 {
2681         int res = 0;
2682         int i;
2683         struct iscsi_thread_t *thr;
2684
2685         for (i = 0; i < count; i++) {
2686                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
2687                 if (!thr) {
2688                         res = -ENOMEM;
2689                         PRINT_ERROR("Failed to allocate thr %d", res);
2690                         goto out;
2691                 }
2692                 thr->thr = kthread_run(fn, NULL, "%s%d", name, i);
2693                 if (IS_ERR(thr->thr)) {
2694                         res = PTR_ERR(thr->thr);
2695                         PRINT_ERROR("kthread_create() failed: %d", res);
2696                         kfree(thr);
2697                         goto out;
2698                 }
2699                 list_add(&thr->threads_list_entry, &iscsi_threads_list);
2700         }
2701
2702 out:
2703         return res;
2704 }
2705
2706 static void iscsi_stop_threads(void)
2707 {
2708         struct iscsi_thread_t *t, *tmp;
2709
2710         list_for_each_entry_safe(t, tmp, &iscsi_threads_list,
2711                                 threads_list_entry) {
2712                 int rc = kthread_stop(t->thr);
2713                 if (rc < 0) {
2714                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
2715                 }
2716                 list_del(&t->threads_list_entry);
2717                 kfree(t);
2718         }
2719 }
2720
2721 static int __init iscsi_init(void)
2722 {
2723         int err;
2724         int num;
2725
2726         PRINT_INFO("iSCSI SCST Target - version %s", ISCSI_VERSION_STRING);
2727
2728 #ifdef NET_PAGE_CALLBACKS_DEFINED
2729         err = net_set_get_put_page_callbacks(iscsi_get_page_callback,
2730                         iscsi_put_page_callback);
2731         if (err != 0) {
2732                 PRINT_INFO("Unable to set page callbackes: %d", err);
2733                 goto out;
2734         }
2735 #else
2736         PRINT_INFO("%s", "Patch put_page_callback-<kernel-version>.patch "
2737                 "not applied on your kernel. Running in the performance "
2738                 "degraded mode. Refer README file for details");
2739 #endif
2740
2741         BUILD_BUG_ON(MAX_DATA_SEG_LEN != (ISCSI_CONN_IOV_MAX<<PAGE_SHIFT));
2742
2743         if ((ctr_major = register_chrdev(0, ctr_name, &ctr_fops)) < 0) {
2744                 PRINT_ERROR("failed to register the control device %d", ctr_major);
2745                 err = ctr_major;
2746                 goto out_callb;
2747         }
2748
2749         if ((err = event_init()) < 0)
2750                 goto out_reg;
2751
2752         iscsi_cmnd_cache = KMEM_CACHE(iscsi_cmnd, SCST_SLAB_FLAGS);
2753         if (!iscsi_cmnd_cache) {
2754                 err = -ENOMEM;
2755                 goto out_event;
2756         }
2757
2758         if (scst_register_target_template(&iscsi_template) < 0) {
2759                 err = -ENODEV;
2760                 goto out_kmem;
2761         }
2762         iscsi_template_registered = 1;
2763
2764         if ((err = iscsi_procfs_init()) < 0)
2765                 goto out_reg_tmpl;
2766
2767         num = max(num_online_cpus(), 2);
2768
2769         err = iscsi_run_threads(num, "iscsird", istrd);
2770         if (err != 0)
2771                 goto out_thr;
2772
2773         err = iscsi_run_threads(num, "iscsiwr", istwr);
2774         if (err != 0)
2775                 goto out_thr;
2776
2777 out:
2778         return err;
2779
2780 out_thr:
2781         iscsi_procfs_exit();
2782         iscsi_stop_threads();
2783
2784 out_reg_tmpl:
2785         scst_unregister_target_template(&iscsi_template);
2786
2787 out_kmem:
2788         kmem_cache_destroy(iscsi_cmnd_cache);
2789
2790 out_event:
2791         event_exit();
2792
2793 out_reg:
2794         unregister_chrdev(ctr_major, ctr_name);
2795
2796 out_callb:
2797 #ifdef NET_PAGE_CALLBACKS_DEFINED
2798         net_set_get_put_page_callbacks(NULL, NULL);
2799 #endif  
2800         goto out;
2801 }
2802
2803 static void __exit iscsi_exit(void)
2804 {
2805         iscsi_stop_threads();
2806
2807         unregister_chrdev(ctr_major, ctr_name);
2808
2809         iscsi_procfs_exit();
2810         event_exit();
2811
2812         kmem_cache_destroy(iscsi_cmnd_cache);
2813
2814         scst_unregister_target_template(&iscsi_template);
2815
2816 #ifdef NET_PAGE_CALLBACKS_DEFINED
2817         net_set_get_put_page_callbacks(NULL, NULL);
2818 #endif
2819 }
2820
2821 module_init(iscsi_init);
2822 module_exit(iscsi_exit);
2823
2824 MODULE_LICENSE("GPL");