A bunch of pending changes:
[mirror/scst/.git] / iscsi-scst / kernel / iscsi.c
1 /*
2  *  Copyright (C) 2002-2003 Ardis Technolgies <roman@ardistech.com>
3  *  Copyright (C) 2007 Vladislav Bolkhovitin
4  *  Copyright (C) 2007 CMS Distribution Limited
5  * 
6  *  This program is free software; you can redistribute it and/or
7  *  modify it under the terms of the GNU General Public License
8  *  as published by the Free Software Foundation, version 2
9  *  of the License.
10  * 
11  *  This program is distributed in the hope that it will be useful,
12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  *  GNU General Public License for more details.
15  */
16
17 #include <linux/module.h>
18 #include <linux/hash.h>
19 #include <linux/kthread.h>
20 #include <net/tcp.h>
21 #include <scsi/scsi.h>
22
23 #include "iscsi.h"
24 #include "digest.h"
25
26 #ifndef NET_PAGE_CALLBACKS_DEFINED
27 #warning Patch put_page_callback-<kernel-version>.patch not applied on your \
28         kernel. ISCSI-SCST will run in the performance degraded mode. Refer \
29         README file for details.
30 #endif
31
32 #define ISCSI_INIT_WRITE_WAKE           0x1
33 #define ISCSI_INIT_WRITE_REMOVE_HASH    0x2
34
35 static int ctr_major;
36 static char ctr_name[] = "iscsi-scst-ctl";
37 static int iscsi_template_registered;
38
39 #if defined(DEBUG) || defined(TRACING)
40 unsigned long iscsi_trace_flag = ISCSI_DEFAULT_LOG_FLAGS;
41 #endif
42
43 static struct kmem_cache *iscsi_cmnd_cache;
44
45 spinlock_t iscsi_rd_lock = SPIN_LOCK_UNLOCKED;
46 LIST_HEAD(iscsi_rd_list);
47 DECLARE_WAIT_QUEUE_HEAD(iscsi_rd_waitQ);
48
49 spinlock_t iscsi_wr_lock = SPIN_LOCK_UNLOCKED;
50 LIST_HEAD(iscsi_wr_list);
51 DECLARE_WAIT_QUEUE_HEAD(iscsi_wr_waitQ);
52
53 static char dummy_data[1024];
54
55 struct iscsi_thread_t {
56         struct task_struct *thr;
57         struct list_head threads_list_entry;
58 };
59
60 static LIST_HEAD(iscsi_threads_list);
61
62 static void cmnd_remove_hash(struct iscsi_cmnd *cmnd);
63 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status);
64 static void cmnd_prepare_skip_pdu(struct iscsi_cmnd *cmnd);
65 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess);
66 static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd);
67
68 static inline u32 cmnd_write_size(struct iscsi_cmnd *cmnd)
69 {
70         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
71
72         if (hdr->flags & ISCSI_CMD_WRITE)
73                 return be32_to_cpu(hdr->data_length);
74         return 0;
75 }
76
77 static inline u32 cmnd_read_size(struct iscsi_cmnd *cmnd)
78 {
79         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
80
81         if (hdr->flags & ISCSI_CMD_READ) {
82                 struct iscsi_rlength_ahdr *ahdr =
83                         (struct iscsi_rlength_ahdr *)cmnd->pdu.ahs;
84
85                 if (!(hdr->flags & ISCSI_CMD_WRITE))
86                         return be32_to_cpu(hdr->data_length);
87                 if (ahdr && ahdr->ahstype == ISCSI_AHSTYPE_RLENGTH)
88                         return be32_to_cpu(ahdr->read_length);
89         }
90         return 0;
91 }
92
93 static inline void iscsi_restart_cmnd(struct iscsi_cmnd *cmnd)
94 {
95         EXTRACHECKS_BUG_ON(cmnd->data_waiting);
96
97         cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
98         scst_restart_cmd(cmnd->scst_cmd, SCST_PREPROCESS_STATUS_SUCCESS,
99                 SCST_CONTEXT_THREAD);
100 }
101
102 static inline void iscsi_restart_waiting_cmnd(struct iscsi_cmnd *cmnd)
103 {
104         /*
105          * There is no race with conn_abort(), since all functions
106          * called from single read thread
107          */
108         iscsi_extracheck_is_rd_thread(cmnd->conn);
109         cmnd->data_waiting = 0;
110
111         iscsi_restart_cmnd(cmnd);
112 }
113
114 static inline void iscsi_fail_waiting_cmnd(struct iscsi_cmnd *cmnd)
115 {
116         TRACE_DBG("Failing data waiting cmd %p", cmnd);
117
118         /*
119          * There is no race with conn_abort(), since all functions
120          * called from single read thread
121          */
122         iscsi_extracheck_is_rd_thread(cmnd->conn);
123         cmnd->data_waiting = 0;
124
125         req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
126 }
127
128 struct iscsi_cmnd *cmnd_alloc(struct iscsi_conn *conn, struct iscsi_cmnd *parent)
129 {
130         struct iscsi_cmnd *cmnd;
131
132         /* ToDo: __GFP_NOFAIL?? */
133 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,17)
134         cmnd = kmem_cache_alloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
135         memset(cmnd, 0, sizeof(*cmnd));
136 #else
137         cmnd = kmem_cache_zalloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
138 #endif
139
140         atomic_set(&cmnd->ref_cnt, 1);
141         cmnd->scst_state = ISCSI_CMD_STATE_NEW;
142         cmnd->conn = conn;
143         cmnd->parent_req = parent;
144         init_waitqueue_head(&cmnd->scst_waitQ);
145
146         if (parent == NULL) {
147                 conn_get(conn);
148
149 #ifdef NET_PAGE_CALLBACKS_DEFINED
150                 atomic_set(&cmnd->net_ref_cnt, 0);
151 #endif          
152                 cmnd->target = conn->target;
153                 spin_lock_init(&cmnd->rsp_cmd_lock);
154                 INIT_LIST_HEAD(&cmnd->rsp_cmd_list);
155                 INIT_LIST_HEAD(&cmnd->rx_ddigest_cmd_list);
156
157                 spin_lock_bh(&conn->cmd_list_lock);
158                 list_add_tail(&cmnd->cmd_list_entry, &conn->cmd_list);
159                 spin_unlock_bh(&conn->cmd_list_lock);
160         }
161
162         TRACE_DBG("conn %p, parent %p, cmnd %p", conn, parent, cmnd);
163         return cmnd;
164 }
165
166 /* Frees a command. Also frees the additional header. */
167 void cmnd_free(struct iscsi_cmnd *cmnd)
168 {
169         TRACE_DBG("%p", cmnd);
170
171         if (unlikely(cmnd->tmfabort)) {
172                 TRACE_MGMT_DBG("Free aborted cmd %p (scst cmd %p, state %d, "
173                         "parent_req %p)", cmnd, cmnd->scst_cmd, cmnd->scst_state,
174                         cmnd->parent_req);
175         }
176
177         /* Catch users from cmd_list or rsp_cmd_list */
178         EXTRACHECKS_BUG_ON(atomic_read(&cmnd->ref_cnt) != 0);
179
180         kfree(cmnd->pdu.ahs);
181
182         if (unlikely(cmnd->on_write_list || cmnd->on_written_list)) {
183                 struct iscsi_scsi_cmd_hdr *req = cmnd_hdr(cmnd);
184
185                 PRINT_ERROR("cmnd %p still on some list?, %x, %x, %x, %x, %x, %x, %x",
186                         cmnd, req->opcode, req->scb[0], req->flags, req->itt,
187                         be32_to_cpu(req->data_length),
188                         req->cmd_sn, be32_to_cpu(cmnd->pdu.datasize));
189
190                 if (unlikely(cmnd->parent_req)) {
191                         struct iscsi_scsi_cmd_hdr *req =
192                                         cmnd_hdr(cmnd->parent_req);
193                         PRINT_ERROR("%p %x %u", req, req->opcode, req->scb[0]);
194                 }
195                 sBUG();
196         }
197
198         kmem_cache_free(iscsi_cmnd_cache, cmnd);
199         return;
200 }
201
202 void cmnd_done(struct iscsi_cmnd *cmnd)
203 {
204         TRACE_DBG("%p", cmnd);
205
206         if (unlikely(cmnd->tmfabort)) {
207                 TRACE_MGMT_DBG("Done aborted cmd %p (scst cmd %p, state %d, "
208                         "parent_req %p)", cmnd, cmnd->scst_cmd, cmnd->scst_state,
209                         cmnd->parent_req);
210         }
211
212         EXTRACHECKS_BUG_ON(cmnd->on_rx_digest_list);
213
214         if (cmnd->on_written_list) {
215                 struct iscsi_conn *conn = cmnd->conn;
216                 TRACE_DBG("Deleting cmd %p from conn %p written_list", cmnd,
217                         conn);
218                 spin_lock_bh(&conn->write_list_lock);
219                 list_del(&cmnd->write_list_entry);
220                 cmnd->on_written_list = 0;
221                 spin_unlock_bh(&conn->write_list_lock);
222         }
223
224         if (cmnd->parent_req == NULL) {
225                 struct iscsi_conn *conn = cmnd->conn;
226                 TRACE_DBG("Deleting req %p from conn %p", cmnd, conn);
227
228                 spin_lock_bh(&conn->cmd_list_lock);
229                 list_del(&cmnd->cmd_list_entry);
230                 spin_unlock_bh(&conn->cmd_list_lock);
231
232                 conn_put(conn);
233
234                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rsp_cmd_list));
235                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rx_ddigest_cmd_list));
236
237                 /* Order between above and below code is important! */
238
239                 if (cmnd->scst_cmd) {
240                         switch(cmnd->scst_state) {
241                         case ISCSI_CMD_STATE_AFTER_PREPROC:
242                         {
243                                 struct scst_cmd *scst_cmd = cmnd->scst_cmd;
244                                 TRACE_DBG("cmd %p AFTER_PREPROC", cmnd);
245                                 cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
246                                 cmnd->scst_cmd = NULL;
247                                 scst_restart_cmd(scst_cmd,
248                                         SCST_PREPROCESS_STATUS_ERROR_FATAL,
249                                         SCST_CONTEXT_THREAD);
250                                 break;
251                         }
252                         case ISCSI_CMD_STATE_PROCESSED:
253                                 TRACE_DBG("cmd %p PROCESSED", cmnd);
254                                 scst_tgt_cmd_done(cmnd->scst_cmd);
255                                 break;
256                         default:
257                                 PRINT_ERROR("Unexpected cmnd scst state %d",
258                                         cmnd->scst_state);
259                                 sBUG();
260                                 break;
261                         }
262                 }
263         } else {
264                 EXTRACHECKS_BUG_ON(cmnd->scst_cmd != NULL);
265                 TRACE_DBG("Deleting rsp %p from parent %p", cmnd,
266                         cmnd->parent_req);
267
268                 spin_lock_bh(&cmnd->parent_req->rsp_cmd_lock);
269                 list_del(&cmnd->rsp_cmd_list_entry);
270                 spin_unlock_bh(&cmnd->parent_req->rsp_cmd_lock);
271
272                 cmnd_put(cmnd->parent_req);
273         }
274
275         /* Order between above and below code is important! */
276
277         if (cmnd->own_sg) {
278                 TRACE_DBG("%s", "own_sg");
279                 scst_free(cmnd->sg, cmnd->sg_cnt);
280 #ifdef DEBUG
281                 cmnd->own_sg = 0;
282                 cmnd->sg = NULL;
283                 cmnd->sg_cnt = -1;
284 #endif
285         }
286
287         if (cmnd->dec_active_cmnds) {
288                 struct iscsi_session *sess = cmnd->conn->session;
289                 TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
290                         "new value %d)", cmnd, sess,
291                         atomic_read(&sess->active_cmds)-1);
292                 atomic_dec(&sess->active_cmds);
293         }
294
295         cmnd_free(cmnd);
296         return;
297 }
298
299 /*
300  * Corresponding conn may also gets destroyed atfer this function, except only
301  * if it's called from the read thread!
302  *
303  * It can't be called in parallel with iscsi_cmnds_init_write()!
304  */
305 void req_cmnd_release_force(struct iscsi_cmnd *req, int flags)
306 {
307         struct iscsi_cmnd *rsp, *t;
308         struct iscsi_conn *conn = req->conn;
309         LIST_HEAD(cmds_list);
310
311         TRACE_ENTRY();
312
313         TRACE_DBG("%p", req);
314
315         sBUG_ON(req == conn->read_cmnd);
316
317         if (flags & ISCSI_FORCE_RELEASE_WRITE) {
318                 spin_lock_bh(&conn->write_list_lock);
319                 list_for_each_entry_safe(rsp, t, &conn->write_list,
320                                                 write_list_entry) {
321                         if (rsp->parent_req != req)
322                                 continue;
323
324                         cmd_del_from_write_list(rsp);
325
326                         list_add_tail(&rsp->write_list_entry, &cmds_list);
327                 }
328                 spin_unlock_bh(&conn->write_list_lock);
329
330                 list_for_each_entry_safe(rsp, t, &cmds_list, write_list_entry) {
331                         list_del(&rsp->write_list_entry);
332                         cmnd_put(rsp);
333                 }
334         }
335
336 again_rsp:
337         spin_lock_bh(&req->rsp_cmd_lock);
338         list_for_each_entry_reverse(rsp, &req->rsp_cmd_list, rsp_cmd_list_entry) {
339                 bool r;
340
341                 if (rsp->force_cleanup_done)
342                         continue;
343
344                 rsp->force_cleanup_done = 1;
345
346                 if (cmnd_get_check(rsp))
347                         continue;
348
349                 spin_unlock_bh(&req->rsp_cmd_lock);
350
351                 spin_lock_bh(&conn->write_list_lock);
352                 r = rsp->on_write_list || rsp->write_processing_started;
353                 spin_unlock_bh(&conn->write_list_lock);
354
355                 cmnd_put(rsp);
356
357                 if (r)
358                         goto again_rsp;
359
360                 /*
361                  * If both on_write_list and write_processing_started not set,
362                  * we can safely put() rsp.
363                  */
364                 cmnd_put(rsp);
365                 goto again_rsp;
366         }
367         spin_unlock_bh(&req->rsp_cmd_lock);
368
369         req_cmnd_release(req);
370
371         TRACE_EXIT();
372         return;
373 }
374
375 /*
376  * Corresponding conn may also gets destroyed atfer this function, except only
377  * if it's called from the read thread! 
378  */
379 void req_cmnd_release(struct iscsi_cmnd *req)
380 {
381         struct iscsi_cmnd *c, *t;
382
383         TRACE_ENTRY();
384
385         TRACE_DBG("%p", req);
386
387 #ifdef EXTRACHECKS
388         sBUG_ON(req->release_called);
389         req->release_called = 1;
390 #endif
391
392         if (unlikely(req->tmfabort)) {
393                 TRACE_MGMT_DBG("Release aborted req cmd %p (scst cmd %p, "
394                         "state %d)", req, req->scst_cmd, req->scst_state);
395         }
396
397         sBUG_ON(req->parent_req != NULL);
398
399         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
400                                 rx_ddigest_cmd_list_entry) {
401                 cmd_del_from_rx_ddigest_list(c);
402                 cmnd_put(c);
403         }
404
405         if (req->hashed)
406                 cmnd_remove_hash(req);
407
408         if (req->dec_active_cmnds) {
409                 struct iscsi_session *sess = req->conn->session;
410                 TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
411                         "new value %d)", req, sess,
412                         atomic_read(&sess->active_cmds)-1);
413                 atomic_dec(&sess->active_cmds);
414                 req->dec_active_cmnds = 0;
415         }
416
417         cmnd_put(req);
418
419         TRACE_EXIT();
420         return;
421 }
422
423 /*
424  * Corresponding conn may also gets destroyed atfer this function, except only
425  * if it's called from the read thread! 
426  */
427 void rsp_cmnd_release(struct iscsi_cmnd *cmnd)
428 {
429         TRACE_DBG("%p", cmnd);
430
431 #ifdef EXTRACHECKS
432         sBUG_ON(cmnd->release_called);
433         cmnd->release_called = 1;
434 #endif
435
436         sBUG_ON(cmnd->hashed);
437         sBUG_ON(cmnd->parent_req == NULL);
438
439         cmnd_put(cmnd);
440         return;
441 }
442
443 /**
444  * create a new command used as response.
445  *
446  * iscsi_cmnd_create_rsp_cmnd - 
447  * @cmnd: ptr to request command
448  *
449  * @return    ptr to response command or NULL
450  */
451 static struct iscsi_cmnd *iscsi_cmnd_create_rsp_cmnd(struct iscsi_cmnd *parent)
452 {
453         struct iscsi_cmnd *rsp;
454
455         rsp = cmnd_alloc(parent->conn, parent);
456
457         spin_lock_bh(&parent->rsp_cmd_lock);
458         TRACE_DBG("Adding rsp %p to parent %p", rsp, parent);
459         list_add_tail(&rsp->rsp_cmd_list_entry, &parent->rsp_cmd_list);
460         spin_unlock_bh(&parent->rsp_cmd_lock);
461         cmnd_get(parent);
462         return rsp;
463 }
464
465 static inline struct iscsi_cmnd *get_rsp_cmnd(struct iscsi_cmnd *req)
466 {
467         struct iscsi_cmnd *res = NULL;
468
469         /* Currently this lock isn't needed, but just in case.. */
470         spin_lock_bh(&req->rsp_cmd_lock);
471         if (!list_empty(&req->rsp_cmd_list)) {
472                 res = list_entry(req->rsp_cmd_list.prev, struct iscsi_cmnd,
473                         rsp_cmd_list_entry);
474         }
475         spin_unlock_bh(&req->rsp_cmd_lock);
476
477         return res;
478 }
479
480 static void iscsi_cmnds_init_write(struct list_head *send, int flags)
481 {
482         struct iscsi_cmnd *rsp = list_entry(send->next, struct iscsi_cmnd, 
483                                                 write_list_entry);
484         struct iscsi_conn *conn = rsp->conn;
485         struct list_head *pos, *next;
486
487         sBUG_ON(list_empty(send));
488
489         /*
490          * If we don't remove hashed req cmd from the hash list here, before
491          * submitting it for transmittion, we will have a race, when for
492          * some reason cmd's release is delayed after transmittion and
493          * initiator sends cmd with the same ITT => this command will be
494          * erroneously rejected as a duplicate.
495          */
496         if ((flags & ISCSI_INIT_WRITE_REMOVE_HASH) && rsp->parent_req->hashed &&
497             (rsp->parent_req->r2t_length == 0) &&
498             (rsp->parent_req->outstanding_r2t == 0))
499                 cmnd_remove_hash(rsp->parent_req);
500
501         if (!(conn->ddigest_type & DIGEST_NONE)) {
502                 list_for_each(pos, send) {
503                         rsp = list_entry(pos, struct iscsi_cmnd,
504                                                 write_list_entry);
505
506                         if (rsp->pdu.datasize != 0) {
507                                 TRACE_DBG("Doing data digest (%p:%x)", rsp,
508                                         cmnd_opcode(rsp));
509                                 digest_tx_data(rsp);
510                         }
511                 }
512         }
513
514         spin_lock_bh(&conn->write_list_lock);
515         list_for_each_safe(pos, next, send) {
516                 rsp = list_entry(pos, struct iscsi_cmnd, write_list_entry);
517
518                 TRACE_DBG("%p:%x", rsp, cmnd_opcode(rsp));
519
520                 sBUG_ON(conn != rsp->conn);
521
522                 list_del(&rsp->write_list_entry);
523                 cmd_add_on_write_list(conn, rsp);
524         }
525         spin_unlock_bh(&conn->write_list_lock);
526
527         if (flags & ISCSI_INIT_WRITE_WAKE)
528                 iscsi_make_conn_wr_active(conn);
529 }
530
531 static void iscsi_cmnd_init_write(struct iscsi_cmnd *rsp, int flags)
532 {
533         LIST_HEAD(head);
534
535         if (unlikely(rsp->on_write_list)) {
536                 PRINT_ERROR("cmd already on write list (%x %x %x %x %u %u "
537                         "%u %u %u %u %u %d %d",
538                         cmnd_itt(rsp), cmnd_ttt(rsp), cmnd_opcode(rsp),
539                         cmnd_scsicode(rsp), rsp->r2t_sn,
540                         rsp->r2t_length, rsp->is_unsolicited_data,
541                         rsp->target_task_tag, rsp->outstanding_r2t,
542                         rsp->hdigest, rsp->ddigest,
543                         list_empty(&rsp->rsp_cmd_list), rsp->hashed);
544                 sBUG();
545         }
546         list_add(&rsp->write_list_entry, &head);
547         iscsi_cmnds_init_write(&head, flags);
548 }
549
550 static void iscsi_set_datasize(struct iscsi_cmnd *cmnd, u32 offset, u32 size)
551 {
552         cmnd->pdu.datasize = size;
553
554         if (cmnd->pdu.datasize & 3) {
555                 int idx = (offset + cmnd->pdu.datasize) >> PAGE_SHIFT;
556                 u8 *p = (u8 *)page_address(sg_page(&cmnd->sg[idx])) + 
557                         ((offset + cmnd->pdu.datasize) & ~PAGE_MASK);
558                 int i = 4 - (cmnd->pdu.datasize & 3);
559                 while (i--) 
560                     *p++ = 0;
561         }
562 }
563
564 static void send_data_rsp(struct iscsi_cmnd *req, u8 status, int send_status)
565 {
566         struct iscsi_cmnd *rsp;
567         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
568         struct iscsi_data_in_hdr *rsp_hdr;
569         u32 pdusize, expsize, scsisize, size, offset, sn;
570         LIST_HEAD(send);
571
572         TRACE_DBG("req %p", req);
573         pdusize = req->conn->session->sess_param.max_xmit_data_length;
574         expsize = cmnd_read_size(req);
575         size = min(expsize, (u32)req->bufflen);
576         offset = 0;
577         sn = 0;
578
579         while (1) {
580                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
581                 TRACE_DBG("rsp %p", rsp);
582                 rsp->sg = req->sg;
583                 rsp->bufflen = req->bufflen;
584                 rsp_hdr = (struct iscsi_data_in_hdr *)&rsp->pdu.bhs;
585
586                 rsp_hdr->opcode = ISCSI_OP_SCSI_DATA_IN;
587                 rsp_hdr->itt = req_hdr->itt;
588                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
589                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
590                 rsp_hdr->data_sn = cpu_to_be32(sn);
591
592                 if (size <= pdusize) {
593                         iscsi_set_datasize(rsp, offset, size);
594                         if (send_status) {
595                                 TRACE_DBG("status %x", status);
596                                 rsp_hdr->flags =
597                                         ISCSI_FLG_FINAL | ISCSI_FLG_STATUS;
598                                 rsp_hdr->cmd_status = status;
599                         }
600                         scsisize = req->bufflen;
601                         if (scsisize < expsize) {
602                                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
603                                 size = expsize - scsisize;
604                         } else if (scsisize > expsize) {
605                                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
606                                 size = scsisize - expsize;
607                         } else
608                                 size = 0;
609                         rsp_hdr->residual_count = cpu_to_be32(size);
610                         list_add_tail(&rsp->write_list_entry, &send);
611                         break;
612                 }
613
614                 iscsi_set_datasize(rsp, offset, pdusize);
615
616                 size -= pdusize;
617                 offset += pdusize;
618                 sn++;
619
620                 list_add_tail(&rsp->write_list_entry, &send);
621         }
622         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_REMOVE_HASH);
623 }
624
625 static struct iscsi_cmnd *create_status_rsp(struct iscsi_cmnd *req, int status,
626         const u8 *sense_buf, int sense_len)
627 {
628         struct iscsi_cmnd *rsp;
629         struct iscsi_scsi_rsp_hdr *rsp_hdr;
630         struct iscsi_sense_data *sense;
631         struct scatterlist *sg;
632
633         rsp = iscsi_cmnd_create_rsp_cmnd(req);
634         TRACE_DBG("%p", rsp);
635
636         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
637         rsp_hdr->opcode = ISCSI_OP_SCSI_RSP;
638         rsp_hdr->flags = ISCSI_FLG_FINAL;
639         rsp_hdr->response = ISCSI_RESPONSE_COMMAND_COMPLETED;
640         rsp_hdr->cmd_status = status;
641         rsp_hdr->itt = cmnd_hdr(req)->itt;
642
643         if (SCST_SENSE_VALID(sense_buf)) {
644                 TRACE_DBG("%s", "SENSE VALID");
645                 /* ToDo: __GFP_NOFAIL ?? */
646                 sg = rsp->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
647                                         &rsp->sg_cnt);
648                 if (sg == NULL) {
649                         /* ToDo(); */
650                 }
651                 rsp->own_sg = 1;
652                 sense = (struct iscsi_sense_data *)page_address(sg_page(&sg[0]));
653                 sense->length = cpu_to_be16(sense_len);
654                 memcpy(sense->data, sense_buf, sense_len);
655                 rsp->pdu.datasize = sizeof(struct iscsi_sense_data) + sense_len;
656                 rsp->bufflen = (rsp->pdu.datasize + 3) & -4;
657                 if (rsp->bufflen - rsp->pdu.datasize) {
658                     int i = rsp->pdu.datasize;
659                     u8 *p = (u8 *)sense + i;
660                     
661                     while (i < rsp->bufflen) {
662                         *p ++ = 0;
663                         i++;
664                     }
665                 }
666         } else {
667                 rsp->pdu.datasize = 0;
668                 rsp->bufflen = 0;
669         }
670
671         return rsp;
672 }
673
674 static struct iscsi_cmnd *create_sense_rsp(struct iscsi_cmnd *req,
675         u8 sense_key, u8 asc, u8 ascq)
676 {
677         u8 sense[14];
678         memset(sense, 0, sizeof(sense));
679         sense[0] = 0xf0;
680         sense[2] = sense_key;
681         sense[7] = 6;   // Additional sense length
682         sense[12] = asc;
683         sense[13] = ascq;
684         return create_status_rsp(req, SAM_STAT_CHECK_CONDITION, sense,
685                 sizeof(sense));
686 }
687
688 static void iscsi_cmnd_reject(struct iscsi_cmnd *req, int reason)
689 {
690         struct iscsi_cmnd *rsp;
691         struct iscsi_reject_hdr *rsp_hdr;
692         struct scatterlist *sg;
693         char *addr;
694
695         TRACE_MGMT_DBG("Reject: req %p, reason %x", req, reason);
696
697         rsp = iscsi_cmnd_create_rsp_cmnd(req);
698         rsp_hdr = (struct iscsi_reject_hdr *)&rsp->pdu.bhs;
699
700         rsp_hdr->opcode = ISCSI_OP_REJECT;
701         rsp_hdr->ffffffff = ISCSI_RESERVED_TAG;
702         rsp_hdr->reason = reason;
703
704         /* ToDo: __GFP_NOFAIL ?? */
705         sg = rsp->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
706                                 &rsp->sg_cnt);
707         if (sg == NULL) {
708                 /* ToDo(); */
709         }
710         rsp->own_sg = 1;
711         addr = page_address(sg_page(&sg[0]));
712         clear_page(addr);
713         memcpy(addr, &req->pdu.bhs, sizeof(struct iscsi_hdr));
714         rsp->bufflen = rsp->pdu.datasize = sizeof(struct iscsi_hdr);
715         cmnd_prepare_skip_pdu(req);
716
717         req->pdu.bhs.opcode = ISCSI_OP_PDU_REJECT;
718 }
719
720 static inline int iscsi_get_allowed_cmds(struct iscsi_session *sess)
721 {
722         int res = max(-1, (int)sess->max_queued_cmnds - 
723                                 atomic_read(&sess->active_cmds)-1);
724         TRACE_DBG("allowed cmds %d (sess %p, active_cmds %d)", res,
725                 sess, atomic_read(&sess->active_cmds));
726         return res;
727 }
728
729 static u32 cmnd_set_sn(struct iscsi_cmnd *cmnd, int set_stat_sn)
730 {
731         struct iscsi_conn *conn = cmnd->conn;
732         struct iscsi_session *sess = conn->session;
733         u32 res;
734
735         spin_lock(&sess->sn_lock);
736
737         if (set_stat_sn)
738                 cmnd->pdu.bhs.sn = cpu_to_be32(conn->stat_sn++);
739         cmnd->pdu.bhs.exp_sn = cpu_to_be32(sess->exp_cmd_sn);
740         cmnd->pdu.bhs.max_sn = cpu_to_be32(sess->exp_cmd_sn +
741                                  iscsi_get_allowed_cmds(sess));
742
743         res = cpu_to_be32(conn->stat_sn);
744
745         spin_unlock(&sess->sn_lock);
746         return res;
747 }
748
749 /* Called under sn_lock */
750 static void __update_stat_sn(struct iscsi_cmnd *cmnd)
751 {
752         struct iscsi_conn *conn = cmnd->conn;
753         u32 exp_stat_sn;
754
755         cmnd->pdu.bhs.exp_sn = exp_stat_sn = be32_to_cpu(cmnd->pdu.bhs.exp_sn);
756         TRACE_DBG("%x,%x", cmnd_opcode(cmnd), exp_stat_sn);
757         if ((int)(exp_stat_sn - conn->exp_stat_sn) > 0 &&
758             (int)(exp_stat_sn - conn->stat_sn) <= 0) {
759                 // free pdu resources
760                 cmnd->conn->exp_stat_sn = exp_stat_sn;
761         }
762 }
763
764 static inline void update_stat_sn(struct iscsi_cmnd *cmnd)
765 {
766         spin_lock(&cmnd->conn->session->sn_lock);
767         __update_stat_sn(cmnd);
768         spin_unlock(&cmnd->conn->session->sn_lock);
769 }
770
771 /* Called under sn_lock */
772 static int check_cmd_sn(struct iscsi_cmnd *cmnd)
773 {
774         struct iscsi_session *session = cmnd->conn->session;
775         u32 cmd_sn;
776
777         cmnd->pdu.bhs.sn = cmd_sn = be32_to_cpu(cmnd->pdu.bhs.sn);
778         TRACE_DBG("%d(%d)", cmd_sn, session->exp_cmd_sn);
779         if (likely((s32)(cmd_sn - session->exp_cmd_sn) >= 0))
780                 return 0;
781         PRINT_ERROR("sequence error (%x,%x)", cmd_sn, session->exp_cmd_sn);
782         return -ISCSI_REASON_PROTOCOL_ERROR;
783 }
784
785 static inline struct iscsi_cmnd *__cmnd_find_hash(struct iscsi_session *session,
786         u32 itt, u32 ttt)
787 {
788         struct list_head *head;
789         struct iscsi_cmnd *cmnd;
790
791         head = &session->cmnd_hash[cmnd_hashfn(itt)];
792
793         list_for_each_entry(cmnd, head, hash_list_entry) {
794                 if (cmnd->pdu.bhs.itt == itt) {
795                         if ((ttt != ISCSI_RESERVED_TAG) && (ttt != cmnd->target_task_tag))
796                                 continue;
797                         return cmnd;
798                 }
799         }
800         return NULL;
801 }
802
803 static struct iscsi_cmnd *cmnd_find_hash(struct iscsi_session *session,
804         u32 itt, u32 ttt)
805 {
806         struct iscsi_cmnd *cmnd;
807
808         spin_lock(&session->cmnd_hash_lock);
809         cmnd = __cmnd_find_hash(session, itt, ttt);
810         spin_unlock(&session->cmnd_hash_lock);
811
812         return cmnd;
813 }
814
815 static struct iscsi_cmnd *cmnd_find_hash_get(struct iscsi_session *session,
816         u32 itt, u32 ttt)
817 {
818         struct iscsi_cmnd *cmnd;
819
820         spin_lock(&session->cmnd_hash_lock);
821         cmnd = __cmnd_find_hash(session, itt, ttt);
822         if (cmnd != NULL) {
823                 if (unlikely(cmnd_get_check(cmnd)))
824                         cmnd = NULL;
825         }
826         spin_unlock(&session->cmnd_hash_lock);
827
828         return cmnd;
829 }
830
831 static int cmnd_insert_hash(struct iscsi_cmnd *cmnd)
832 {
833         struct iscsi_session *session = cmnd->conn->session;
834         struct iscsi_cmnd *tmp;
835         struct list_head *head;
836         int err = 0;
837         u32 itt = cmnd->pdu.bhs.itt;
838
839         TRACE_DBG("%p:%x", cmnd, itt);
840         if (unlikely(itt == ISCSI_RESERVED_TAG)) {
841                 PRINT_ERROR("%s", "ITT is RESERVED_TAG");
842                 PRINT_BUFFER("Incorrect BHS", &cmnd->pdu.bhs,
843                         sizeof(cmnd->pdu.bhs));
844                 err = -ISCSI_REASON_PROTOCOL_ERROR;
845                 goto out;
846         }
847
848         spin_lock(&session->cmnd_hash_lock);
849
850         head = &session->cmnd_hash[cmnd_hashfn(cmnd->pdu.bhs.itt)];
851
852         tmp = __cmnd_find_hash(session, itt, ISCSI_RESERVED_TAG);
853         if (likely(!tmp)) {
854                 list_add_tail(&cmnd->hash_list_entry, head);
855                 cmnd->hashed = 1;
856         } else {
857                 PRINT_ERROR("Task %x in progress, cmnd %p", itt, cmnd);
858                 err = -ISCSI_REASON_TASK_IN_PROGRESS;
859         }
860
861         spin_unlock(&session->cmnd_hash_lock);
862
863         if (likely(!err)) {
864                 spin_lock(&session->sn_lock);
865                 __update_stat_sn(cmnd);
866                 err = check_cmd_sn(cmnd);
867                 spin_unlock(&session->sn_lock);
868         }
869
870 out:
871         return err;
872 }
873
874 static void cmnd_remove_hash(struct iscsi_cmnd *cmnd)
875 {
876         struct iscsi_session *session = cmnd->conn->session;
877         struct iscsi_cmnd *tmp;
878
879         spin_lock(&session->cmnd_hash_lock);
880
881         tmp = __cmnd_find_hash(session, cmnd->pdu.bhs.itt, ISCSI_RESERVED_TAG);
882
883         if (likely(tmp && tmp == cmnd)) {
884                 list_del(&cmnd->hash_list_entry);
885                 cmnd->hashed = 0;
886         } else {
887                 PRINT_ERROR("%p:%x not found", cmnd, cmnd_itt(cmnd));
888         }
889
890         spin_unlock(&session->cmnd_hash_lock);
891 }
892
893 static void cmnd_prepare_skip_pdu(struct iscsi_cmnd *cmnd)
894 {
895         struct iscsi_conn *conn = cmnd->conn;
896         struct scatterlist *sg = cmnd->sg;
897         char *addr;
898         u32 size;
899         int i;
900
901         TRACE_MGMT_DBG("Skipping (%p, %x %x %x %u, %p, scst state %d)", cmnd,
902                 cmnd_itt(cmnd), cmnd_opcode(cmnd), cmnd_hdr(cmnd)->scb[0],
903                 cmnd->pdu.datasize, cmnd->scst_cmd, cmnd->scst_state);
904
905         iscsi_extracheck_is_rd_thread(conn);
906
907         if (!(size = cmnd->pdu.datasize))
908                 return;
909
910         if (sg == NULL) {
911                 /* ToDo: __GFP_NOFAIL ?? */
912                 sg = cmnd->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
913                                         &cmnd->sg_cnt);
914                 if (sg == NULL) {
915                         /* ToDo(); */
916                 }
917                 cmnd->own_sg = 1;
918                 cmnd->bufflen = PAGE_SIZE;
919         }
920
921         addr = page_address(sg_page(&sg[0]));
922         sBUG_ON(addr == NULL);
923         size = (size + 3) & -4;
924         conn->read_size = size;
925         for (i = 0; size > PAGE_SIZE; i++, size -= cmnd->bufflen) {
926                 sBUG_ON(i >= ISCSI_CONN_IOV_MAX);
927                 conn->read_iov[i].iov_base = addr;
928                 conn->read_iov[i].iov_len = cmnd->bufflen;
929         }
930         conn->read_iov[i].iov_base = addr;
931         conn->read_iov[i].iov_len = size;
932         conn->read_msg.msg_iov = conn->read_iov;
933         conn->read_msg.msg_iovlen = ++i;
934 }
935
936 static void cmnd_prepare_skip_pdu_set_resid(struct iscsi_cmnd *req)
937 {
938         struct iscsi_cmnd *rsp;
939         struct iscsi_scsi_rsp_hdr *rsp_hdr;
940         u32 size;
941
942         TRACE_DBG("%p", req);
943
944         rsp = get_rsp_cmnd(req);
945         if (rsp == NULL)
946                 goto skip;
947
948         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
949
950         sBUG_ON(cmnd_opcode(rsp) != ISCSI_OP_SCSI_RSP);
951
952         size = cmnd_write_size(req);
953         if (size) {
954                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
955                 rsp_hdr->residual_count = cpu_to_be32(size);
956         }
957         size = cmnd_read_size(req);
958         if (size) {
959                 if (cmnd_hdr(req)->flags & ISCSI_CMD_WRITE) {
960                         rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_UNDERFLOW;
961                         rsp_hdr->bi_residual_count = cpu_to_be32(size);
962                 } else {
963                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
964                         rsp_hdr->residual_count = cpu_to_be32(size);
965                 }
966         }
967
968 skip:
969         req->pdu.bhs.opcode =
970                 (req->pdu.bhs.opcode & ~ISCSI_OPCODE_MASK) | ISCSI_OP_SCSI_REJECT;
971
972         cmnd_prepare_skip_pdu(req);
973 }
974
975 static int cmnd_prepare_recv_pdu(struct iscsi_conn *conn,
976         struct iscsi_cmnd *cmd, u32 offset, u32 size)
977 {
978         struct scatterlist *sg = cmd->sg;
979         int bufflen = cmd->bufflen;
980         int idx, i;
981         char *addr;
982         int res = 0;
983
984         TRACE_DBG("%p %u,%u", cmd->sg, offset, size);
985
986         iscsi_extracheck_is_rd_thread(conn);
987
988         if (unlikely((offset >= bufflen) ||
989                      (offset + size > bufflen))) {
990                 PRINT_ERROR("Wrong ltn (%u %u %u)", offset, size, bufflen);
991                 mark_conn_closed(conn);
992                 res = -EIO;
993                 goto out;
994         }
995
996         offset += sg[0].offset;
997         idx = offset >> PAGE_SHIFT;
998         offset &= ~PAGE_MASK;
999
1000         conn->read_msg.msg_iov = conn->read_iov;
1001         conn->read_size = size = (size + 3) & -4;
1002
1003         i = 0;
1004         while (1) {
1005                 addr = page_address(sg_page(&sg[idx]));
1006                 sBUG_ON(addr == NULL);
1007                 conn->read_iov[i].iov_base = addr + offset;
1008                 if (offset + size <= PAGE_SIZE) {
1009                         TRACE_DBG("idx=%d, offset=%u, size=%d, addr=%p",
1010                                 idx, offset, size, addr);
1011                         conn->read_iov[i].iov_len = size;
1012                         conn->read_msg.msg_iovlen = ++i;
1013                         break;
1014                 }
1015                 conn->read_iov[i].iov_len = PAGE_SIZE - offset;
1016                 TRACE_DBG("idx=%d, offset=%u, size=%d, iov_len=%d, addr=%p",
1017                         idx, offset, size, conn->read_iov[i].iov_len, addr);
1018                 size -= conn->read_iov[i].iov_len;
1019                 offset = 0;
1020                 if (unlikely(++i >= ISCSI_CONN_IOV_MAX)) {
1021                         PRINT_ERROR("Initiator %s violated negotiated "
1022                                 "parameters by sending too much data (size "
1023                                 "left %d)", conn->session->initiator_name, size);
1024                         mark_conn_closed(conn);
1025                         res = -EINVAL;
1026                         break;
1027                 }
1028                 idx++;
1029         }
1030         TRACE_DBG("msg_iov=%p, msg_iovlen=%d",
1031                 conn->read_msg.msg_iov, conn->read_msg.msg_iovlen);
1032
1033 out:
1034         return res;
1035 }
1036
1037 static void send_r2t(struct iscsi_cmnd *req)
1038 {
1039         struct iscsi_session *session = req->conn->session;
1040         struct iscsi_cmnd *rsp;
1041         struct iscsi_r2t_hdr *rsp_hdr;
1042         u32 offset, burst;
1043         LIST_HEAD(send);
1044
1045         if (unlikely(req->tmfabort)) {
1046                 TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted on R2T "
1047                         "(r2t_length %d, outstanding_r2t %d)", req,
1048                         req->scst_cmd, req->r2t_length, req->outstanding_r2t);
1049                 if (req->outstanding_r2t == 0)
1050                         iscsi_fail_waiting_cmnd(req);
1051                 goto out;
1052         }
1053
1054         /*
1055          * There is no race with data_out_start() and conn_abort(), since
1056          * all functions called from single read thread
1057          */
1058         iscsi_extracheck_is_rd_thread(req->conn);
1059
1060         burst = session->sess_param.max_burst_length;
1061         offset = be32_to_cpu(cmnd_hdr(req)->data_length) - req->r2t_length;
1062
1063         do {
1064                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
1065                 rsp->pdu.bhs.ttt = req->target_task_tag;
1066                 rsp_hdr = (struct iscsi_r2t_hdr *)&rsp->pdu.bhs;
1067                 rsp_hdr->opcode = ISCSI_OP_R2T;
1068                 rsp_hdr->flags = ISCSI_FLG_FINAL;
1069                 rsp_hdr->lun = cmnd_hdr(req)->lun;
1070                 rsp_hdr->itt = cmnd_hdr(req)->itt;
1071                 rsp_hdr->r2t_sn = cpu_to_be32(req->r2t_sn++);
1072                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
1073                 if (req->r2t_length > burst) {
1074                         rsp_hdr->data_length = cpu_to_be32(burst);
1075                         req->r2t_length -= burst;
1076                         offset += burst;
1077                 } else {
1078                         rsp_hdr->data_length = cpu_to_be32(req->r2t_length);
1079                         req->r2t_length = 0;
1080                 }
1081
1082                 TRACE_WRITE("%x %u %u %u %u", cmnd_itt(req),
1083                         be32_to_cpu(rsp_hdr->data_length),
1084                         be32_to_cpu(rsp_hdr->buffer_offset),
1085                         be32_to_cpu(rsp_hdr->r2t_sn), req->outstanding_r2t);
1086
1087                 list_add_tail(&rsp->write_list_entry, &send);
1088
1089                 if (++req->outstanding_r2t >= session->sess_param.max_outstanding_r2t)
1090                         break;
1091
1092         } while(req->r2t_length != 0);
1093
1094         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_WAKE);
1095
1096 out:
1097         return;
1098 }
1099
1100 static int iscsi_pre_exec(struct scst_cmd *scst_cmd)
1101 {
1102         int res = SCST_PREPROCESS_STATUS_SUCCESS;
1103         struct iscsi_cmnd *req = (struct iscsi_cmnd*)
1104                 scst_cmd_get_tgt_priv(scst_cmd);
1105         struct iscsi_cmnd *c, *t;
1106
1107         TRACE_ENTRY();
1108
1109         EXTRACHECKS_BUG_ON(scst_cmd_atomic(scst_cmd));
1110
1111         if (scst_cmd_get_data_direction(scst_cmd) == SCST_DATA_READ) {
1112                 if (!(req->conn->ddigest_type & DIGEST_NONE))
1113                         scst_set_long_xmit(scst_cmd);
1114 #ifndef NET_PAGE_CALLBACKS_DEFINED
1115                 else if (cmnd_hdr(req)->data_length > 8*1024)
1116                         scst_set_long_xmit(scst_cmd);
1117 #endif
1118                 EXTRACHECKS_BUG_ON(!list_empty(&req->rx_ddigest_cmd_list));
1119                 goto out;
1120         }
1121
1122         /* If data digest isn't used this list will be empty */
1123         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
1124                                 rx_ddigest_cmd_list_entry) {
1125                 TRACE_DBG("Checking digest of RX ddigest cmd %p", c);
1126                 if (digest_rx_data(c) != 0) {
1127                         scst_set_cmd_error(scst_cmd,
1128                                 SCST_LOAD_SENSE(iscsi_sense_crc_error));
1129                         res = SCST_PREPROCESS_STATUS_ERROR_SENSE_SET;
1130                         /*
1131                          * The rest of rx_ddigest_cmd_list will be freed 
1132                          * in req_cmnd_release()
1133                          */
1134                         goto out;
1135                 }
1136                 cmd_del_from_rx_ddigest_list(c);
1137                 cmnd_put(c);
1138         }
1139
1140 out:
1141         TRACE_EXIT_RES(res);
1142         return res;
1143 }
1144
1145 static int noop_out_start(struct iscsi_cmnd *cmnd)
1146 {
1147         struct iscsi_conn *conn = cmnd->conn;
1148         u32 size, tmp;
1149         int i, err = 0;
1150
1151         TRACE_DBG("%p", cmnd);
1152
1153         iscsi_extracheck_is_rd_thread(conn);
1154
1155         if (unlikely(cmnd_ttt(cmnd) != cpu_to_be32(ISCSI_RESERVED_TAG))) {
1156                 /*
1157                  * We don't request a NOP-Out by sending a NOP-In.
1158                  * See 10.18.2 in the draft 20.
1159                  */
1160                 PRINT_ERROR("Initiator sent command with not RESERVED tag and "
1161                         "TTT %x", cmnd_itt(cmnd));
1162                 err = -ISCSI_REASON_PROTOCOL_ERROR;
1163                 goto out;
1164         }
1165
1166         if (cmnd_itt(cmnd) == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1167                 if (unlikely(!(cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE)))
1168                         PRINT_ERROR("%s", "Initiator sent RESERVED tag for "
1169                                 "non-immediate command");
1170                 spin_lock(&conn->session->sn_lock);
1171                 __update_stat_sn(cmnd);
1172                 err = check_cmd_sn(cmnd);
1173                 spin_unlock(&conn->session->sn_lock);
1174                 if (unlikely(err))
1175                         goto out;
1176         } else if (unlikely((err = cmnd_insert_hash(cmnd)) < 0)) {
1177                 PRINT_ERROR("Can't insert in hash: ignore this request %x",
1178                         cmnd_itt(cmnd));
1179                 goto out;
1180         }
1181
1182         if ((size = cmnd->pdu.datasize)) {
1183                 size = (size + 3) & -4;
1184                 conn->read_msg.msg_iov = conn->read_iov;
1185                 if (cmnd->pdu.bhs.itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1186                         struct scatterlist *sg;
1187
1188                         /* ToDo: __GFP_NOFAIL ?? */
1189                         cmnd->sg = sg = scst_alloc(size,
1190                                 GFP_KERNEL|__GFP_NOFAIL, &cmnd->sg_cnt);
1191                         if (sg == NULL) {
1192                                 /* ToDo(); */
1193                         }
1194                         if (cmnd->sg_cnt > ISCSI_CONN_IOV_MAX) {
1195                                 /* ToDo(); */
1196                         }
1197                         cmnd->own_sg = 1;
1198                         cmnd->bufflen = size;
1199
1200                         for (i = 0; i < cmnd->sg_cnt; i++) {
1201                                 conn->read_iov[i].iov_base =
1202                                         page_address(sg_page(&sg[i]));
1203                                 tmp = min_t(u32, size, PAGE_SIZE);
1204                                 conn->read_iov[i].iov_len = tmp;
1205                                 conn->read_size += tmp;
1206                                 size -= tmp;
1207                         }
1208                 } else {
1209                         /*
1210                          * There are no problems with the safety from concurrent
1211                          * accesses to dummy_data, since for ISCSI_RESERVED_TAG
1212                          * the data only read and then discarded.
1213                          */
1214                         for (i = 0; i < ISCSI_CONN_IOV_MAX; i++) {
1215                                 conn->read_iov[i].iov_base = dummy_data;
1216                                 tmp = min_t(u32, size, sizeof(dummy_data));
1217                                 conn->read_iov[i].iov_len = tmp;
1218                                 conn->read_size += tmp;
1219                                 size -= tmp;
1220                         }
1221                 }
1222                 sBUG_ON(size == 0);
1223                 conn->read_msg.msg_iovlen = i;
1224                 TRACE_DBG("msg_iov=%p, msg_iovlen=%d", conn->read_msg.msg_iov,
1225                         conn->read_msg.msg_iovlen);
1226         }
1227 out:
1228         return err;
1229 }
1230
1231 static inline u32 get_next_ttt(struct iscsi_conn *conn)
1232 {
1233         u32 ttt;
1234         struct iscsi_session *session = conn->session;
1235
1236         iscsi_extracheck_is_rd_thread(conn);
1237
1238         if (session->next_ttt == ISCSI_RESERVED_TAG)
1239                 session->next_ttt++;
1240         ttt = session->next_ttt++;
1241
1242         return cpu_to_be32(ttt);
1243 }
1244
1245 static int scsi_cmnd_start(struct iscsi_cmnd *req)
1246 {
1247         struct iscsi_conn *conn = req->conn;
1248         struct iscsi_session *session = conn->session;
1249         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1250         struct scst_cmd *scst_cmd;
1251         scst_data_direction dir;
1252         int res = 0;
1253
1254         TRACE_ENTRY();
1255
1256         TRACE_DBG("scsi command: %02x", req_hdr->scb[0]);
1257
1258         TRACE_DBG("Incrementing active_cmds (cmd %p, sess %p, "
1259                 "new value %d)", req, session,
1260                 atomic_read(&session->active_cmds)+1);
1261         atomic_inc(&session->active_cmds);
1262         req->dec_active_cmnds = 1;
1263
1264         scst_cmd = scst_rx_cmd(session->scst_sess,
1265                 (uint8_t*)&req_hdr->lun, sizeof(req_hdr->lun),
1266                 req_hdr->scb, sizeof(req_hdr->scb), SCST_NON_ATOMIC);
1267         if (scst_cmd == NULL) {
1268                 create_status_rsp(req, SAM_STAT_BUSY, NULL, 0);
1269                 cmnd_prepare_skip_pdu_set_resid(req);
1270                 goto out;
1271         }
1272
1273         req->scst_cmd = scst_cmd;
1274         scst_cmd_set_tag(scst_cmd, req_hdr->itt);
1275         scst_cmd_set_tgt_priv(scst_cmd, req);
1276 #ifndef NET_PAGE_CALLBACKS_DEFINED
1277         scst_cmd_set_data_buf_tgt_alloc(scst_cmd);
1278 #endif
1279
1280         if (req_hdr->flags & ISCSI_CMD_READ)
1281                 dir = SCST_DATA_READ;
1282         else if (req_hdr->flags & ISCSI_CMD_WRITE)
1283                 dir = SCST_DATA_WRITE;
1284         else
1285                 dir = SCST_DATA_NONE;
1286         scst_cmd_set_expected(scst_cmd, dir, be32_to_cpu(req_hdr->data_length));
1287
1288         switch(req_hdr->flags & ISCSI_CMD_ATTR_MASK) {
1289         case ISCSI_CMD_SIMPLE:
1290                 scst_cmd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1291                 break;
1292         case ISCSI_CMD_HEAD_OF_QUEUE:
1293                 scst_cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1294                 break;
1295         case ISCSI_CMD_ORDERED:
1296                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1297                 break;
1298         case ISCSI_CMD_ACA:
1299                 scst_cmd->queue_type = SCST_CMD_QUEUE_ACA;
1300                 break;
1301         case ISCSI_CMD_UNTAGGED:
1302                 scst_cmd->queue_type = SCST_CMD_QUEUE_UNTAGGED;
1303                 break;
1304         default:
1305                 PRINT_ERROR("Unknown task code %x, use ORDERED instead",
1306                         req_hdr->flags & ISCSI_CMD_ATTR_MASK);
1307                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1308                 break;
1309         }
1310
1311         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
1312         scst_cmd_set_tgt_sn(scst_cmd, req_hdr->cmd_sn);
1313
1314         TRACE_DBG("START Command (tag %d, queue_type %d)",
1315                 req_hdr->itt, scst_cmd->queue_type);
1316         req->scst_state = ISCSI_CMD_STATE_RX_CMD;
1317         scst_cmd_init_stage1_done(scst_cmd, SCST_CONTEXT_DIRECT, 0);
1318
1319         wait_event(req->scst_waitQ, (req->scst_state != ISCSI_CMD_STATE_RX_CMD));
1320
1321         if (unlikely(req->scst_state != ISCSI_CMD_STATE_AFTER_PREPROC)) {
1322                 TRACE_DBG("req %p is in %x state", req, req->scst_state);
1323                 if (req->scst_state == ISCSI_CMD_STATE_PROCESSED) {
1324                         cmnd_prepare_skip_pdu_set_resid(req);
1325                         goto out;
1326                 }
1327                 if (unlikely(req->tmfabort)) {
1328                         TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
1329                                 req->scst_cmd);
1330                         cmnd_prepare_skip_pdu(req);
1331                         goto out;
1332                 }
1333                 sBUG();
1334         }
1335
1336         dir = scst_cmd_get_data_direction(scst_cmd);
1337         if (dir != SCST_DATA_WRITE) {
1338                 if (unlikely(!(req_hdr->flags & ISCSI_CMD_FINAL) ||
1339                              req->pdu.datasize)) {
1340                         PRINT_ERROR("Unexpected unsolicited data (ITT %x "
1341                                 "CDB %x", cmnd_itt(req), req_hdr->scb[0]);
1342                         create_sense_rsp(req, ABORTED_COMMAND, 0xc, 0xc);
1343                         cmnd_prepare_skip_pdu_set_resid(req);
1344                         goto out;
1345                 }
1346         }
1347
1348         if (dir == SCST_DATA_WRITE) {
1349                 req->is_unsolicited_data = !(req_hdr->flags & ISCSI_CMD_FINAL);
1350                 req->r2t_length = be32_to_cpu(req_hdr->data_length) - req->pdu.datasize;
1351                 if (req->r2t_length > 0)
1352                         req->data_waiting = 1;
1353         }
1354         req->target_task_tag = get_next_ttt(conn);
1355         req->sg = scst_cmd_get_sg(scst_cmd);
1356         req->bufflen = scst_cmd_get_bufflen(scst_cmd);
1357         if (unlikely(req->r2t_length > req->bufflen)) {
1358                 PRINT_ERROR("req->r2t_length %d > req->bufflen %d",
1359                         req->r2t_length, req->bufflen);
1360                 req->r2t_length = req->bufflen;
1361         }
1362
1363         TRACE_DBG("req=%p, dir=%d, is_unsolicited_data=%d, "
1364                 "r2t_length=%d, bufflen=%d", req, dir,
1365                 req->is_unsolicited_data, req->r2t_length, req->bufflen);
1366
1367         if (unlikely(!session->sess_param.immediate_data &&
1368                      req->pdu.datasize)) {
1369                 PRINT_ERROR("Initiator %s violated negotiated paremeters: "
1370                         "forbidden immediate data sent (ITT %x, op  %x)",
1371                         session->initiator_name, cmnd_itt(req), req_hdr->scb[0]);
1372                 res = -EINVAL;
1373                 goto out;
1374         }
1375
1376         if (unlikely(session->sess_param.initial_r2t &&
1377                      !(req_hdr->flags & ISCSI_CMD_FINAL))) {
1378                 PRINT_ERROR("Initiator %s violated negotiated paremeters: "
1379                         "initial R2T is required (ITT %x, op  %x)",
1380                         session->initiator_name, cmnd_itt(req), req_hdr->scb[0]);
1381                 res = -EINVAL;
1382                 goto out;
1383         }
1384
1385         if (req->pdu.datasize) {
1386                 if (unlikely(dir != SCST_DATA_WRITE)) {
1387                         PRINT_ERROR("pdu.datasize(%d) >0, but dir(%x) isn't WRITE",
1388                                 req->pdu.datasize, dir);
1389                         create_sense_rsp(req, ABORTED_COMMAND, 0xc, 0xc);
1390                         cmnd_prepare_skip_pdu_set_resid(req);
1391                 } else
1392                         res = cmnd_prepare_recv_pdu(conn, req, 0, req->pdu.datasize);
1393         }
1394 out:
1395         /* Aborted commands will be freed in cmnd_rx_end() */
1396         TRACE_EXIT_RES(res);
1397         return res;
1398 }
1399
1400 static int data_out_start(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
1401 {
1402         struct iscsi_data_out_hdr *req_hdr = (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1403         struct iscsi_cmnd *req = NULL;
1404         u32 offset = be32_to_cpu(req_hdr->buffer_offset);
1405         int res = 0;
1406
1407         TRACE_ENTRY();
1408
1409         /*
1410          * There is no race with send_r2t() and conn_abort(), since
1411          * all functions called from single read thread
1412          */
1413         iscsi_extracheck_is_rd_thread(cmnd->conn);
1414
1415         update_stat_sn(cmnd);
1416
1417         cmnd->cmd_req = req = cmnd_find_hash(conn->session, req_hdr->itt,
1418                                         req_hdr->ttt);
1419         if (unlikely(req == NULL)) {
1420                 /* It might happen if req was aborted and then freed */
1421                 TRACE(TRACE_MGMT_MINOR, "Unable to find scsi task %x %x",
1422                         cmnd_itt(cmnd), cmnd_ttt(cmnd));
1423                 goto skip_pdu;
1424         }
1425
1426         if (req->is_unsolicited_data) {
1427                 if (unlikely(req->r2t_length < cmnd->pdu.datasize)) {
1428                         PRINT_ERROR("Data size (%d) > R2T length (%d)",
1429                                 cmnd->pdu.datasize, req->r2t_length);
1430                         mark_conn_closed(conn);
1431                         res = -EINVAL;
1432                         goto out;
1433                 }
1434                 req->r2t_length -= cmnd->pdu.datasize;
1435         }
1436
1437         /* Check unsolicited burst data */
1438         if (unlikely((req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) &&
1439                      (req->pdu.bhs.flags & ISCSI_FLG_FINAL))) {
1440                 PRINT_ERROR("Unexpected data from %x %x",
1441                         cmnd_itt(cmnd), cmnd_ttt(cmnd));
1442                 mark_conn_closed(conn);
1443                 res = -EINVAL;
1444                 goto out;
1445         }
1446
1447         TRACE_WRITE("%u %p %p %u %u", req_hdr->ttt, cmnd, req,
1448                 offset, cmnd->pdu.datasize);
1449
1450         res = cmnd_prepare_recv_pdu(conn, req, offset, cmnd->pdu.datasize);
1451
1452 out:
1453         TRACE_EXIT_RES(res);
1454         return res;
1455
1456 skip_pdu:
1457         cmnd->pdu.bhs.opcode = ISCSI_OP_DATA_REJECT;
1458         cmnd_prepare_skip_pdu(cmnd);
1459         goto out;
1460 }
1461
1462 static void data_out_end(struct iscsi_cmnd *cmnd)
1463 {
1464         struct iscsi_data_out_hdr *req_hdr = (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1465         struct iscsi_cmnd *req;
1466
1467         sBUG_ON(cmnd == NULL);
1468         req = cmnd->cmd_req;
1469         sBUG_ON(req == NULL);
1470
1471         TRACE_DBG("cmnd %p, req %p", cmnd, req);
1472
1473         iscsi_extracheck_is_rd_thread(cmnd->conn);
1474
1475         if (!(cmnd->conn->ddigest_type & DIGEST_NONE) &&
1476             !cmnd->ddigest_checked) {
1477                 cmd_add_on_rx_ddigest_list(req, cmnd);
1478                 cmnd_get(cmnd);
1479         }
1480
1481         if (req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1482                 TRACE_DBG("ISCSI_RESERVED_TAG, FINAL %x",
1483                         req_hdr->flags & ISCSI_FLG_FINAL);
1484
1485                 if (req_hdr->flags & ISCSI_FLG_FINAL) {
1486                         req->is_unsolicited_data = 0;
1487                         if (req->pending)
1488                                 goto out_put;
1489                 } else
1490                         goto out_put;
1491         } else {
1492                 TRACE_DBG("FINAL %x, outstanding_r2t %d, r2t_length %d",
1493                         req_hdr->flags & ISCSI_FLG_FINAL,
1494                         req->outstanding_r2t, req->r2t_length);
1495
1496                 if (req_hdr->flags & ISCSI_FLG_FINAL) {
1497                         if (unlikely(req->is_unsolicited_data)) {
1498                                 PRINT_ERROR("Unexpected unsolicited data "
1499                                         "(r2t_length %u, outstanding_r2t %d)",
1500                                         req->r2t_length, req->is_unsolicited_data);
1501                                 mark_conn_closed(req->conn);
1502                                 goto out_put;
1503                         }
1504                         req->outstanding_r2t--;
1505                 } else
1506                         goto out_put;
1507         }
1508
1509         if (req->r2t_length != 0) {
1510                 if (!req->is_unsolicited_data)
1511                         send_r2t(req);
1512         } else
1513                 iscsi_restart_waiting_cmnd(req);
1514
1515 out_put:
1516         cmnd_put(cmnd);
1517         return;
1518 }
1519
1520 static void __cmnd_abort(struct iscsi_cmnd *cmnd)
1521 {
1522         TRACE_MGMT_DBG("Aborting cmd %p, scst_cmd %p (scst state %x, "
1523                 "ref_cnt %d, itt %x, sn %u, op %x, r2t_len %x, CDB op %x, "
1524                 "size to write %u, is_unsolicited_data %d, "
1525                 "outstanding_r2t %d, data_waiting %d, sess->exp_cmd_sn %u, "
1526                 "conn %p, rd_task %p)", cmnd, cmnd->scst_cmd, cmnd->scst_state,
1527                 atomic_read(&cmnd->ref_cnt), cmnd_itt(cmnd), cmnd->pdu.bhs.sn,
1528                 cmnd_opcode(cmnd), cmnd->r2t_length, cmnd_scsicode(cmnd),
1529                 cmnd_write_size(cmnd), cmnd->is_unsolicited_data,
1530                 cmnd->outstanding_r2t, cmnd->data_waiting,
1531                 cmnd->conn->session->exp_cmd_sn, cmnd->conn,
1532                 cmnd->conn->rd_task);
1533
1534 #ifdef NET_PAGE_CALLBACKS_DEFINED
1535         TRACE_MGMT_DBG("net_ref_cnt %d", atomic_read(&cmnd->net_ref_cnt));
1536 #endif
1537
1538         cmnd->tmfabort = 1;
1539
1540         return;
1541 }
1542
1543 /* Must be called from the read thread */
1544 static int cmnd_abort(struct iscsi_cmnd *req)
1545 {
1546         struct iscsi_session *session = req->conn->session;
1547         struct iscsi_task_mgt_hdr *req_hdr =
1548                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1549         struct iscsi_cmnd *cmnd;
1550         int err;
1551
1552         req_hdr->ref_cmd_sn = be32_to_cpu(req_hdr->ref_cmd_sn);
1553
1554         if (after(req_hdr->ref_cmd_sn, req_hdr->cmd_sn)) {
1555                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) > CmdSN(%u)",
1556                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn);
1557                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1558                 goto out;
1559         }
1560
1561         if ((cmnd = cmnd_find_hash_get(session, req_hdr->rtt, ISCSI_RESERVED_TAG))) {
1562                 struct iscsi_conn *conn = cmnd->conn;
1563                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
1564
1565                 if (req_hdr->lun != hdr->lun) {
1566                          PRINT_ERROR("ABORT TASK: LUN mismatch: req LUN "
1567                                 "%Lx, cmd LUN %Lx, rtt %u", req_hdr->lun,
1568                                 hdr->lun, req_hdr->rtt);
1569                         err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1570                         goto out_put;
1571                 }
1572
1573                 if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
1574                         if (req_hdr->ref_cmd_sn != req_hdr->cmd_sn) {
1575                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != TM cmd "
1576                                         "CmdSN(%u) for immediate command %p",
1577                                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn,
1578                                         cmnd);
1579                                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1580                                 goto out_put;
1581                         }
1582                 } else {
1583                         if (req_hdr->ref_cmd_sn != hdr->cmd_sn) {
1584                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != "
1585                                         "CmdSN(%u) for command %p",
1586                                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn,
1587                                         cmnd);
1588                                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1589                                 goto out_put;
1590                         }
1591                 }
1592
1593                 if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
1594                     (req_hdr->cmd_sn == hdr->cmd_sn)) {
1595                         PRINT_ERROR("ABORT TASK: SN mismatch: req SN %x, "
1596                                 "cmd SN %x, rtt %u", req_hdr->cmd_sn,
1597                                 hdr->cmd_sn, req_hdr->rtt);
1598                         err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1599                         goto out_put;
1600                 }
1601
1602                 spin_lock_bh(&conn->cmd_list_lock);
1603                 __cmnd_abort(cmnd);
1604                 spin_unlock_bh(&conn->cmd_list_lock);
1605
1606                 cmnd_put(cmnd);
1607                 err = 0;
1608         } else
1609                 err = ISCSI_RESPONSE_UNKNOWN_TASK;
1610
1611 out:
1612         return err;
1613
1614 out_put:
1615         cmnd_put(cmnd);
1616         goto out;
1617 }
1618
1619 /* Must be called from the read thread */
1620 static int target_abort(struct iscsi_cmnd *req, int all)
1621 {
1622         struct iscsi_target *target = req->conn->session->target;
1623         struct iscsi_task_mgt_hdr *req_hdr =
1624                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1625         struct iscsi_session *session;
1626         struct iscsi_conn *conn;
1627         struct iscsi_cmnd *cmnd;
1628
1629         mutex_lock(&target->target_mutex);
1630
1631         list_for_each_entry(session, &target->session_list, session_list_entry) {
1632                 list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
1633                         spin_lock_bh(&conn->cmd_list_lock);
1634                         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1635                                 if (cmnd == req)
1636                                         continue;
1637                                 if (all)
1638                                         __cmnd_abort(cmnd);
1639                                 else if (req_hdr->lun == cmnd_hdr(cmnd)->lun)
1640                                         __cmnd_abort(cmnd);
1641                         }
1642                         spin_unlock_bh(&conn->cmd_list_lock);
1643                 }
1644         }
1645
1646         mutex_unlock(&target->target_mutex);
1647         return 0;
1648 }
1649
1650 /* Must be called from the read thread */
1651 static void task_set_abort(struct iscsi_cmnd *req)
1652 {
1653         struct iscsi_session *session = req->conn->session;
1654         struct iscsi_task_mgt_hdr *req_hdr =
1655                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1656         struct iscsi_target *target = session->target;
1657         struct iscsi_conn *conn;
1658         struct iscsi_cmnd *cmnd;
1659
1660         mutex_lock(&target->target_mutex);
1661
1662         list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
1663                 spin_lock_bh(&conn->cmd_list_lock);
1664                 list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1665                         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
1666                         if (cmnd == req)
1667                                 continue;
1668                         if (req_hdr->lun != hdr->lun)
1669                                 continue;
1670                         if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
1671                             req_hdr->cmd_sn == hdr->cmd_sn)
1672                                 continue;
1673                         __cmnd_abort(cmnd);
1674                 }
1675                 spin_unlock_bh(&conn->cmd_list_lock);
1676         }
1677
1678         mutex_unlock(&target->target_mutex);
1679         return;
1680 }
1681
1682 /* Must be called from the read thread */
1683 void conn_abort(struct iscsi_conn *conn)
1684 {
1685         struct iscsi_cmnd *cmnd;
1686
1687         TRACE_MGMT_DBG("Aborting conn %p", conn);
1688
1689         iscsi_extracheck_is_rd_thread(conn);
1690
1691         spin_lock_bh(&conn->cmd_list_lock);
1692 again:
1693         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1694                 __cmnd_abort(cmnd);
1695                 if (cmnd->data_waiting) {
1696                         if (!cmnd_get_check(cmnd)) {
1697                                 spin_unlock_bh(&conn->cmd_list_lock);
1698
1699                                 /* ToDo: this is racy for MC/S */
1700                                 TRACE_MGMT_DBG("Restarting data waiting cmd %p",
1701                                         cmnd);
1702                                 iscsi_fail_waiting_cmnd(cmnd);
1703
1704                                 cmnd_put(cmnd);
1705
1706                                 /*
1707                                  * We are in the read thread, so we may not
1708                                  * worry that after cmnd release conn gets
1709                                  * released as well.
1710                                  */
1711                                 spin_lock_bh(&conn->cmd_list_lock);
1712                                 goto again;
1713                         }
1714                 }
1715         }
1716         spin_unlock_bh(&conn->cmd_list_lock);
1717
1718         return;
1719 }
1720
1721 static void execute_task_management(struct iscsi_cmnd *req)
1722 {
1723         struct iscsi_conn *conn = req->conn;
1724         struct iscsi_session *sess = conn->session;
1725         struct iscsi_task_mgt_hdr *req_hdr =
1726                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1727         int err = 0, function = req_hdr->function & ISCSI_FUNCTION_MASK;
1728         struct scst_rx_mgmt_params params;
1729
1730         TRACE((function == ISCSI_FUNCTION_ABORT_TASK) ? TRACE_MGMT_MINOR : TRACE_MGMT,
1731                 "TM fn %d", function);
1732
1733         TRACE_MGMT_DBG("TM req %p, itt %x, rtt %x, sn %u, con %p", req,
1734                 cmnd_itt(req), req_hdr->rtt, req_hdr->cmd_sn, conn);
1735
1736         iscsi_extracheck_is_rd_thread(conn);
1737
1738         spin_lock(&sess->sn_lock);
1739         sess->tm_active = 1;
1740         sess->tm_sn = req_hdr->cmd_sn;
1741         if (sess->tm_rsp != NULL) {
1742                 struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
1743                 TRACE(TRACE_MGMT_MINOR, "Dropping delayed TM rsp %p", tm_rsp);
1744                 sess->tm_rsp = NULL;
1745                 spin_unlock(&sess->sn_lock);
1746                 rsp_cmnd_release(tm_rsp);
1747         } else
1748                 spin_unlock(&sess->sn_lock);
1749
1750         memset(&params, 0, sizeof(params));
1751         params.atomic = SCST_NON_ATOMIC;
1752         params.tgt_priv = req;
1753
1754         if ((function != ISCSI_FUNCTION_ABORT_TASK) &&
1755             (req_hdr->rtt != ISCSI_RESERVED_TAG)) {
1756                 PRINT_ERROR("Invalid RTT %x (TM fn %x)", req_hdr->rtt,
1757                         function);
1758                 err = -1;
1759                 goto reject;
1760         }
1761
1762         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
1763
1764         switch (function) {
1765         case ISCSI_FUNCTION_ABORT_TASK:
1766                 err = cmnd_abort(req);
1767                 if (err == 0) {
1768                         params.fn = SCST_ABORT_TASK;
1769                         params.tag = req_hdr->rtt;
1770                         params.tag_set = 1;
1771                         params.lun = (uint8_t *)&req_hdr->lun;
1772                         params.lun_len = sizeof(req_hdr->lun);
1773                         params.lun_set = 1;
1774                         params.cmd_sn = req_hdr->cmd_sn;
1775                         params.cmd_sn_set = 1;
1776                         err = scst_rx_mgmt_fn(conn->session->scst_sess,
1777                                 &params);
1778                 }
1779                 break;
1780         case ISCSI_FUNCTION_ABORT_TASK_SET:
1781                 task_set_abort(req);
1782                 params.fn = SCST_ABORT_TASK_SET;
1783                 params.lun = (uint8_t *)&req_hdr->lun;
1784                 params.lun_len = sizeof(req_hdr->lun);
1785                 params.lun_set = 1;
1786                 params.cmd_sn = req_hdr->cmd_sn;
1787                 params.cmd_sn_set = 1;
1788                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1789                         &params);
1790                 break;
1791         case ISCSI_FUNCTION_CLEAR_TASK_SET:
1792                 task_set_abort(req);
1793                 params.fn = SCST_CLEAR_TASK_SET;
1794                 params.lun = (uint8_t *)&req_hdr->lun;
1795                 params.lun_len = sizeof(req_hdr->lun);
1796                 params.lun_set = 1;
1797                 params.cmd_sn = req_hdr->cmd_sn;
1798                 params.cmd_sn_set = 1;
1799                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1800                         &params);
1801                 break;
1802         case ISCSI_FUNCTION_CLEAR_ACA:
1803                 params.fn = SCST_CLEAR_ACA;
1804                 params.lun = (uint8_t *)&req_hdr->lun;
1805                 params.lun_len = sizeof(req_hdr->lun);
1806                 params.lun_set = 1;
1807                 params.cmd_sn = req_hdr->cmd_sn;
1808                 params.cmd_sn_set = 1;
1809                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1810                         &params);
1811                 break;
1812         case ISCSI_FUNCTION_TARGET_COLD_RESET:
1813         case ISCSI_FUNCTION_TARGET_WARM_RESET:
1814                 target_abort(req, 1);
1815                 params.fn = SCST_TARGET_RESET;
1816                 params.cmd_sn = req_hdr->cmd_sn;
1817                 params.cmd_sn_set = 1;
1818                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1819                         &params);
1820                 break;
1821         case ISCSI_FUNCTION_LOGICAL_UNIT_RESET:
1822                 target_abort(req, 0);
1823                 params.fn = SCST_LUN_RESET;
1824                 params.lun = (uint8_t *)&req_hdr->lun;
1825                 params.lun_len = sizeof(req_hdr->lun);
1826                 params.lun_set = 1;
1827                 params.cmd_sn = req_hdr->cmd_sn;
1828                 params.cmd_sn_set = 1;
1829                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1830                         &params);
1831                 break;
1832         case ISCSI_FUNCTION_TASK_REASSIGN:
1833                 iscsi_send_task_mgmt_resp(req, 
1834                         ISCSI_RESPONSE_FUNCTION_UNSUPPORTED);
1835                 break;
1836         default:
1837                 iscsi_send_task_mgmt_resp(req,
1838                         ISCSI_RESPONSE_FUNCTION_REJECTED);
1839                 break;
1840         }
1841
1842 reject:
1843         if (err != 0) {
1844                 iscsi_send_task_mgmt_resp(req,
1845                         ISCSI_RESPONSE_FUNCTION_REJECTED);
1846         }
1847 }
1848
1849 static void noop_out_exec(struct iscsi_cmnd *req)
1850 {
1851         struct iscsi_cmnd *rsp;
1852         struct iscsi_nop_in_hdr *rsp_hdr;
1853
1854         TRACE_DBG("%p", req);
1855
1856         if (cmnd_itt(req) != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1857                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
1858
1859                 rsp_hdr = (struct iscsi_nop_in_hdr *)&rsp->pdu.bhs;
1860                 rsp_hdr->opcode = ISCSI_OP_NOOP_IN;
1861                 rsp_hdr->flags = ISCSI_FLG_FINAL;
1862                 rsp_hdr->itt = req->pdu.bhs.itt;
1863                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
1864
1865                 if (req->pdu.datasize)
1866                         sBUG_ON(req->sg == NULL);
1867                 else
1868                         sBUG_ON(req->sg != NULL);
1869
1870                 if (req->sg) {
1871                         rsp->sg = req->sg;
1872                         rsp->bufflen = req->bufflen;
1873                 }
1874
1875                 sBUG_ON(get_pgcnt(req->pdu.datasize, 0) > ISCSI_CONN_IOV_MAX);
1876                 rsp->pdu.datasize = req->pdu.datasize;
1877                 iscsi_cmnd_init_write(rsp,
1878                         ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
1879                 req_cmnd_release(req);
1880         } else
1881                 cmnd_put(req);
1882 }
1883
1884 static void logout_exec(struct iscsi_cmnd *req)
1885 {
1886         struct iscsi_logout_req_hdr *req_hdr;
1887         struct iscsi_cmnd *rsp;
1888         struct iscsi_logout_rsp_hdr *rsp_hdr;
1889
1890         PRINT_INFO("Logout received from initiator %s",
1891                 req->conn->session->initiator_name);
1892         TRACE_DBG("%p", req);
1893
1894         req_hdr = (struct iscsi_logout_req_hdr *)&req->pdu.bhs;
1895         rsp = iscsi_cmnd_create_rsp_cmnd(req);
1896         rsp_hdr = (struct iscsi_logout_rsp_hdr *)&rsp->pdu.bhs;
1897         rsp_hdr->opcode = ISCSI_OP_LOGOUT_RSP;
1898         rsp_hdr->flags = ISCSI_FLG_FINAL;
1899         rsp_hdr->itt = req_hdr->itt;
1900         rsp->should_close_conn = 1;
1901         iscsi_cmnd_init_write(rsp,
1902                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
1903         req_cmnd_release(req);
1904 }
1905
1906 static void iscsi_cmnd_exec(struct iscsi_cmnd *cmnd)
1907 {
1908         TRACE_DBG("%p,%x,%u", cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn);
1909
1910         if (unlikely(cmnd->tmfabort)) {
1911                 TRACE_MGMT_DBG("cmnd %p (scst_cmd %p) aborted", cmnd,
1912                         cmnd->scst_cmd);
1913                 req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
1914                 goto out;
1915         }
1916
1917         iscsi_extracheck_is_rd_thread(cmnd->conn);
1918
1919         switch (cmnd_opcode(cmnd)) {
1920         case ISCSI_OP_SCSI_CMD:
1921                 if (cmnd->r2t_length != 0) {
1922                         if (!cmnd->is_unsolicited_data) {
1923                                 send_r2t(cmnd);
1924                                 break;
1925                         }
1926                 } else
1927                         iscsi_restart_cmnd(cmnd);
1928                 break;
1929         case ISCSI_OP_NOOP_OUT:
1930                 noop_out_exec(cmnd);
1931                 break;
1932         case ISCSI_OP_SCSI_TASK_MGT_MSG:
1933                 execute_task_management(cmnd);
1934                 break;
1935         case ISCSI_OP_LOGOUT_CMD:
1936                 logout_exec(cmnd);
1937                 break;
1938         case ISCSI_OP_SCSI_REJECT:
1939         {
1940                 struct iscsi_cmnd *rsp = get_rsp_cmnd(cmnd);
1941                 TRACE_MGMT_DBG("REJECT cmnd %p (scst_cmd %p), rsp %p", cmnd,
1942                         cmnd->scst_cmd, rsp);
1943                 if (rsp != NULL)
1944                         iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH |
1945                                                          ISCSI_INIT_WRITE_WAKE);
1946                 req_cmnd_release(cmnd);
1947                 break;
1948         }
1949         default:
1950                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
1951                 req_cmnd_release(cmnd);
1952                 break;
1953         }
1954 out:
1955         return;
1956 }
1957
1958 static void __cmnd_send_pdu(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd,
1959         u32 offset, u32 size)
1960 {
1961         TRACE_DBG("%p %u,%u,%u", cmnd, offset, size, cmnd->bufflen);
1962
1963         iscsi_extracheck_is_wr_thread(conn);
1964
1965         sBUG_ON(offset > cmnd->bufflen);
1966         sBUG_ON(offset + size > cmnd->bufflen);
1967
1968         conn->write_offset = offset;
1969         conn->write_size += size;
1970 }
1971
1972 static void cmnd_send_pdu(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
1973 {
1974         u32 size;
1975
1976         if (!cmnd->pdu.datasize)
1977                 return;
1978
1979         size = (cmnd->pdu.datasize + 3) & -4;
1980         sBUG_ON(cmnd->sg == NULL);
1981         sBUG_ON(cmnd->bufflen != size);
1982         __cmnd_send_pdu(conn, cmnd, 0, size);
1983 }
1984
1985 static void set_cork(struct socket *sock, int on)
1986 {
1987         int opt = on;
1988         mm_segment_t oldfs;
1989
1990         oldfs = get_fs();
1991         set_fs(get_ds());
1992         sock->ops->setsockopt(sock, SOL_TCP, TCP_CORK, (void *)&opt, sizeof(opt));
1993         set_fs(oldfs);
1994 }
1995
1996 void cmnd_tx_start(struct iscsi_cmnd *cmnd)
1997 {
1998         struct iscsi_conn *conn = cmnd->conn;
1999
2000         TRACE_DBG("%p:%p:%x", conn, cmnd, cmnd_opcode(cmnd));
2001         iscsi_cmnd_set_length(&cmnd->pdu);
2002
2003         iscsi_extracheck_is_wr_thread(conn);
2004
2005         set_cork(conn->sock, 1);
2006
2007         conn->write_iop = conn->write_iov;
2008         conn->write_iop->iov_base = &cmnd->pdu.bhs;
2009         conn->write_iop->iov_len = sizeof(cmnd->pdu.bhs);
2010         conn->write_iop_used = 1;
2011         conn->write_size = sizeof(cmnd->pdu.bhs);
2012
2013         switch (cmnd_opcode(cmnd)) {
2014         case ISCSI_OP_NOOP_IN:
2015                 cmnd_set_sn(cmnd, 1);
2016                 cmnd_send_pdu(conn, cmnd);
2017                 break;
2018         case ISCSI_OP_SCSI_RSP:
2019                 cmnd_set_sn(cmnd, 1);
2020                 cmnd_send_pdu(conn, cmnd);
2021                 break;
2022         case ISCSI_OP_SCSI_TASK_MGT_RSP:
2023                 cmnd_set_sn(cmnd, 1);
2024                 break;
2025         case ISCSI_OP_TEXT_RSP:
2026                 cmnd_set_sn(cmnd, 1);
2027                 break;
2028         case ISCSI_OP_SCSI_DATA_IN:
2029         {
2030                 struct iscsi_data_in_hdr *rsp = (struct iscsi_data_in_hdr *)&cmnd->pdu.bhs;
2031                 u32 offset = cpu_to_be32(rsp->buffer_offset);
2032
2033                 cmnd_set_sn(cmnd, (rsp->flags & ISCSI_FLG_FINAL) ? 1 : 0);
2034                 __cmnd_send_pdu(conn, cmnd, offset, cmnd->pdu.datasize);
2035                 break;
2036         }
2037         case ISCSI_OP_LOGOUT_RSP:
2038                 cmnd_set_sn(cmnd, 1);
2039                 break;
2040         case ISCSI_OP_R2T:
2041                 cmnd->pdu.bhs.sn = cmnd_set_sn(cmnd, 0);
2042                 break;
2043         case ISCSI_OP_ASYNC_MSG:
2044                 cmnd_set_sn(cmnd, 1);
2045                 break;
2046         case ISCSI_OP_REJECT:
2047                 cmnd_set_sn(cmnd, 1);
2048                 cmnd_send_pdu(conn, cmnd);
2049                 break;
2050         default:
2051                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2052                 break;
2053         }
2054
2055         // move this?
2056         conn->write_size = (conn->write_size + 3) & -4;
2057         iscsi_dump_pdu(&cmnd->pdu);
2058 }
2059
2060 void cmnd_tx_end(struct iscsi_cmnd *cmnd)
2061 {
2062         struct iscsi_conn *conn = cmnd->conn;
2063
2064         TRACE_DBG("%p:%x (should_close_conn %d)", cmnd, cmnd_opcode(cmnd),
2065                 cmnd->should_close_conn);
2066
2067         switch (cmnd_opcode(cmnd)) {
2068         case ISCSI_OP_NOOP_IN:
2069         case ISCSI_OP_SCSI_RSP:
2070         case ISCSI_OP_SCSI_TASK_MGT_RSP:
2071         case ISCSI_OP_TEXT_RSP:
2072         case ISCSI_OP_R2T:
2073         case ISCSI_OP_ASYNC_MSG:
2074         case ISCSI_OP_REJECT:
2075         case ISCSI_OP_SCSI_DATA_IN:
2076         case ISCSI_OP_LOGOUT_RSP:
2077                 break;
2078         default:
2079                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2080                 sBUG();
2081                 break;
2082         }
2083
2084         if (cmnd->should_close_conn) {
2085                 PRINT_INFO("Closing connection at initiator %s request",
2086                         conn->session->initiator_name);
2087                 mark_conn_closed(conn);
2088         }
2089
2090         set_cork(cmnd->conn->sock, 0);
2091 }
2092
2093 /*
2094  * Push the command for execution. This functions reorders the commands.
2095  * Called from the read thread.
2096  */
2097 static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd)
2098 {
2099         struct iscsi_session *session = cmnd->conn->session;
2100         struct list_head *entry;
2101         u32 cmd_sn;
2102
2103         TRACE_DBG("%p:%x %u,%u",
2104                 cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn, session->exp_cmd_sn);
2105
2106         iscsi_extracheck_is_rd_thread(cmnd->conn);
2107
2108         sBUG_ON(cmnd->parent_req != NULL);
2109
2110         if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
2111                 TRACE_DBG("Immediate cmd %p (cmd_sn %u)", cmnd,
2112                         cmnd->pdu.bhs.sn);
2113                 iscsi_cmnd_exec(cmnd);
2114                 goto out;
2115         }
2116
2117         spin_lock(&session->sn_lock);
2118
2119         cmd_sn = cmnd->pdu.bhs.sn;
2120         if (cmd_sn == session->exp_cmd_sn) {
2121                 while (1) {
2122                         session->exp_cmd_sn = ++cmd_sn;
2123
2124                         if (unlikely(session->tm_active)) {
2125                                 if (before(cmd_sn, session->tm_sn)) {
2126                                         struct iscsi_conn *conn = cmnd->conn;
2127
2128                                         spin_unlock(&session->sn_lock);
2129
2130                                         spin_lock_bh(&conn->cmd_list_lock);
2131                                         __cmnd_abort(cmnd);
2132                                         spin_unlock_bh(&conn->cmd_list_lock);
2133
2134                                         spin_lock(&session->sn_lock);
2135                                 }
2136                                 iscsi_check_send_delayed_tm_resp(session);
2137                         }
2138
2139                         spin_unlock(&session->sn_lock);
2140
2141                         iscsi_cmnd_exec(cmnd);
2142
2143                         if (list_empty(&session->pending_list))
2144                                 break;
2145                         cmnd = list_entry(session->pending_list.next, struct iscsi_cmnd,
2146                                                 pending_list_entry);
2147                         if (cmnd->pdu.bhs.sn != cmd_sn)
2148                                 break;
2149
2150                         list_del(&cmnd->pending_list_entry);
2151                         cmnd->pending = 0;
2152
2153                         TRACE_DBG("Processing pending cmd %p (cmd_sn %u)",
2154                                 cmnd, cmd_sn);
2155
2156                         spin_lock(&session->sn_lock);
2157                 }
2158         } else {
2159                 int drop = 0;
2160
2161                 TRACE_DBG("Pending cmd %p (cmd_sn %u, exp_cmd_sn %u)",
2162                         cmnd, cmd_sn, session->exp_cmd_sn);
2163
2164                 /*
2165                  * iSCSI RFC 3720: "The target MUST silently ignore any
2166                  * non-immediate command outside of [from ExpCmdSN to MaxCmdSN
2167                  * inclusive] range". But we won't honor the MaxCmdSN
2168                  * requirement, because, since we adjust MaxCmdSN from the
2169                  * separate write thread, rarery it is possible that initiator
2170                  * can legally send command with CmdSN>MaxSN. But it won't
2171                  * hurt anything, in the worst case it will lead to
2172                  * additional QUEUE FULL status.
2173                  */
2174
2175                 if (unlikely(before(cmd_sn, session->exp_cmd_sn))) {
2176                         PRINT_ERROR("Unexpected cmd_sn (%u,%u)", cmd_sn,
2177                                 session->exp_cmd_sn);
2178                         drop = 1;
2179                 }
2180
2181 #if 0
2182                 if (unlikely(after(cmd_sn, session->exp_cmd_sn +
2183                                         iscsi_get_allowed_cmds(session)))) {
2184                         TRACE_MGMT_DBG("Too large cmd_sn %u (exp_cmd_sn %u, "
2185                                 "max_sn %u)", cmd_sn, session->exp_cmd_sn,
2186                                 iscsi_get_allowed_cmds(session));
2187                 }
2188 #endif
2189
2190                 spin_unlock(&session->sn_lock);
2191
2192                 if (unlikely(drop)) {
2193                         req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
2194                         goto out;
2195                 }
2196
2197                 if (unlikely(cmnd->tmfabort)) {
2198                         struct iscsi_cmnd *tm_clone;
2199
2200                         TRACE_MGMT_DBG("Pending aborted cmnd %p, creating TM "
2201                                 "clone (scst cmd %p, state %d)", cmnd,
2202                                 cmnd->scst_cmd, cmnd->scst_state);
2203
2204                         tm_clone = cmnd_alloc(cmnd->conn, NULL);
2205                         if (tm_clone != NULL) {
2206                                 tm_clone->tmfabort = 1;
2207                                 tm_clone->pdu = cmnd->pdu;
2208
2209                                 TRACE_MGMT_DBG("TM clone %p created", tm_clone);
2210
2211                                 iscsi_cmnd_exec(cmnd);
2212                                 cmnd = tm_clone;
2213                         } else
2214                                 PRINT_ERROR("%s", "Unable to create TM clone");
2215                 }
2216
2217                 list_for_each(entry, &session->pending_list) {
2218                         struct iscsi_cmnd *tmp = list_entry(entry, struct iscsi_cmnd,
2219                                                         pending_list_entry);
2220                         if (before(cmd_sn, tmp->pdu.bhs.sn))
2221                                 break;
2222                 }
2223
2224                 list_add_tail(&cmnd->pending_list_entry, entry);
2225                 cmnd->pending = 1;
2226         }
2227 out:
2228         return;
2229 }
2230
2231 static int check_segment_length(struct iscsi_cmnd *cmnd)
2232 {
2233         struct iscsi_conn *conn = cmnd->conn;
2234         struct iscsi_session *session = conn->session;
2235
2236         if (unlikely(cmnd->pdu.datasize > session->sess_param.max_recv_data_length)) {
2237                 PRINT_ERROR("Initiator %s violated negotiated parameters: "
2238                         "data too long (ITT %x, datasize %u, "
2239                         "max_recv_data_length %u", session->initiator_name,
2240                         cmnd_itt(cmnd), cmnd->pdu.datasize,
2241                         session->sess_param.max_recv_data_length);
2242                 mark_conn_closed(conn);
2243                 return -EINVAL;
2244         }
2245         return 0;
2246 }
2247
2248 int cmnd_rx_start(struct iscsi_cmnd *cmnd)
2249 {
2250         struct iscsi_conn *conn = cmnd->conn;
2251         int res, rc;
2252
2253         iscsi_dump_pdu(&cmnd->pdu);
2254
2255         res = check_segment_length(cmnd);
2256         if (res != 0)
2257                 goto out;
2258
2259         switch (cmnd_opcode(cmnd)) {
2260         case ISCSI_OP_NOOP_OUT:
2261                 rc = noop_out_start(cmnd);
2262                 break;
2263         case ISCSI_OP_SCSI_CMD:
2264                 rc = cmnd_insert_hash(cmnd);
2265                 if (likely(rc == 0)) {
2266                         res = scsi_cmnd_start(cmnd);
2267                         if (unlikely(res != 0))
2268                                 goto out;
2269                 }
2270                 break;
2271         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2272                 rc = cmnd_insert_hash(cmnd);
2273                 break;
2274         case ISCSI_OP_SCSI_DATA_OUT:
2275                 res = data_out_start(conn, cmnd);
2276                 rc = 0; /* to avoid compiler warning */
2277                 if (unlikely(res != 0))
2278                         goto out;
2279                 break;
2280         case ISCSI_OP_LOGOUT_CMD:
2281                 rc = cmnd_insert_hash(cmnd);
2282                 break;
2283         case ISCSI_OP_TEXT_CMD:
2284         case ISCSI_OP_SNACK_CMD:
2285                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2286                 break;
2287         default:
2288                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2289                 break;
2290         }
2291
2292         if (unlikely(rc < 0)) {
2293                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
2294                 PRINT_ERROR("Error %d (iSCSI opcode %x, ITT %x, op %x)", rc,
2295                         cmnd_opcode(cmnd), cmnd_itt(cmnd),
2296                         (cmnd_opcode(cmnd) == ISCSI_OP_SCSI_CMD ?
2297                                 hdr->scb[0] : -1));
2298                 iscsi_cmnd_reject(cmnd, -rc);
2299         }
2300
2301 out:
2302         TRACE_EXIT_RES(res);
2303         return res;
2304 }
2305
2306 void cmnd_rx_end(struct iscsi_cmnd *cmnd)
2307 {
2308         TRACE_ENTRY();
2309
2310         TRACE_DBG("%p:%x", cmnd, cmnd_opcode(cmnd));
2311
2312         switch (cmnd_opcode(cmnd)) {
2313         case ISCSI_OP_SCSI_CMD:
2314         case ISCSI_OP_SCSI_REJECT:
2315         case ISCSI_OP_NOOP_OUT:
2316         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2317         case ISCSI_OP_LOGOUT_CMD:
2318                 iscsi_session_push_cmnd(cmnd);
2319                 break;
2320         case ISCSI_OP_SCSI_DATA_OUT:
2321                 data_out_end(cmnd);
2322                 break;
2323         case ISCSI_OP_PDU_REJECT:
2324         {
2325                 struct iscsi_cmnd *rsp = get_rsp_cmnd(cmnd);
2326                 if (rsp != NULL)
2327                         iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH |
2328                                                         ISCSI_INIT_WRITE_WAKE);
2329                 req_cmnd_release(cmnd);
2330                 break;
2331         }
2332         case ISCSI_OP_DATA_REJECT:
2333                 req_cmnd_release(cmnd);
2334                 break;
2335         default:
2336                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2337                 req_cmnd_release(cmnd);
2338                 break;
2339         }
2340
2341         TRACE_EXIT();
2342         return;
2343 }
2344
2345 #ifndef NET_PAGE_CALLBACKS_DEFINED
2346 static int iscsi_alloc_data_buf(struct scst_cmd *cmd)
2347 {
2348         if (scst_cmd_get_data_direction(cmd) == SCST_DATA_READ) {
2349                 /*
2350                  * sock->ops->sendpage() is async zero copy operation,
2351                  * so we must be sure not to free and reuse
2352                  * the command's buffer before the sending was completed
2353                  * by the network layers. It is possible only if we
2354                  * don't use SGV cache.
2355                  */
2356                 scst_cmd_set_no_sgv(cmd);
2357         }
2358         return 1;
2359 }
2360 #endif
2361
2362 static inline void iscsi_set_state_wake_up(struct iscsi_cmnd *req,
2363         int new_state)
2364 {
2365         /*
2366          * We use wait_event() to wait for the state change, but it checks its
2367          * condition without any protection, so without cmnd_get() it is
2368          * possible that req will die "immediately" after the state assignment
2369          * and wake_up() will operate on dead data.
2370          */
2371         cmnd_get_ordered(req);
2372         req->scst_state = new_state;
2373         wake_up(&req->scst_waitQ);
2374         cmnd_put(req);
2375         return;
2376 }
2377
2378 static void iscsi_preprocessing_done(struct scst_cmd *scst_cmd)
2379 {
2380         struct iscsi_cmnd *req = (struct iscsi_cmnd*)
2381                                 scst_cmd_get_tgt_priv(scst_cmd);
2382
2383         TRACE_DBG("req %p", req);
2384
2385         EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_RX_CMD);
2386
2387         iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_AFTER_PREPROC);
2388         return;
2389 }
2390
2391 /* 
2392  * No locks.
2393  *
2394  * IMPORTANT! Connection conn must be protected by additional conn_get()
2395  * upon entrance in this function, because otherwise it could be destroyed
2396  * inside as a result of iscsi_send(), which releases sent commands.
2397  */
2398 static void iscsi_try_local_processing(struct iscsi_conn *conn,
2399         bool single_only)
2400 {
2401         int local;
2402
2403         TRACE_ENTRY();
2404
2405         spin_lock_bh(&iscsi_wr_lock);
2406         switch(conn->wr_state) {
2407         case ISCSI_CONN_WR_STATE_IN_LIST:
2408                 list_del(&conn->wr_list_entry);
2409                 /* go through */
2410         case ISCSI_CONN_WR_STATE_IDLE:
2411 #ifdef EXTRACHECKS
2412                 conn->wr_task = current;
2413 #endif
2414                 conn->wr_state = ISCSI_CONN_WR_STATE_PROCESSING;
2415                 conn->wr_space_ready = 0;
2416                 local = 1;
2417                 break;
2418         default:
2419                 local = 0;
2420                 break;
2421         }
2422         spin_unlock_bh(&iscsi_wr_lock);
2423
2424         if (local) {
2425                 int rc = 1;
2426                 while(test_write_ready(conn)) {
2427                         rc = iscsi_send(conn);
2428                         if ((rc <= 0) || single_only) {
2429                                 break;
2430                         }
2431                 }
2432
2433                 spin_lock_bh(&iscsi_wr_lock);
2434 #ifdef EXTRACHECKS
2435                 conn->wr_task = NULL;
2436 #endif
2437                 if ((rc <= 0) || test_write_ready(conn)) {
2438                         list_add_tail(&conn->wr_list_entry, &iscsi_wr_list);
2439                         conn->wr_state = ISCSI_CONN_WR_STATE_IN_LIST;
2440                         wake_up(&iscsi_wr_waitQ);
2441                 } else
2442                         conn->wr_state = ISCSI_CONN_WR_STATE_IDLE;
2443                 spin_unlock_bh(&iscsi_wr_lock);
2444         }
2445
2446         TRACE_EXIT();
2447         return;
2448 }
2449
2450 static int iscsi_xmit_response(struct scst_cmd *scst_cmd)
2451 {
2452         int resp_flags = scst_cmd_get_tgt_resp_flags(scst_cmd);
2453         struct iscsi_cmnd *req = (struct iscsi_cmnd*)
2454                                         scst_cmd_get_tgt_priv(scst_cmd);
2455         struct iscsi_conn *conn = req->conn;
2456         int status = scst_cmd_get_status(scst_cmd);
2457         u8 *sense = scst_cmd_get_sense_buffer(scst_cmd);
2458         int sense_len = scst_cmd_get_sense_buffer_len(scst_cmd);
2459         int old_state = req->scst_state;
2460         bool single_only = !scst_get_long_xmit(scst_cmd);
2461
2462         scst_cmd_set_tgt_priv(scst_cmd, NULL);
2463
2464         req->tmfabort |= scst_cmd_aborted(scst_cmd) ? 1 : 0;
2465         if (unlikely(req->tmfabort)) {
2466                 TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
2467                         req->scst_cmd);
2468                 if (old_state == ISCSI_CMD_STATE_RESTARTED) {
2469                         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
2470                         req_cmnd_release_force(req, ISCSI_FORCE_RELEASE_WRITE);
2471                 } else
2472                         iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_PROCESSED);
2473                 goto out;
2474         }
2475
2476         if (unlikely(old_state != ISCSI_CMD_STATE_RESTARTED)) {
2477                 TRACE_DBG("req %p on %d state", req, old_state);
2478                 create_status_rsp(req, status, sense, sense_len);
2479                 switch(old_state) {
2480                 case ISCSI_CMD_STATE_RX_CMD:
2481                 case ISCSI_CMD_STATE_AFTER_PREPROC:
2482                         break;
2483                 default:
2484                         sBUG();
2485                 }
2486                 iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_PROCESSED);
2487                 goto out;
2488         }
2489
2490         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
2491
2492         req->bufflen = scst_cmd_get_resp_data_len(scst_cmd);
2493         req->sg = scst_cmd_get_sg(scst_cmd);
2494
2495         TRACE_DBG("req %p, resp_flags=%x, req->bufflen=%d, req->sg=%p", req,
2496                 resp_flags, req->bufflen, req->sg);
2497
2498         if (unlikely((req->bufflen != 0) &&
2499                      !(resp_flags & SCST_TSC_FLAG_STATUS))) {
2500                 PRINT_ERROR("%s", "Sending DATA without STATUS is unsupported");
2501                 scst_set_cmd_error(scst_cmd,
2502                         SCST_LOAD_SENSE(scst_sense_hardw_error));
2503                 resp_flags = scst_cmd_get_tgt_resp_flags(scst_cmd);
2504                 sBUG();
2505         }
2506
2507         if (req->bufflen != 0) {
2508                 /* 
2509                  * Check above makes sure that SCST_TSC_FLAG_STATUS is set,
2510                  * so status is valid here, but in future that could change.
2511                  * ToDo
2512                  */
2513                 if (status != SAM_STAT_CHECK_CONDITION) {
2514                         send_data_rsp(req, status,
2515                                 resp_flags & SCST_TSC_FLAG_STATUS);
2516                 } else {
2517                         struct iscsi_cmnd *rsp;
2518                         struct iscsi_scsi_rsp_hdr *rsp_hdr;
2519                         int resid;
2520                         send_data_rsp(req, 0, 0);
2521                         if (resp_flags & SCST_TSC_FLAG_STATUS) {
2522                                 rsp = create_status_rsp(req, status, sense,
2523                                         sense_len);
2524                                 rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
2525                                 resid = cmnd_read_size(req) - req->bufflen;
2526                                 if (resid > 0) {
2527                                         rsp_hdr->flags |=
2528                                                 ISCSI_FLG_RESIDUAL_UNDERFLOW;
2529                                         rsp_hdr->residual_count = cpu_to_be32(resid);
2530                                 } else if (resid < 0) {
2531                                         rsp_hdr->flags |=
2532                                                 ISCSI_FLG_RESIDUAL_OVERFLOW;
2533                                         rsp_hdr->residual_count = cpu_to_be32(-resid);
2534                                 }
2535                                 iscsi_cmnd_init_write(rsp,
2536                                         ISCSI_INIT_WRITE_REMOVE_HASH);
2537                         }
2538                 }
2539         } else if (resp_flags & SCST_TSC_FLAG_STATUS) {
2540                 struct iscsi_cmnd *rsp;
2541                 struct iscsi_scsi_rsp_hdr *rsp_hdr;
2542                 u32 resid;
2543                 rsp = create_status_rsp(req, status, sense, sense_len);
2544                 rsp_hdr = (struct iscsi_scsi_rsp_hdr *) &rsp->pdu.bhs;
2545                 resid = cmnd_read_size(req);
2546                 if (resid != 0) {
2547                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
2548                         rsp_hdr->residual_count = cpu_to_be32(resid);
2549                 }
2550                 iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH);
2551         }
2552 #ifdef EXTRACHECKS
2553         else
2554                 sBUG();
2555 #endif
2556
2557         conn_get_ordered(conn);
2558         req_cmnd_release(req);
2559         iscsi_try_local_processing(conn, single_only);
2560         conn_put(conn);
2561
2562 out:
2563         return SCST_TGT_RES_SUCCESS;
2564 }
2565
2566 /* Called under sn_lock */
2567 static bool iscsi_is_delay_tm_resp(struct iscsi_cmnd *rsp)
2568 {
2569         bool res = 0;
2570         struct iscsi_task_mgt_hdr *req_hdr =
2571                 (struct iscsi_task_mgt_hdr *)&rsp->parent_req->pdu.bhs;
2572         int function = req_hdr->function & ISCSI_FUNCTION_MASK;
2573         struct iscsi_session *sess = rsp->conn->session;
2574
2575         TRACE_ENTRY();
2576
2577         /* This should be checked for immediate TM commands as well */
2578
2579         switch(function) {
2580         default:
2581                 if (before(sess->exp_cmd_sn, req_hdr->cmd_sn))
2582                         res = 1;
2583                 break;
2584         }
2585
2586         TRACE_EXIT_RES(res);
2587         return res;
2588 }
2589
2590 /* Called under sn_lock, but might drop it inside, then reaquire */
2591 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess)
2592 {
2593         struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
2594
2595         TRACE_ENTRY();
2596
2597         if (tm_rsp == NULL)
2598                 goto out;
2599
2600         if (iscsi_is_delay_tm_resp(tm_rsp))
2601                 goto out;
2602
2603         TRACE(TRACE_MGMT_MINOR, "Sending delayed rsp %p", tm_rsp);
2604
2605         sess->tm_rsp = NULL;
2606         sess->tm_active = 0;
2607
2608         spin_unlock(&sess->sn_lock);
2609
2610         iscsi_cmnd_init_write(tm_rsp,
2611                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2612
2613         spin_lock(&sess->sn_lock);
2614
2615 out:
2616         TRACE_EXIT();
2617         return;
2618 }
2619
2620 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status)
2621 {
2622         struct iscsi_cmnd *rsp;
2623         struct iscsi_task_mgt_hdr *req_hdr =
2624                                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2625         struct iscsi_task_rsp_hdr *rsp_hdr;
2626         struct iscsi_session *sess = req->conn->session;
2627         int fn = req_hdr->function & ISCSI_FUNCTION_MASK;
2628
2629         TRACE_ENTRY();
2630
2631         TRACE_MGMT_DBG("TM req %p finished", req);
2632         TRACE((req_hdr->function == ISCSI_FUNCTION_ABORT_TASK) ?
2633                          TRACE_MGMT_MINOR : TRACE_MGMT,
2634                 "TM fn %d finished, status %d", fn, status);
2635
2636         rsp = iscsi_cmnd_create_rsp_cmnd(req);
2637         rsp_hdr = (struct iscsi_task_rsp_hdr *)&rsp->pdu.bhs;
2638
2639         rsp_hdr->opcode = ISCSI_OP_SCSI_TASK_MGT_RSP;
2640         rsp_hdr->flags = ISCSI_FLG_FINAL;
2641         rsp_hdr->itt = req_hdr->itt;
2642         rsp_hdr->response = status;
2643
2644         if (fn == ISCSI_FUNCTION_TARGET_COLD_RESET)
2645                 rsp->should_close_conn = 1;
2646
2647         sBUG_ON(sess->tm_rsp != NULL);
2648
2649         spin_lock(&sess->sn_lock);
2650         if (iscsi_is_delay_tm_resp(rsp)) {
2651                 TRACE(TRACE_MGMT_MINOR, "Delaying TM fn %x response %p "
2652                         "(req %p), because not all affected commands received "
2653                         "(TM cmd sn %u, exp sn %u)",
2654                         req_hdr->function & ISCSI_FUNCTION_MASK, rsp, req,
2655                         req_hdr->cmd_sn, sess->exp_cmd_sn);
2656                 sess->tm_rsp = rsp;
2657                 spin_unlock(&sess->sn_lock);
2658                 goto out_release;
2659         }
2660         sess->tm_active = 0;
2661         spin_unlock(&sess->sn_lock);
2662
2663         iscsi_cmnd_init_write(rsp,
2664                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2665
2666 out_release:
2667         req_cmnd_release(req);
2668
2669         TRACE_EXIT();
2670         return;
2671 }
2672
2673 static inline int iscsi_get_mgmt_response(int status)
2674 {
2675         switch(status) {
2676         case SCST_MGMT_STATUS_SUCCESS:
2677                 return ISCSI_RESPONSE_FUNCTION_COMPLETE;
2678
2679         case SCST_MGMT_STATUS_TASK_NOT_EXIST:
2680                 return ISCSI_RESPONSE_UNKNOWN_TASK;
2681
2682         case SCST_MGMT_STATUS_LUN_NOT_EXIST:
2683                 return ISCSI_RESPONSE_UNKNOWN_LUN;
2684
2685         case SCST_MGMT_STATUS_FN_NOT_SUPPORTED:
2686                 return ISCSI_RESPONSE_FUNCTION_UNSUPPORTED;
2687
2688         case SCST_MGMT_STATUS_REJECTED:
2689         case SCST_MGMT_STATUS_FAILED:
2690         default:
2691                 return ISCSI_RESPONSE_FUNCTION_REJECTED;
2692         }
2693 }
2694
2695 static void iscsi_task_mgmt_fn_done(struct scst_mgmt_cmd *scst_mcmd)
2696 {
2697         struct iscsi_cmnd *req = (struct iscsi_cmnd*)
2698                                 scst_mgmt_cmd_get_tgt_priv(scst_mcmd);
2699         int status = iscsi_get_mgmt_response(scst_mgmt_cmd_get_status(scst_mcmd));
2700
2701         TRACE_MGMT_DBG("req %p, scst_mcmd %p, fn %d, scst status %d",
2702                 req, scst_mcmd, scst_mgmt_cmd_get_fn(scst_mcmd),
2703                 scst_mgmt_cmd_get_status(scst_mcmd));
2704
2705         iscsi_send_task_mgmt_resp(req, status);
2706
2707         scst_mgmt_cmd_set_tgt_priv(scst_mcmd, NULL);
2708
2709         return;
2710 }
2711
2712 static int iscsi_target_detect(struct scst_tgt_template *templ)
2713 {
2714         /* Nothing to do */
2715         return 0;
2716 }
2717
2718 static int iscsi_target_release(struct scst_tgt *scst_tgt)
2719 {
2720         /* Nothing to do */
2721         return 0;
2722 }
2723
2724 struct scst_tgt_template iscsi_template = {
2725         .name = "iscsi",
2726         .sg_tablesize = ISCSI_CONN_IOV_MAX,
2727         .threads_num = 0,
2728         .no_clustering = 1,
2729         .xmit_response_atomic = 0,
2730         .preprocessing_done_atomic = 1,
2731         .detect = iscsi_target_detect,
2732         .release = iscsi_target_release,
2733         .xmit_response = iscsi_xmit_response,
2734 #ifndef NET_PAGE_CALLBACKS_DEFINED
2735         .alloc_data_buf = iscsi_alloc_data_buf,
2736 #endif
2737         .preprocessing_done = iscsi_preprocessing_done,
2738         .pre_exec = iscsi_pre_exec,
2739         .task_mgmt_fn_done = iscsi_task_mgmt_fn_done,
2740 };
2741
2742 static __init int iscsi_run_threads(int count, char *name, int (*fn)(void *))
2743 {
2744         int res = 0;
2745         int i;
2746         struct iscsi_thread_t *thr;
2747
2748         for (i = 0; i < count; i++) {
2749                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
2750                 if (!thr) {
2751                         res = -ENOMEM;
2752                         PRINT_ERROR("Failed to allocate thr %d", res);
2753                         goto out;
2754                 }
2755                 thr->thr = kthread_run(fn, NULL, "%s%d", name, i);
2756                 if (IS_ERR(thr->thr)) {
2757                         res = PTR_ERR(thr->thr);
2758                         PRINT_ERROR("kthread_create() failed: %d", res);
2759                         kfree(thr);
2760                         goto out;
2761                 }
2762                 list_add(&thr->threads_list_entry, &iscsi_threads_list);
2763         }
2764
2765 out:
2766         return res;
2767 }
2768
2769 static void iscsi_stop_threads(void)
2770 {
2771         struct iscsi_thread_t *t, *tmp;
2772
2773         list_for_each_entry_safe(t, tmp, &iscsi_threads_list,
2774                                 threads_list_entry) {
2775                 int rc = kthread_stop(t->thr);
2776                 if (rc < 0) {
2777                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
2778                 }
2779                 list_del(&t->threads_list_entry);
2780                 kfree(t);
2781         }
2782 }
2783
2784 static int __init iscsi_init(void)
2785 {
2786         int err;
2787         int num;
2788
2789         PRINT_INFO("iSCSI SCST Target - version %s", ISCSI_VERSION_STRING);
2790
2791 #ifdef NET_PAGE_CALLBACKS_DEFINED
2792         err = net_set_get_put_page_callbacks(iscsi_get_page_callback,
2793                         iscsi_put_page_callback);
2794         if (err != 0) {
2795                 PRINT_INFO("Unable to set page callbackes: %d", err);
2796                 goto out;
2797         }
2798 #else
2799         PRINT_INFO("%s", "Patch put_page_callback-<kernel-version>.patch "
2800                 "not applied on your kernel. Running in the performance "
2801                 "degraded mode. Refer README file for details");
2802 #endif
2803
2804         BUILD_BUG_ON(MAX_DATA_SEG_LEN != (ISCSI_CONN_IOV_MAX<<PAGE_SHIFT));
2805
2806         if ((ctr_major = register_chrdev(0, ctr_name, &ctr_fops)) < 0) {
2807                 PRINT_ERROR("failed to register the control device %d", ctr_major);
2808                 err = ctr_major;
2809                 goto out_callb;
2810         }
2811
2812         if ((err = event_init()) < 0)
2813                 goto out_reg;
2814
2815         iscsi_cmnd_cache = KMEM_CACHE(iscsi_cmnd, SCST_SLAB_FLAGS);
2816         if (!iscsi_cmnd_cache) {
2817                 err = -ENOMEM;
2818                 goto out_event;
2819         }
2820
2821         err = scst_register_target_template(&iscsi_template);
2822         if (err < 0)
2823                 goto out_kmem;
2824
2825         iscsi_template_registered = 1;
2826
2827         if ((err = iscsi_procfs_init()) < 0)
2828                 goto out_reg_tmpl;
2829
2830         num = max(num_online_cpus(), 2);
2831
2832         err = iscsi_run_threads(num, "iscsird", istrd);
2833         if (err != 0)
2834                 goto out_thr;
2835
2836         err = iscsi_run_threads(num, "iscsiwr", istwr);
2837         if (err != 0)
2838                 goto out_thr;
2839
2840 out:
2841         return err;
2842
2843 out_thr:
2844         iscsi_procfs_exit();
2845         iscsi_stop_threads();
2846
2847 out_reg_tmpl:
2848         scst_unregister_target_template(&iscsi_template);
2849
2850 out_kmem:
2851         kmem_cache_destroy(iscsi_cmnd_cache);
2852
2853 out_event:
2854         event_exit();
2855
2856 out_reg:
2857         unregister_chrdev(ctr_major, ctr_name);
2858
2859 out_callb:
2860 #ifdef NET_PAGE_CALLBACKS_DEFINED
2861         net_set_get_put_page_callbacks(NULL, NULL);
2862 #endif  
2863         goto out;
2864 }
2865
2866 static void __exit iscsi_exit(void)
2867 {
2868         iscsi_stop_threads();
2869
2870         unregister_chrdev(ctr_major, ctr_name);
2871
2872         iscsi_procfs_exit();
2873         event_exit();
2874
2875         kmem_cache_destroy(iscsi_cmnd_cache);
2876
2877         scst_unregister_target_template(&iscsi_template);
2878
2879 #ifdef NET_PAGE_CALLBACKS_DEFINED
2880         net_set_get_put_page_callbacks(NULL, NULL);
2881 #endif
2882 }
2883
2884 module_init(iscsi_init);
2885 module_exit(iscsi_exit);
2886
2887 MODULE_LICENSE("GPL");