- Removed artificial limitation on sg_tablesize. It isn't needed, because the regula...
[mirror/scst/.git] / iscsi-scst / kernel / iscsi.c
1 /*
2  *  Copyright (C) 2002 - 2003 Ardis Technolgies <roman@ardistech.com>
3  *  Copyright (C) 2007 - 2008 Vladislav Bolkhovitin
4  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
5  *
6  *  This program is free software; you can redistribute it and/or
7  *  modify it under the terms of the GNU General Public License
8  *  as published by the Free Software Foundation, version 2
9  *  of the License.
10  *
11  *  This program is distributed in the hope that it will be useful,
12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  *  GNU General Public License for more details.
15  */
16
17 #include <linux/module.h>
18 #include <linux/hash.h>
19 #include <linux/kthread.h>
20 #include <net/tcp.h>
21 #include <scsi/scsi.h>
22
23 #include "iscsi.h"
24 #include "digest.h"
25
26 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
27 #warning "Patch put_page_callback-<kernel-version>.patch not applied on your \
28         kernel or CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION \
29         config option not set. ISCSI-SCST will be working with not the best \
30         performance. Refer README file for details."
31 #endif
32
33 #define ISCSI_INIT_WRITE_WAKE           0x1
34 #define ISCSI_INIT_WRITE_REMOVE_HASH    0x2
35
36 static int ctr_major;
37 static char ctr_name[] = "iscsi-scst-ctl";
38 static int iscsi_template_registered;
39
40 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
41 unsigned long iscsi_trace_flag = ISCSI_DEFAULT_LOG_FLAGS;
42 #endif
43
44 static struct kmem_cache *iscsi_cmnd_cache;
45
46 DEFINE_SPINLOCK(iscsi_rd_lock);
47 LIST_HEAD(iscsi_rd_list);
48 DECLARE_WAIT_QUEUE_HEAD(iscsi_rd_waitQ);
49
50 DEFINE_SPINLOCK(iscsi_wr_lock);
51 LIST_HEAD(iscsi_wr_list);
52 DECLARE_WAIT_QUEUE_HEAD(iscsi_wr_waitQ);
53
54 static struct page *dummy_page;
55 static struct scatterlist dummy_sg;
56
57 struct iscsi_thread_t {
58         struct task_struct *thr;
59         struct list_head threads_list_entry;
60 };
61
62 static LIST_HEAD(iscsi_threads_list);
63
64 static void cmnd_remove_hash(struct iscsi_cmnd *cmnd);
65 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status);
66 static void cmnd_prepare_get_rejected_cmd_data(struct iscsi_cmnd *cmnd);
67 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess);
68 static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd);
69 static void req_cmnd_release(struct iscsi_cmnd *req);
70
71 static inline u32 cmnd_write_size(struct iscsi_cmnd *cmnd)
72 {
73         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
74
75         if (hdr->flags & ISCSI_CMD_WRITE)
76                 return be32_to_cpu(hdr->data_length);
77         return 0;
78 }
79
80 static inline u32 cmnd_read_size(struct iscsi_cmnd *cmnd)
81 {
82         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
83
84         if (hdr->flags & ISCSI_CMD_READ) {
85                 struct iscsi_rlength_ahdr *ahdr =
86                         (struct iscsi_rlength_ahdr *)cmnd->pdu.ahs;
87
88                 if (!(hdr->flags & ISCSI_CMD_WRITE))
89                         return be32_to_cpu(hdr->data_length);
90                 if (ahdr && ahdr->ahstype == ISCSI_AHSTYPE_RLENGTH)
91                         return be32_to_cpu(ahdr->read_length);
92         }
93         return 0;
94 }
95
96 static inline void iscsi_restart_cmnd(struct iscsi_cmnd *cmnd)
97 {
98         EXTRACHECKS_BUG_ON(cmnd->data_waiting);
99
100         cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
101         scst_restart_cmd(cmnd->scst_cmd, SCST_PREPROCESS_STATUS_SUCCESS,
102                 SCST_CONTEXT_THREAD);
103         return;
104 }
105
106 static inline void iscsi_restart_waiting_cmnd(struct iscsi_cmnd *cmnd)
107 {
108         /*
109          * There is no race with conn_abort(), since all functions
110          * called from single read thread
111          */
112         iscsi_extracheck_is_rd_thread(cmnd->conn);
113         cmnd->data_waiting = 0;
114
115         iscsi_restart_cmnd(cmnd);
116         return;
117 }
118
119 static inline void iscsi_fail_waiting_cmnd(struct iscsi_cmnd *cmnd)
120 {
121         TRACE_MGMT_DBG("Failing data waiting cmd %p", cmnd);
122
123         /*
124          * There is no race with conn_abort(), since all functions
125          * called from single read thread
126          */
127         iscsi_extracheck_is_rd_thread(cmnd->conn);
128         cmnd->data_waiting = 0;
129
130         req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
131 }
132
133 struct iscsi_cmnd *cmnd_alloc(struct iscsi_conn *conn,
134                               struct iscsi_cmnd *parent)
135 {
136         struct iscsi_cmnd *cmnd;
137
138         /* ToDo: __GFP_NOFAIL?? */
139 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17)
140         cmnd = kmem_cache_alloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
141         memset(cmnd, 0, sizeof(*cmnd));
142 #else
143         cmnd = kmem_cache_zalloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
144 #endif
145
146         atomic_set(&cmnd->ref_cnt, 1);
147         cmnd->scst_state = ISCSI_CMD_STATE_NEW;
148         cmnd->conn = conn;
149         cmnd->parent_req = parent;
150         init_waitqueue_head(&cmnd->scst_waitQ);
151
152         if (parent == NULL) {
153                 conn_get(conn);
154
155 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
156                 atomic_set(&cmnd->net_ref_cnt, 0);
157 #endif
158                 spin_lock_init(&cmnd->rsp_cmd_lock);
159                 INIT_LIST_HEAD(&cmnd->rsp_cmd_list);
160                 INIT_LIST_HEAD(&cmnd->rx_ddigest_cmd_list);
161
162                 spin_lock_bh(&conn->cmd_list_lock);
163                 list_add_tail(&cmnd->cmd_list_entry, &conn->cmd_list);
164                 spin_unlock_bh(&conn->cmd_list_lock);
165         }
166
167         TRACE_DBG("conn %p, parent %p, cmnd %p", conn, parent, cmnd);
168         return cmnd;
169 }
170
171 /* Frees a command. Also frees the additional header. */
172 static void cmnd_free(struct iscsi_cmnd *cmnd)
173 {
174         TRACE_DBG("%p", cmnd);
175
176         if (unlikely(cmnd->tm_aborted)) {
177                 TRACE_MGMT_DBG("Free aborted cmd %p (scst cmd %p, state %d, "
178                         "parent_req %p)", cmnd, cmnd->scst_cmd,
179                         cmnd->scst_state, cmnd->parent_req);
180         }
181
182         /* Catch users from cmd_list or rsp_cmd_list */
183         EXTRACHECKS_BUG_ON(atomic_read(&cmnd->ref_cnt) != 0);
184
185         kfree(cmnd->pdu.ahs);
186
187         if (unlikely(cmnd->on_write_list || cmnd->on_written_list)) {
188                 struct iscsi_scsi_cmd_hdr *req = cmnd_hdr(cmnd);
189
190                 PRINT_CRIT_ERROR("cmnd %p still on some list?, %x, %x, %x, "
191                         "%x, %x, %x, %x", cmnd, req->opcode, req->scb[0],
192                         req->flags, req->itt, be32_to_cpu(req->data_length),
193                         req->cmd_sn, be32_to_cpu(cmnd->pdu.datasize));
194
195                 if (unlikely(cmnd->parent_req)) {
196                         struct iscsi_scsi_cmd_hdr *preq =
197                                         cmnd_hdr(cmnd->parent_req);
198                         PRINT_CRIT_ERROR("%p %x %u", preq, preq->opcode,
199                                 preq->scb[0]);
200                 }
201                 sBUG();
202         }
203
204         kmem_cache_free(iscsi_cmnd_cache, cmnd);
205         return;
206 }
207
208 void cmnd_done(struct iscsi_cmnd *cmnd)
209 {
210         TRACE_DBG("%p", cmnd);
211
212         if (unlikely(cmnd->tm_aborted)) {
213                 TRACE_MGMT_DBG("Done aborted cmd %p (scst cmd %p, state %d, "
214                         "parent_req %p)", cmnd, cmnd->scst_cmd,
215                         cmnd->scst_state, cmnd->parent_req);
216         }
217
218         EXTRACHECKS_BUG_ON(cmnd->on_rx_digest_list);
219
220         if (cmnd->on_written_list) {
221                 struct iscsi_conn *conn = cmnd->conn;
222                 TRACE_DBG("Deleting cmd %p from conn %p written_list", cmnd,
223                         conn);
224                 spin_lock_bh(&conn->write_list_lock);
225                 list_del(&cmnd->write_list_entry);
226                 cmnd->on_written_list = 0;
227                 spin_unlock_bh(&conn->write_list_lock);
228         }
229
230         if (cmnd->parent_req == NULL) {
231                 struct iscsi_conn *conn = cmnd->conn;
232                 TRACE_DBG("Deleting req %p from conn %p", cmnd, conn);
233
234                 spin_lock_bh(&conn->cmd_list_lock);
235                 list_del(&cmnd->cmd_list_entry);
236                 spin_unlock_bh(&conn->cmd_list_lock);
237
238                 conn_put(conn);
239
240                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rsp_cmd_list));
241                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rx_ddigest_cmd_list));
242
243                 /* Order between above and below code is important! */
244
245                 if (cmnd->scst_cmd) {
246                         switch (cmnd->scst_state) {
247                         case ISCSI_CMD_STATE_PROCESSED:
248                                 TRACE_DBG("cmd %p PROCESSED", cmnd);
249                                 scst_tgt_cmd_done(cmnd->scst_cmd,
250                                         SCST_CONTEXT_DIRECT);
251                                 break;
252                         case ISCSI_CMD_STATE_AFTER_PREPROC:
253                         {
254                                 struct scst_cmd *scst_cmd = cmnd->scst_cmd;
255                                 TRACE_DBG("cmd %p AFTER_PREPROC", cmnd);
256                                 cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
257                                 cmnd->scst_cmd = NULL;
258                                 scst_restart_cmd(scst_cmd,
259                                         SCST_PREPROCESS_STATUS_ERROR_FATAL,
260                                         SCST_CONTEXT_THREAD);
261                                 break;
262                         }
263                         default:
264                                 PRINT_CRIT_ERROR("Unexpected cmnd scst state "
265                                         "%d", cmnd->scst_state);
266                                 sBUG();
267                                 break;
268                         }
269                 }
270         } else {
271                 EXTRACHECKS_BUG_ON(cmnd->scst_cmd != NULL);
272                 TRACE_DBG("Deleting rsp %p from parent %p", cmnd,
273                         cmnd->parent_req);
274
275                 spin_lock_bh(&cmnd->parent_req->rsp_cmd_lock);
276                 list_del(&cmnd->rsp_cmd_list_entry);
277                 spin_unlock_bh(&cmnd->parent_req->rsp_cmd_lock);
278
279                 cmnd_put(cmnd->parent_req);
280         }
281
282         /* Order between above and below code is important! */
283
284         if (cmnd->own_sg) {
285                 TRACE_DBG("%s", "own_sg");
286                 if (cmnd->sg != &dummy_sg)
287                         scst_free(cmnd->sg, cmnd->sg_cnt);
288 #ifdef CONFIG_SCST_DEBUG
289                 cmnd->own_sg = 0;
290                 cmnd->sg = NULL;
291                 cmnd->sg_cnt = -1;
292 #endif
293         }
294
295         if (cmnd->dec_active_cmnds) {
296                 struct iscsi_session *sess = cmnd->conn->session;
297                 TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
298                         "new value %d)", cmnd, sess,
299                         atomic_read(&sess->active_cmds)-1);
300                 atomic_dec(&sess->active_cmds);
301 #ifdef CONFIG_SCST_EXTRACHECKS
302                 if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
303                         PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
304                                 atomic_read(&sess->active_cmds));
305                         sBUG();
306                 }
307 #endif
308         }
309
310         cmnd_free(cmnd);
311         return;
312 }
313
314 /*
315  * Corresponding conn may also gets destroyed atfer this function, except only
316  * if it's called from the read thread!
317  *
318  * It can't be called in parallel with iscsi_cmnds_init_write()!
319  */
320 void req_cmnd_release_force(struct iscsi_cmnd *req, int flags)
321 {
322         struct iscsi_cmnd *rsp, *t;
323         struct iscsi_conn *conn = req->conn;
324         LIST_HEAD(cmds_list);
325
326         TRACE_ENTRY();
327
328         TRACE_MGMT_DBG("%p", req);
329
330         sBUG_ON(req == conn->read_cmnd);
331
332         if (flags & ISCSI_FORCE_RELEASE_WRITE) {
333                 spin_lock_bh(&conn->write_list_lock);
334                 list_for_each_entry_safe(rsp, t, &conn->write_list,
335                                                 write_list_entry) {
336                         if (rsp->parent_req != req)
337                                 continue;
338
339                         cmd_del_from_write_list(rsp);
340
341                         list_add_tail(&rsp->write_list_entry, &cmds_list);
342                 }
343                 spin_unlock_bh(&conn->write_list_lock);
344
345                 list_for_each_entry_safe(rsp, t, &cmds_list,
346                                                 write_list_entry) {
347                         TRACE_MGMT_DBG("Putting write rsp %p", rsp);
348                         list_del(&rsp->write_list_entry);
349                         cmnd_put(rsp);
350                 }
351         }
352
353 again_rsp:
354         spin_lock_bh(&req->rsp_cmd_lock);
355         list_for_each_entry_reverse(rsp, &req->rsp_cmd_list,
356                         rsp_cmd_list_entry) {
357                 bool r;
358
359                 if (rsp->force_cleanup_done)
360                         continue;
361
362                 rsp->force_cleanup_done = 1;
363
364                 if (cmnd_get_check(rsp))
365                         continue;
366
367                 spin_unlock_bh(&req->rsp_cmd_lock);
368
369                 spin_lock_bh(&conn->write_list_lock);
370                 r = rsp->on_write_list || rsp->write_processing_started;
371                 spin_unlock_bh(&conn->write_list_lock);
372
373                 cmnd_put(rsp);
374
375                 if (r)
376                         goto again_rsp;
377
378                 /*
379                  * If both on_write_list and write_processing_started not set,
380                  * we can safely put() rsp.
381                  */
382                 TRACE_MGMT_DBG("Putting rsp %p", rsp);
383                 cmnd_put(rsp);
384                 goto again_rsp;
385         }
386         spin_unlock_bh(&req->rsp_cmd_lock);
387
388         req_cmnd_release(req);
389
390         TRACE_EXIT();
391         return;
392 }
393
394 /*
395  * Corresponding conn may also gets destroyed atfer this function, except only
396  * if it's called from the read thread!
397  */
398 static void req_cmnd_release(struct iscsi_cmnd *req)
399 {
400         struct iscsi_cmnd *c, *t;
401
402         TRACE_ENTRY();
403
404         TRACE_DBG("%p", req);
405
406 #ifdef CONFIG_SCST_EXTRACHECKS
407         sBUG_ON(req->release_called);
408         req->release_called = 1;
409 #endif
410
411         if (unlikely(req->tm_aborted)) {
412                 TRACE_MGMT_DBG("Release aborted req cmd %p (scst cmd %p, "
413                         "state %d)", req, req->scst_cmd, req->scst_state);
414         }
415
416         sBUG_ON(req->parent_req != NULL);
417
418         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
419                                 rx_ddigest_cmd_list_entry) {
420                 cmd_del_from_rx_ddigest_list(c);
421                 cmnd_put(c);
422         }
423
424         if (req->hashed)
425                 cmnd_remove_hash(req);
426
427         if (req->dec_active_cmnds) {
428                 struct iscsi_session *sess = req->conn->session;
429                 TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
430                         "new value %d)", req, sess,
431                         atomic_read(&sess->active_cmds)-1);
432                 atomic_dec(&sess->active_cmds);
433                 req->dec_active_cmnds = 0;
434 #ifdef CONFIG_SCST_EXTRACHECKS
435                 if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
436                         PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
437                                 atomic_read(&sess->active_cmds));
438                         sBUG();
439                 }
440 #endif
441         }
442
443         cmnd_put(req);
444
445         TRACE_EXIT();
446         return;
447 }
448
449 /*
450  * Corresponding conn may also gets destroyed atfer this function, except only
451  * if it's called from the read thread!
452  */
453 void rsp_cmnd_release(struct iscsi_cmnd *cmnd)
454 {
455         TRACE_DBG("%p", cmnd);
456
457 #ifdef CONFIG_SCST_EXTRACHECKS
458         sBUG_ON(cmnd->release_called);
459         cmnd->release_called = 1;
460 #endif
461
462         sBUG_ON(cmnd->hashed);
463         sBUG_ON(cmnd->parent_req == NULL);
464
465         cmnd_put(cmnd);
466         return;
467 }
468
469 /**
470  * create a new command used as response.
471  *
472  * iscsi_cmnd_create_rsp_cmnd -
473  * @cmnd: ptr to request command
474  *
475  * @return    ptr to response command or NULL
476  */
477 static struct iscsi_cmnd *iscsi_cmnd_create_rsp_cmnd(struct iscsi_cmnd *parent)
478 {
479         struct iscsi_cmnd *rsp;
480
481         rsp = cmnd_alloc(parent->conn, parent);
482
483         spin_lock_bh(&parent->rsp_cmd_lock);
484         TRACE_DBG("Adding rsp %p to parent %p", rsp, parent);
485         list_add_tail(&rsp->rsp_cmd_list_entry, &parent->rsp_cmd_list);
486         spin_unlock_bh(&parent->rsp_cmd_lock);
487         cmnd_get(parent);
488         return rsp;
489 }
490
491 static inline struct iscsi_cmnd *get_rsp_cmnd(struct iscsi_cmnd *req)
492 {
493         struct iscsi_cmnd *res = NULL;
494
495         /* Currently this lock isn't needed, but just in case.. */
496         spin_lock_bh(&req->rsp_cmd_lock);
497         if (!list_empty(&req->rsp_cmd_list)) {
498                 res = list_entry(req->rsp_cmd_list.prev, struct iscsi_cmnd,
499                         rsp_cmd_list_entry);
500         }
501         spin_unlock_bh(&req->rsp_cmd_lock);
502
503         return res;
504 }
505
506 static void iscsi_cmnds_init_write(struct list_head *send, int flags)
507 {
508         struct iscsi_cmnd *rsp = list_entry(send->next, struct iscsi_cmnd,
509                                                 write_list_entry);
510         struct iscsi_conn *conn = rsp->conn;
511         struct list_head *pos, *next;
512
513         sBUG_ON(list_empty(send));
514
515         /*
516          * If we don't remove hashed req cmd from the hash list here, before
517          * submitting it for transmittion, we will have a race, when for
518          * some reason cmd's release is delayed after transmittion and
519          * initiator sends cmd with the same ITT => this command will be
520          * erroneously rejected as a duplicate.
521          */
522         if ((flags & ISCSI_INIT_WRITE_REMOVE_HASH) &&
523             rsp->parent_req->hashed &&
524             (rsp->parent_req->r2t_length == 0) &&
525             (rsp->parent_req->outstanding_r2t == 0))
526                 cmnd_remove_hash(rsp->parent_req);
527
528         if (!(conn->ddigest_type & DIGEST_NONE)) {
529                 list_for_each(pos, send) {
530                         rsp = list_entry(pos, struct iscsi_cmnd,
531                                                 write_list_entry);
532
533                         if (rsp->pdu.datasize != 0) {
534                                 TRACE_DBG("Doing data digest (%p:%x)", rsp,
535                                         cmnd_opcode(rsp));
536                                 digest_tx_data(rsp);
537                         }
538                 }
539         }
540
541         spin_lock_bh(&conn->write_list_lock);
542         list_for_each_safe(pos, next, send) {
543                 rsp = list_entry(pos, struct iscsi_cmnd, write_list_entry);
544
545                 TRACE_DBG("%p:%x", rsp, cmnd_opcode(rsp));
546
547                 sBUG_ON(conn != rsp->conn);
548
549                 list_del(&rsp->write_list_entry);
550                 cmd_add_on_write_list(conn, rsp);
551         }
552         spin_unlock_bh(&conn->write_list_lock);
553
554         if (flags & ISCSI_INIT_WRITE_WAKE)
555                 iscsi_make_conn_wr_active(conn);
556
557         return;
558 }
559
560 static void iscsi_cmnd_init_write(struct iscsi_cmnd *rsp, int flags)
561 {
562         LIST_HEAD(head);
563
564         if (unlikely(rsp->on_write_list)) {
565                 PRINT_CRIT_ERROR("cmd already on write list (%x %x %x %x %u "
566                         "%u %u %u %u %u %u %d %d",
567                         cmnd_itt(rsp), cmnd_ttt(rsp), cmnd_opcode(rsp),
568                         cmnd_scsicode(rsp), rsp->r2t_sn,
569                         rsp->r2t_length, rsp->is_unsolicited_data,
570                         rsp->target_task_tag, rsp->outstanding_r2t,
571                         rsp->hdigest, rsp->ddigest,
572                         list_empty(&rsp->rsp_cmd_list), rsp->hashed);
573                 sBUG();
574         }
575         list_add(&rsp->write_list_entry, &head);
576         iscsi_cmnds_init_write(&head, flags);
577         return;
578 }
579
580 static void iscsi_set_datasize(struct iscsi_cmnd *cmnd, u32 offset, u32 size)
581 {
582         cmnd->pdu.datasize = size;
583
584         if (size & 3) {
585                 u32 last_off = offset + size;
586                 int idx = last_off >> PAGE_SHIFT;
587                 u8 *p = (u8 *)page_address(sg_page(&cmnd->sg[idx])) +
588                         (last_off & ~PAGE_MASK);
589                 int i = 4 - (size & 3);
590                 while (i--)
591                         *p++ = 0;
592         }
593         return;
594 }
595
596 static void send_data_rsp(struct iscsi_cmnd *req, u8 status, int send_status)
597 {
598         struct iscsi_cmnd *rsp;
599         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
600         struct iscsi_data_in_hdr *rsp_hdr;
601         u32 pdusize, expsize, scsisize, size, offset, sn;
602         LIST_HEAD(send);
603
604         TRACE_DBG("req %p", req);
605
606         pdusize = req->conn->session->sess_param.max_xmit_data_length;
607         expsize = cmnd_read_size(req);
608         size = min(expsize, (u32)req->bufflen);
609         offset = 0;
610         sn = 0;
611
612         while (1) {
613                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
614                 TRACE_DBG("rsp %p", rsp);
615                 rsp->sg = req->sg;
616                 rsp->sg_cnt = req->sg_cnt;
617                 rsp->bufflen = req->bufflen;
618                 rsp_hdr = (struct iscsi_data_in_hdr *)&rsp->pdu.bhs;
619
620                 rsp_hdr->opcode = ISCSI_OP_SCSI_DATA_IN;
621                 rsp_hdr->itt = req_hdr->itt;
622                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
623                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
624                 rsp_hdr->data_sn = cpu_to_be32(sn);
625
626                 if (size <= pdusize) {
627                         TRACE_DBG("offset %d, size %d", offset, size);
628                         iscsi_set_datasize(rsp, offset, size);
629                         if (send_status) {
630                                 TRACE_DBG("status %x", status);
631                                 rsp_hdr->flags =
632                                         ISCSI_FLG_FINAL | ISCSI_FLG_STATUS;
633                                 rsp_hdr->cmd_status = status;
634                         }
635                         scsisize = req->bufflen;
636                         if (scsisize < expsize) {
637                                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
638                                 size = expsize - scsisize;
639                         } else if (scsisize > expsize) {
640                                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
641                                 size = scsisize - expsize;
642                         } else
643                                 size = 0;
644                         rsp_hdr->residual_count = cpu_to_be32(size);
645                         list_add_tail(&rsp->write_list_entry, &send);
646                         break;
647                 }
648
649                 TRACE_DBG("pdusize %d, offset %d, size %d", pdusize, offset,
650                         size);
651
652                 iscsi_set_datasize(rsp, offset, pdusize);
653
654                 size -= pdusize;
655                 offset += pdusize;
656                 sn++;
657
658                 list_add_tail(&rsp->write_list_entry, &send);
659         }
660         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_REMOVE_HASH);
661         return;
662 }
663
664 static struct iscsi_cmnd *create_status_rsp(struct iscsi_cmnd *req, int status,
665         const u8 *sense_buf, int sense_len)
666 {
667         struct iscsi_cmnd *rsp;
668         struct iscsi_scsi_rsp_hdr *rsp_hdr;
669         struct iscsi_sense_data *sense;
670         struct scatterlist *sg;
671
672         rsp = iscsi_cmnd_create_rsp_cmnd(req);
673         TRACE_DBG("%p", rsp);
674
675         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
676         rsp_hdr->opcode = ISCSI_OP_SCSI_RSP;
677         rsp_hdr->flags = ISCSI_FLG_FINAL;
678         rsp_hdr->response = ISCSI_RESPONSE_COMMAND_COMPLETED;
679         rsp_hdr->cmd_status = status;
680         rsp_hdr->itt = cmnd_hdr(req)->itt;
681
682         if (SCST_SENSE_VALID(sense_buf)) {
683                 TRACE_DBG("%s", "SENSE VALID");
684                 /* ToDo: __GFP_NOFAIL ?? */
685                 sg = rsp->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
686                                         &rsp->sg_cnt);
687                 if (sg == NULL) {
688                         ;/* ToDo */;
689                 }
690                 rsp->own_sg = 1;
691                 sense = (struct iscsi_sense_data *)page_address(sg_page(&sg[0]));
692                 sense->length = cpu_to_be16(sense_len);
693                 memcpy(sense->data, sense_buf, sense_len);
694                 rsp->pdu.datasize = sizeof(struct iscsi_sense_data) + sense_len;
695                 rsp->bufflen = (rsp->pdu.datasize + 3) & -4;
696                 if (rsp->bufflen - rsp->pdu.datasize) {
697                         unsigned int i = rsp->pdu.datasize;
698                         u8 *p = (u8 *)sense + i;
699
700                         while (i < rsp->bufflen) {
701                                 *p++ = 0;
702                                 i++;
703                         }
704                 }
705         } else {
706                 rsp->pdu.datasize = 0;
707                 rsp->bufflen = 0;
708         }
709
710         return rsp;
711 }
712
713 static struct iscsi_cmnd *create_sense_rsp(struct iscsi_cmnd *req,
714         u8 sense_key, u8 asc, u8 ascq)
715 {
716         u8 sense[14];
717         memset(sense, 0, sizeof(sense));
718         sense[0] = 0xf0;
719         sense[2] = sense_key;
720         sense[7] = 6;   /* Additional sense length */
721         sense[12] = asc;
722         sense[13] = ascq;
723         return create_status_rsp(req, SAM_STAT_CHECK_CONDITION, sense,
724                 sizeof(sense));
725 }
726
727 static void iscsi_cmnd_reject(struct iscsi_cmnd *req, int reason)
728 {
729         struct iscsi_cmnd *rsp;
730         struct iscsi_reject_hdr *rsp_hdr;
731         struct scatterlist *sg;
732         char *addr;
733
734         TRACE_MGMT_DBG("Reject: req %p, reason %x", req, reason);
735
736         sBUG_ON(req->rejected);
737         req->rejected = 1;
738         req->reject_reason = ISCSI_REJECT_CMD;
739
740         rsp = iscsi_cmnd_create_rsp_cmnd(req);
741         rsp_hdr = (struct iscsi_reject_hdr *)&rsp->pdu.bhs;
742
743         rsp_hdr->opcode = ISCSI_OP_REJECT;
744         rsp_hdr->ffffffff = ISCSI_RESERVED_TAG;
745         rsp_hdr->reason = reason;
746
747         /* ToDo: __GFP_NOFAIL ?? */
748         sg = rsp->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
749                                 &rsp->sg_cnt);
750         if (sg == NULL) {
751                 ;/* ToDo */;
752         }
753         rsp->own_sg = 1;
754         addr = page_address(sg_page(&sg[0]));
755         clear_page(addr);
756         memcpy(addr, &req->pdu.bhs, sizeof(struct iscsi_hdr));
757         rsp->bufflen = rsp->pdu.datasize = sizeof(struct iscsi_hdr);
758
759         iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH |
760                                          ISCSI_INIT_WRITE_WAKE);
761
762         cmnd_prepare_get_rejected_cmd_data(req);
763         return;
764 }
765
766 static inline int iscsi_get_allowed_cmds(struct iscsi_session *sess)
767 {
768         int res = max(-1, (int)sess->max_queued_cmnds -
769                                 atomic_read(&sess->active_cmds)-1);
770         TRACE_DBG("allowed cmds %d (sess %p, active_cmds %d)", res,
771                 sess, atomic_read(&sess->active_cmds));
772         return res;
773 }
774
775 static u32 cmnd_set_sn(struct iscsi_cmnd *cmnd, int set_stat_sn)
776 {
777         struct iscsi_conn *conn = cmnd->conn;
778         struct iscsi_session *sess = conn->session;
779         u32 res;
780
781         spin_lock(&sess->sn_lock);
782
783         if (set_stat_sn)
784                 cmnd->pdu.bhs.sn = cpu_to_be32(conn->stat_sn++);
785         cmnd->pdu.bhs.exp_sn = cpu_to_be32(sess->exp_cmd_sn);
786         cmnd->pdu.bhs.max_sn = cpu_to_be32(sess->exp_cmd_sn +
787                                  iscsi_get_allowed_cmds(sess));
788
789         res = cpu_to_be32(conn->stat_sn);
790
791         spin_unlock(&sess->sn_lock);
792         return res;
793 }
794
795 /* Called under sn_lock */
796 static void __update_stat_sn(struct iscsi_cmnd *cmnd)
797 {
798         struct iscsi_conn *conn = cmnd->conn;
799         u32 exp_stat_sn;
800
801         cmnd->pdu.bhs.exp_sn = exp_stat_sn = be32_to_cpu(cmnd->pdu.bhs.exp_sn);
802         TRACE_DBG("%x,%x", cmnd_opcode(cmnd), exp_stat_sn);
803         if ((int)(exp_stat_sn - conn->exp_stat_sn) > 0 &&
804             (int)(exp_stat_sn - conn->stat_sn) <= 0) {
805                 /* free pdu resources */
806                 cmnd->conn->exp_stat_sn = exp_stat_sn;
807         }
808         return;
809 }
810
811 static inline void update_stat_sn(struct iscsi_cmnd *cmnd)
812 {
813         spin_lock(&cmnd->conn->session->sn_lock);
814         __update_stat_sn(cmnd);
815         spin_unlock(&cmnd->conn->session->sn_lock);
816         return;
817 }
818
819 /* Called under sn_lock */
820 static int check_cmd_sn(struct iscsi_cmnd *cmnd)
821 {
822         struct iscsi_session *session = cmnd->conn->session;
823         u32 cmd_sn;
824
825         cmnd->pdu.bhs.sn = cmd_sn = be32_to_cpu(cmnd->pdu.bhs.sn);
826         TRACE_DBG("%d(%d)", cmd_sn, session->exp_cmd_sn);
827         if (likely((s32)(cmd_sn - session->exp_cmd_sn) >= 0))
828                 return 0;
829         PRINT_ERROR("sequence error (%x,%x)", cmd_sn, session->exp_cmd_sn);
830         return -ISCSI_REASON_PROTOCOL_ERROR;
831 }
832
833 static inline struct iscsi_cmnd *__cmnd_find_hash(
834         struct iscsi_session *session, u32 itt, u32 ttt)
835 {
836         struct list_head *head;
837         struct iscsi_cmnd *cmnd;
838
839         head = &session->cmnd_hash[cmnd_hashfn(itt)];
840
841         list_for_each_entry(cmnd, head, hash_list_entry) {
842                 if (cmnd->pdu.bhs.itt == itt) {
843                         if (ttt != ISCSI_RESERVED_TAG &&
844                             ttt != cmnd->target_task_tag)
845                                 continue;
846                         return cmnd;
847                 }
848         }
849         return NULL;
850 }
851
852 static struct iscsi_cmnd *cmnd_find_hash(struct iscsi_session *session,
853         u32 itt, u32 ttt)
854 {
855         struct iscsi_cmnd *cmnd;
856
857         spin_lock(&session->cmnd_hash_lock);
858         cmnd = __cmnd_find_hash(session, itt, ttt);
859         spin_unlock(&session->cmnd_hash_lock);
860
861         return cmnd;
862 }
863
864 static struct iscsi_cmnd *cmnd_find_hash_get(struct iscsi_session *session,
865         u32 itt, u32 ttt)
866 {
867         struct iscsi_cmnd *cmnd;
868
869         spin_lock(&session->cmnd_hash_lock);
870         cmnd = __cmnd_find_hash(session, itt, ttt);
871         if (cmnd != NULL) {
872                 if (unlikely(cmnd_get_check(cmnd)))
873                         cmnd = NULL;
874         }
875         spin_unlock(&session->cmnd_hash_lock);
876
877         return cmnd;
878 }
879
880 static int cmnd_insert_hash(struct iscsi_cmnd *cmnd)
881 {
882         struct iscsi_session *session = cmnd->conn->session;
883         struct iscsi_cmnd *tmp;
884         struct list_head *head;
885         int err = 0;
886         u32 itt = cmnd->pdu.bhs.itt;
887
888         TRACE_DBG("%p:%x", cmnd, itt);
889         if (unlikely(itt == ISCSI_RESERVED_TAG)) {
890                 PRINT_ERROR("%s", "ITT is RESERVED_TAG");
891                 PRINT_BUFFER("Incorrect BHS", &cmnd->pdu.bhs,
892                         sizeof(cmnd->pdu.bhs));
893                 err = -ISCSI_REASON_PROTOCOL_ERROR;
894                 goto out;
895         }
896
897         spin_lock(&session->cmnd_hash_lock);
898
899         head = &session->cmnd_hash[cmnd_hashfn(cmnd->pdu.bhs.itt)];
900
901         tmp = __cmnd_find_hash(session, itt, ISCSI_RESERVED_TAG);
902         if (likely(!tmp)) {
903                 list_add_tail(&cmnd->hash_list_entry, head);
904                 cmnd->hashed = 1;
905         } else {
906                 PRINT_ERROR("Task %x in progress, cmnd %p", itt, cmnd);
907                 err = -ISCSI_REASON_TASK_IN_PROGRESS;
908         }
909
910         spin_unlock(&session->cmnd_hash_lock);
911
912         if (likely(!err)) {
913                 spin_lock(&session->sn_lock);
914                 __update_stat_sn(cmnd);
915                 err = check_cmd_sn(cmnd);
916                 spin_unlock(&session->sn_lock);
917         }
918
919 out:
920         return err;
921 }
922
923 static void cmnd_remove_hash(struct iscsi_cmnd *cmnd)
924 {
925         struct iscsi_session *session = cmnd->conn->session;
926         struct iscsi_cmnd *tmp;
927
928         spin_lock(&session->cmnd_hash_lock);
929
930         tmp = __cmnd_find_hash(session, cmnd->pdu.bhs.itt, ISCSI_RESERVED_TAG);
931
932         if (likely(tmp && tmp == cmnd)) {
933                 list_del(&cmnd->hash_list_entry);
934                 cmnd->hashed = 0;
935         } else {
936                 PRINT_ERROR("%p:%x not found", cmnd, cmnd_itt(cmnd));
937         }
938
939         spin_unlock(&session->cmnd_hash_lock);
940         return;
941 }
942
943 static void cmnd_prepare_get_rejected_cmd_data(struct iscsi_cmnd *cmnd)
944 {
945         struct iscsi_conn *conn = cmnd->conn;
946         struct scatterlist *sg = cmnd->sg;
947         char __user *addr;
948         u32 size;
949         unsigned int i;
950
951         TRACE_MGMT_DBG("Skipping (%p, %x %x %x %u, %p, scst state %d)", cmnd,
952                 cmnd_itt(cmnd), cmnd_opcode(cmnd), cmnd_hdr(cmnd)->scb[0],
953                 cmnd->pdu.datasize, cmnd->scst_cmd, cmnd->scst_state);
954
955         iscsi_extracheck_is_rd_thread(conn);
956
957         size = cmnd->pdu.datasize;
958         if (!size)
959                 return;
960
961         if (sg == NULL) {
962                 /*
963                  * There are no problems with the safety from concurrent
964                  * accesses to dummy_page in dummy_sg, since data only
965                  * will be read and then discarded.
966                  */
967                 sg = cmnd->sg = &dummy_sg;
968                 cmnd->bufflen = PAGE_SIZE;
969                 cmnd->own_sg = 1;
970         }
971
972         addr = (char __force __user *)(page_address(sg_page(&sg[0])));
973         sBUG_ON(addr == NULL);
974         size = (size + 3) & -4;
975         conn->read_size = size;
976         for (i = 0; size > PAGE_SIZE; i++, size -= cmnd->bufflen) {
977                 /* We already checked pdu.datasize in check_segment_length() */
978                 sBUG_ON(i >= ISCSI_CONN_IOV_MAX);
979                 conn->read_iov[i].iov_base = addr;
980                 conn->read_iov[i].iov_len = cmnd->bufflen;
981         }
982         conn->read_iov[i].iov_base = addr;
983         conn->read_iov[i].iov_len = size;
984         conn->read_msg.msg_iov = conn->read_iov;
985         conn->read_msg.msg_iovlen = ++i;
986
987         return;
988 }
989
990 static void cmnd_reject_scsi_cmd(struct iscsi_cmnd *req)
991 {
992         struct iscsi_cmnd *rsp;
993         struct iscsi_scsi_rsp_hdr *rsp_hdr;
994         u32 size;
995
996         TRACE_DBG("%p", req);
997
998         sBUG_ON(req->rejected);
999         req->rejected = 1;
1000         req->reject_reason = ISCSI_REJECT_SCSI_CMD;
1001
1002         rsp = get_rsp_cmnd(req);
1003         if (rsp == NULL) {
1004                 /* That can be true for aborted commands */
1005                 goto out_reject;
1006         }
1007
1008         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
1009
1010         sBUG_ON(cmnd_opcode(rsp) != ISCSI_OP_SCSI_RSP);
1011
1012         size = cmnd_write_size(req);
1013         if (size) {
1014                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
1015                 rsp_hdr->residual_count = cpu_to_be32(size);
1016         }
1017         size = cmnd_read_size(req);
1018         if (size) {
1019                 if (cmnd_hdr(req)->flags & ISCSI_CMD_WRITE) {
1020                         rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_UNDERFLOW;
1021                         rsp_hdr->bi_residual_count = cpu_to_be32(size);
1022                 } else {
1023                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
1024                         rsp_hdr->residual_count = cpu_to_be32(size);
1025                 }
1026         }
1027
1028         iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH |
1029                                          ISCSI_INIT_WRITE_WAKE);
1030
1031 out_reject:
1032         cmnd_prepare_get_rejected_cmd_data(req);
1033         return;
1034 }
1035
1036 static int cmnd_prepare_recv_pdu(struct iscsi_conn *conn,
1037         struct iscsi_cmnd *cmd, u32 offset, u32 size)
1038 {
1039         struct scatterlist *sg = cmd->sg;
1040         unsigned int bufflen = cmd->bufflen;
1041         unsigned int idx, i;
1042         char __user *addr;
1043         int res = 0;
1044
1045         TRACE_DBG("%p %u,%u", cmd->sg, offset, size);
1046
1047         iscsi_extracheck_is_rd_thread(conn);
1048
1049         if (unlikely((offset >= bufflen) ||
1050                      (offset + size > bufflen))) {
1051                 PRINT_ERROR("Wrong ltn (%u %u %u)", offset, size, bufflen);
1052                 mark_conn_closed(conn);
1053                 res = -EIO;
1054                 goto out;
1055         }
1056
1057         offset += sg[0].offset;
1058         idx = offset >> PAGE_SHIFT;
1059         offset &= ~PAGE_MASK;
1060
1061         conn->read_msg.msg_iov = conn->read_iov;
1062         conn->read_size = size = (size + 3) & -4;
1063
1064         i = 0;
1065         while (1) {
1066                 addr = (char __force __user *)(page_address(sg_page(&sg[idx])));
1067                 sBUG_ON(addr == NULL);
1068                 conn->read_iov[i].iov_base = addr + offset;
1069                 if (offset + size <= PAGE_SIZE) {
1070                         TRACE_DBG("idx=%d, offset=%u, size=%d, addr=%p",
1071                                 idx, offset, size, addr);
1072                         conn->read_iov[i].iov_len = size;
1073                         conn->read_msg.msg_iovlen = ++i;
1074                         break;
1075                 }
1076                 conn->read_iov[i].iov_len = PAGE_SIZE - offset;
1077                 TRACE_DBG("idx=%d, offset=%u, size=%d, iov_len=%zd, addr=%p",
1078                         idx, offset, size, conn->read_iov[i].iov_len, addr);
1079                 size -= conn->read_iov[i].iov_len;
1080                 offset = 0;
1081                 if (unlikely(++i >= ISCSI_CONN_IOV_MAX)) {
1082                         PRINT_ERROR("Initiator %s violated negotiated "
1083                                 "parameters by sending too much data (size "
1084                                 "left %d)", conn->session->initiator_name,
1085                                 size);
1086                         mark_conn_closed(conn);
1087                         res = -EINVAL;
1088                         break;
1089                 }
1090                 idx++;
1091         }
1092         TRACE_DBG("msg_iov=%p, msg_iovlen=%zd",
1093                 conn->read_msg.msg_iov, conn->read_msg.msg_iovlen);
1094
1095 out:
1096         return res;
1097 }
1098
1099 static void send_r2t(struct iscsi_cmnd *req)
1100 {
1101         struct iscsi_session *session = req->conn->session;
1102         struct iscsi_cmnd *rsp;
1103         struct iscsi_r2t_hdr *rsp_hdr;
1104         u32 offset, burst;
1105         LIST_HEAD(send);
1106
1107         if (unlikely(req->tm_aborted)) {
1108                 TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted on R2T "
1109                         "(r2t_length %d, outstanding_r2t %d)", req,
1110                         req->scst_cmd, req->r2t_length, req->outstanding_r2t);
1111                 if (req->outstanding_r2t == 0)
1112                         iscsi_fail_waiting_cmnd(req);
1113                 goto out;
1114         }
1115
1116         /*
1117          * There is no race with data_out_start() and conn_abort(), since
1118          * all functions called from single read thread
1119          */
1120         iscsi_extracheck_is_rd_thread(req->conn);
1121
1122         burst = session->sess_param.max_burst_length;
1123         offset = be32_to_cpu(cmnd_hdr(req)->data_length) - req->r2t_length;
1124
1125         do {
1126                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
1127                 rsp->pdu.bhs.ttt = req->target_task_tag;
1128                 rsp_hdr = (struct iscsi_r2t_hdr *)&rsp->pdu.bhs;
1129                 rsp_hdr->opcode = ISCSI_OP_R2T;
1130                 rsp_hdr->flags = ISCSI_FLG_FINAL;
1131                 rsp_hdr->lun = cmnd_hdr(req)->lun;
1132                 rsp_hdr->itt = cmnd_hdr(req)->itt;
1133                 rsp_hdr->r2t_sn = cpu_to_be32(req->r2t_sn++);
1134                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
1135                 if (req->r2t_length > burst) {
1136                         rsp_hdr->data_length = cpu_to_be32(burst);
1137                         req->r2t_length -= burst;
1138                         offset += burst;
1139                 } else {
1140                         rsp_hdr->data_length = cpu_to_be32(req->r2t_length);
1141                         req->r2t_length = 0;
1142                 }
1143
1144                 TRACE_WRITE("%x %u %u %u %u", cmnd_itt(req),
1145                         be32_to_cpu(rsp_hdr->data_length),
1146                         be32_to_cpu(rsp_hdr->buffer_offset),
1147                         be32_to_cpu(rsp_hdr->r2t_sn), req->outstanding_r2t);
1148
1149                 list_add_tail(&rsp->write_list_entry, &send);
1150
1151                 if (++req->outstanding_r2t >= session->sess_param.max_outstanding_r2t)
1152                         break;
1153
1154         } while (req->r2t_length != 0);
1155
1156         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_WAKE);
1157
1158 out:
1159         return;
1160 }
1161
1162 static int iscsi_pre_exec(struct scst_cmd *scst_cmd)
1163 {
1164         int res = SCST_PREPROCESS_STATUS_SUCCESS;
1165         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
1166                 scst_cmd_get_tgt_priv(scst_cmd);
1167         struct iscsi_cmnd *c, *t;
1168
1169         TRACE_ENTRY();
1170
1171         EXTRACHECKS_BUG_ON(scst_cmd_atomic(scst_cmd));
1172
1173         if (scst_cmd_get_data_direction(scst_cmd) == SCST_DATA_READ) {
1174                 EXTRACHECKS_BUG_ON(!list_empty(&req->rx_ddigest_cmd_list));
1175                 goto out;
1176         }
1177
1178         /* If data digest isn't used this list will be empty */
1179         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
1180                                 rx_ddigest_cmd_list_entry) {
1181                 TRACE_DBG("Checking digest of RX ddigest cmd %p", c);
1182                 if (digest_rx_data(c) != 0) {
1183                         scst_set_cmd_error(scst_cmd,
1184                                 SCST_LOAD_SENSE(iscsi_sense_crc_error));
1185                         res = SCST_PREPROCESS_STATUS_ERROR_SENSE_SET;
1186                         /*
1187                          * The rest of rx_ddigest_cmd_list will be freed
1188                          * in req_cmnd_release()
1189                          */
1190                         goto out;
1191                 }
1192                 cmd_del_from_rx_ddigest_list(c);
1193                 cmnd_put(c);
1194         }
1195
1196 out:
1197         TRACE_EXIT_RES(res);
1198         return res;
1199 }
1200
1201 static int noop_out_start(struct iscsi_cmnd *cmnd)
1202 {
1203         struct iscsi_conn *conn = cmnd->conn;
1204         u32 size, tmp;
1205         int i, err = 0;
1206
1207         TRACE_DBG("%p", cmnd);
1208
1209         iscsi_extracheck_is_rd_thread(conn);
1210
1211         if (unlikely(cmnd_ttt(cmnd) != cpu_to_be32(ISCSI_RESERVED_TAG))) {
1212                 /*
1213                  * We don't request a NOP-Out by sending a NOP-In.
1214                  * See 10.18.2 in the draft 20.
1215                  */
1216                 PRINT_ERROR("Initiator sent command with not RESERVED tag and "
1217                         "TTT %x", cmnd_itt(cmnd));
1218                 err = -ISCSI_REASON_PROTOCOL_ERROR;
1219                 goto out;
1220         }
1221
1222         if (cmnd_itt(cmnd) == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1223                 if (unlikely(!(cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE)))
1224                         PRINT_ERROR("%s", "Initiator sent RESERVED tag for "
1225                                 "non-immediate command");
1226                 spin_lock(&conn->session->sn_lock);
1227                 __update_stat_sn(cmnd);
1228                 err = check_cmd_sn(cmnd);
1229                 spin_unlock(&conn->session->sn_lock);
1230                 if (unlikely(err))
1231                         goto out;
1232         } else {
1233                 err = cmnd_insert_hash(cmnd);
1234                 if (unlikely(err < 0)) {
1235                         PRINT_ERROR("Can't insert in hash: ignore this "
1236                                 "request %x", cmnd_itt(cmnd));
1237                         goto out;
1238                 }
1239         }
1240
1241         size = cmnd->pdu.datasize;
1242
1243         if (size) {
1244                 size = (size + 3) & -4;
1245                 conn->read_msg.msg_iov = conn->read_iov;
1246                 if (cmnd->pdu.bhs.itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1247                         struct scatterlist *sg;
1248
1249                         cmnd->sg = sg = scst_alloc(size, GFP_KERNEL,
1250                                                 &cmnd->sg_cnt);
1251                         if (sg == NULL) {
1252                                 TRACE(TRACE_OUT_OF_MEM, "Allocating buffer for"
1253                                       " %d NOP-Out payload failed", size);
1254                                 err = -ISCSI_REASON_OUT_OF_RESOURCES;
1255                                 goto out;
1256                         }
1257
1258                         /* We already checked it in check_segment_length() */
1259                         sBUG_ON(cmnd->sg_cnt > (signed)ISCSI_CONN_IOV_MAX);
1260
1261                         cmnd->own_sg = 1;
1262                         cmnd->bufflen = size;
1263
1264                         for (i = 0; i < cmnd->sg_cnt; i++) {
1265                                 conn->read_iov[i].iov_base =
1266                                         (void __force __user *)(page_address(sg_page(&sg[i])));
1267                                 tmp = min_t(u32, size, PAGE_SIZE);
1268                                 conn->read_iov[i].iov_len = tmp;
1269                                 conn->read_size += tmp;
1270                                 size -= tmp;
1271                         }
1272                         sBUG_ON(size != 0);
1273                 } else {
1274                         /*
1275                          * There are no problems with the safety from concurrent
1276                          * accesses to dummy_page, since for ISCSI_RESERVED_TAG
1277                          * the data only read and then discarded.
1278                          */
1279                         for (i = 0; i < (signed)ISCSI_CONN_IOV_MAX; i++) {
1280                                 conn->read_iov[i].iov_base =
1281                                         (void __force __user *)(page_address(dummy_page));
1282                                 tmp = min_t(u32, size, PAGE_SIZE);
1283                                 conn->read_iov[i].iov_len = tmp;
1284                                 conn->read_size += tmp;
1285                                 size -= tmp;
1286                         }
1287
1288                         /* We already checked size in check_segment_length() */
1289                         sBUG_ON(size != 0);
1290                 }
1291
1292                 conn->read_msg.msg_iovlen = i;
1293                 TRACE_DBG("msg_iov=%p, msg_iovlen=%zd", conn->read_msg.msg_iov,
1294                         conn->read_msg.msg_iovlen);
1295         }
1296
1297 out:
1298         return err;
1299 }
1300
1301 static inline u32 get_next_ttt(struct iscsi_conn *conn)
1302 {
1303         u32 ttt;
1304         struct iscsi_session *session = conn->session;
1305
1306         iscsi_extracheck_is_rd_thread(conn);
1307
1308         if (session->next_ttt == ISCSI_RESERVED_TAG)
1309                 session->next_ttt++;
1310         ttt = session->next_ttt++;
1311
1312         return cpu_to_be32(ttt);
1313 }
1314
1315 static int scsi_cmnd_start(struct iscsi_cmnd *req)
1316 {
1317         struct iscsi_conn *conn = req->conn;
1318         struct iscsi_session *session = conn->session;
1319         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1320         struct scst_cmd *scst_cmd;
1321         scst_data_direction dir;
1322         int res = 0;
1323
1324         TRACE_ENTRY();
1325
1326         TRACE_DBG("scsi command: %02x", req_hdr->scb[0]);
1327
1328         TRACE_DBG("Incrementing active_cmds (cmd %p, sess %p, "
1329                 "new value %d)", req, session,
1330                 atomic_read(&session->active_cmds)+1);
1331         atomic_inc(&session->active_cmds);
1332         req->dec_active_cmnds = 1;
1333
1334         scst_cmd = scst_rx_cmd(session->scst_sess,
1335                 (uint8_t *)&req_hdr->lun, sizeof(req_hdr->lun),
1336                 req_hdr->scb, sizeof(req_hdr->scb), SCST_NON_ATOMIC);
1337         if (scst_cmd == NULL) {
1338                 create_status_rsp(req, SAM_STAT_BUSY, NULL, 0);
1339                 cmnd_reject_scsi_cmd(req);
1340                 goto out;
1341         }
1342
1343         req->scst_cmd = scst_cmd;
1344         scst_cmd_set_tag(scst_cmd, req_hdr->itt);
1345         scst_cmd_set_tgt_priv(scst_cmd, req);
1346
1347         if (req_hdr->flags & ISCSI_CMD_READ) {
1348                 dir = SCST_DATA_READ;
1349 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
1350                 scst_cmd_set_tgt_need_alloc_data_buf(scst_cmd);
1351 #endif
1352         } else if (req_hdr->flags & ISCSI_CMD_WRITE)
1353                 dir = SCST_DATA_WRITE;
1354         else
1355                 dir = SCST_DATA_NONE;
1356         scst_cmd_set_expected(scst_cmd, dir,
1357                 be32_to_cpu(req_hdr->data_length));
1358
1359         switch (req_hdr->flags & ISCSI_CMD_ATTR_MASK) {
1360         case ISCSI_CMD_SIMPLE:
1361                 scst_cmd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1362                 break;
1363         case ISCSI_CMD_HEAD_OF_QUEUE:
1364                 scst_cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1365                 break;
1366         case ISCSI_CMD_ORDERED:
1367                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1368                 break;
1369         case ISCSI_CMD_ACA:
1370                 scst_cmd->queue_type = SCST_CMD_QUEUE_ACA;
1371                 break;
1372         case ISCSI_CMD_UNTAGGED:
1373                 scst_cmd->queue_type = SCST_CMD_QUEUE_UNTAGGED;
1374                 break;
1375         default:
1376                 PRINT_ERROR("Unknown task code %x, use ORDERED instead",
1377                         req_hdr->flags & ISCSI_CMD_ATTR_MASK);
1378                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1379                 break;
1380         }
1381
1382         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
1383         scst_cmd_set_tgt_sn(scst_cmd, req_hdr->cmd_sn);
1384
1385         TRACE_DBG("START Command (tag %d, queue_type %d)",
1386                 req_hdr->itt, scst_cmd->queue_type);
1387         req->scst_state = ISCSI_CMD_STATE_RX_CMD;
1388         scst_cmd_init_stage1_done(scst_cmd, SCST_CONTEXT_DIRECT, 0);
1389
1390         wait_event(req->scst_waitQ, req->scst_state != ISCSI_CMD_STATE_RX_CMD);
1391
1392         if (unlikely(req->scst_state != ISCSI_CMD_STATE_AFTER_PREPROC)) {
1393                 TRACE_DBG("req %p is in %x state", req, req->scst_state);
1394                 if (req->scst_state == ISCSI_CMD_STATE_PROCESSED) {
1395                         cmnd_reject_scsi_cmd(req);
1396                         goto out;
1397                 }
1398                 if (unlikely(req->tm_aborted)) {
1399                         TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
1400                                 req->scst_cmd);
1401                         cmnd_prepare_get_rejected_cmd_data(req);
1402                         goto out;
1403                 }
1404                 sBUG();
1405         }
1406
1407         dir = scst_cmd_get_data_direction(scst_cmd);
1408         if (dir != SCST_DATA_WRITE) {
1409                 if (unlikely(!(req_hdr->flags & ISCSI_CMD_FINAL) ||
1410                              req->pdu.datasize)) {
1411                         PRINT_ERROR("Unexpected unsolicited data (ITT %x "
1412                                 "CDB %x", cmnd_itt(req), req_hdr->scb[0]);
1413                         create_sense_rsp(req, ABORTED_COMMAND, 0xc, 0xc);
1414                         cmnd_reject_scsi_cmd(req);
1415                         goto out;
1416                 }
1417         }
1418
1419         if (dir == SCST_DATA_WRITE) {
1420                 req->is_unsolicited_data = !(req_hdr->flags & ISCSI_CMD_FINAL);
1421                 req->r2t_length = be32_to_cpu(req_hdr->data_length) -
1422                                         req->pdu.datasize;
1423                 if (req->r2t_length > 0)
1424                         req->data_waiting = 1;
1425         }
1426         req->target_task_tag = get_next_ttt(conn);
1427         req->sg = scst_cmd_get_sg(scst_cmd);
1428         req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
1429         req->bufflen = scst_cmd_get_bufflen(scst_cmd);
1430         if (unlikely(req->r2t_length > req->bufflen)) {
1431                 PRINT_ERROR("req->r2t_length %d > req->bufflen %d",
1432                         req->r2t_length, req->bufflen);
1433                 req->r2t_length = req->bufflen;
1434         }
1435
1436         TRACE_DBG("req=%p, dir=%d, is_unsolicited_data=%d, "
1437                 "r2t_length=%d, bufflen=%d", req, dir,
1438                 req->is_unsolicited_data, req->r2t_length, req->bufflen);
1439
1440         if (unlikely(!session->sess_param.immediate_data &&
1441                      req->pdu.datasize)) {
1442                 PRINT_ERROR("Initiator %s violated negotiated paremeters: "
1443                         "forbidden immediate data sent (ITT %x, op  %x)",
1444                         session->initiator_name, cmnd_itt(req),
1445                         req_hdr->scb[0]);
1446                 res = -EINVAL;
1447                 goto out;
1448         }
1449
1450         if (unlikely(session->sess_param.initial_r2t &&
1451                      !(req_hdr->flags & ISCSI_CMD_FINAL))) {
1452                 PRINT_ERROR("Initiator %s violated negotiated paremeters: "
1453                         "initial R2T is required (ITT %x, op  %x)",
1454                         session->initiator_name, cmnd_itt(req),
1455                         req_hdr->scb[0]);
1456                 res = -EINVAL;
1457                 goto out;
1458         }
1459
1460         if (req->pdu.datasize) {
1461                 if (unlikely(dir != SCST_DATA_WRITE)) {
1462                         PRINT_ERROR("pdu.datasize(%d) >0, but dir(%x) isn't "
1463                                 "WRITE", req->pdu.datasize, dir);
1464                         create_sense_rsp(req, ABORTED_COMMAND, 0xc, 0xc);
1465                         cmnd_reject_scsi_cmd(req);
1466                 } else
1467                         res = cmnd_prepare_recv_pdu(conn, req, 0,
1468                                 req->pdu.datasize);
1469         }
1470 out:
1471         /* Aborted commands will be freed in cmnd_rx_end() */
1472         TRACE_EXIT_RES(res);
1473         return res;
1474 }
1475
1476 static int data_out_start(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
1477 {
1478         struct iscsi_data_out_hdr *req_hdr =
1479                 (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1480         struct iscsi_cmnd *orig_req = NULL;
1481         u32 offset = be32_to_cpu(req_hdr->buffer_offset);
1482         int res = 0;
1483
1484         TRACE_ENTRY();
1485
1486         /*
1487          * There is no race with send_r2t() and conn_abort(), since
1488          * all functions called from single read thread
1489          */
1490         iscsi_extracheck_is_rd_thread(cmnd->conn);
1491
1492         update_stat_sn(cmnd);
1493
1494         cmnd->cmd_req = orig_req = cmnd_find_hash(conn->session, req_hdr->itt,
1495                                         req_hdr->ttt);
1496         if (unlikely(orig_req == NULL)) {
1497                 /* It might happen if req was aborted and then freed */
1498                 TRACE(TRACE_MGMT_MINOR, "Unable to find scsi task %x %x",
1499                         cmnd_itt(cmnd), cmnd_ttt(cmnd));
1500                 goto out_reject;
1501         }
1502
1503         if (orig_req->is_unsolicited_data) {
1504                 if (unlikely(orig_req->r2t_length < cmnd->pdu.datasize)) {
1505                         PRINT_ERROR("Data size (%d) > R2T length (%d)",
1506                                 cmnd->pdu.datasize, orig_req->r2t_length);
1507                         mark_conn_closed(conn);
1508                         res = -EINVAL;
1509                         goto out;
1510                 }
1511                 orig_req->r2t_length -= cmnd->pdu.datasize;
1512         }
1513
1514         /* Check unsolicited burst data */
1515         if (unlikely((req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) &&
1516                      (orig_req->pdu.bhs.flags & ISCSI_FLG_FINAL))) {
1517                 PRINT_ERROR("Unexpected data from %x %x",
1518                         cmnd_itt(cmnd), cmnd_ttt(cmnd));
1519                 mark_conn_closed(conn);
1520                 res = -EINVAL;
1521                 goto out;
1522         }
1523
1524         TRACE_WRITE("%u %p %p %u %u", req_hdr->ttt, cmnd, orig_req,
1525                 offset, cmnd->pdu.datasize);
1526
1527         res = cmnd_prepare_recv_pdu(conn, orig_req, offset, cmnd->pdu.datasize);
1528
1529 out:
1530         TRACE_EXIT_RES(res);
1531         return res;
1532
1533 out_reject:
1534         sBUG_ON(cmnd->rejected);
1535         cmnd->rejected = 1;
1536         cmnd->reject_reason = ISCSI_REJECT_DATA;
1537         cmnd_prepare_get_rejected_cmd_data(cmnd);
1538         goto out;
1539 }
1540
1541 static void data_out_end(struct iscsi_cmnd *cmnd)
1542 {
1543         struct iscsi_data_out_hdr *req_hdr =
1544                 (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1545         struct iscsi_cmnd *req;
1546
1547         sBUG_ON(cmnd == NULL);
1548         req = cmnd->cmd_req;
1549         sBUG_ON(req == NULL);
1550
1551         TRACE_DBG("cmnd %p, req %p", cmnd, req);
1552
1553         iscsi_extracheck_is_rd_thread(cmnd->conn);
1554
1555         if (!(cmnd->conn->ddigest_type & DIGEST_NONE) &&
1556             !cmnd->ddigest_checked) {
1557                 cmd_add_on_rx_ddigest_list(req, cmnd);
1558                 cmnd_get(cmnd);
1559         }
1560
1561         if (req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1562                 TRACE_DBG("ISCSI_RESERVED_TAG, FINAL %x",
1563                         req_hdr->flags & ISCSI_FLG_FINAL);
1564
1565                 if (req_hdr->flags & ISCSI_FLG_FINAL) {
1566                         req->is_unsolicited_data = 0;
1567                         if (req->pending)
1568                                 goto out_put;
1569                 } else
1570                         goto out_put;
1571         } else {
1572                 TRACE_DBG("FINAL %x, outstanding_r2t %d, r2t_length %d",
1573                         req_hdr->flags & ISCSI_FLG_FINAL,
1574                         req->outstanding_r2t, req->r2t_length);
1575
1576                 if (req_hdr->flags & ISCSI_FLG_FINAL) {
1577                         if (unlikely(req->is_unsolicited_data)) {
1578                                 PRINT_ERROR("Unexpected unsolicited data "
1579                                         "(r2t_length %u, outstanding_r2t %d)",
1580                                         req->r2t_length,
1581                                         req->is_unsolicited_data);
1582                                 mark_conn_closed(req->conn);
1583                                 goto out_put;
1584                         }
1585                         req->outstanding_r2t--;
1586                 } else
1587                         goto out_put;
1588         }
1589
1590         if (req->r2t_length != 0) {
1591                 if (!req->is_unsolicited_data)
1592                         send_r2t(req);
1593         } else
1594                 iscsi_restart_waiting_cmnd(req);
1595
1596 out_put:
1597         cmnd_put(cmnd);
1598         return;
1599 }
1600
1601 static void __cmnd_abort(struct iscsi_cmnd *cmnd)
1602 {
1603         /*
1604          * Here, if cmnd is data_waiting, we should iscsi_fail_waiting_cmnd()
1605          * it. But, since this function can be called from any thread, not only
1606          * from the read one, we at the moment can't do that, because of
1607          * absence of appropriate locking protection. But this isn't a stuff
1608          * for 1.0.0. So, currently a misbehaving initiator, not sending
1609          * data in R2T state for a sharing between targets device, for which
1610          * for some reason an aborting TM command, e.g. TARGET RESET, from
1611          * another initiator is issued, can block response for this TM command
1612          * virtually forever and by this make the issuing initiator eventually
1613          * put the device offline.
1614          *
1615          * ToDo in the next version, possibly a simple connection mutex, taken
1616          * by the read thread before starting any processing and by this
1617          * function, should be sufficient.
1618          */
1619
1620         TRACE_MGMT_DBG("Aborting cmd %p, scst_cmd %p (scst state %x, "
1621                 "ref_cnt %d, itt %x, sn %u, op %x, r2t_len %x, CDB op %x, "
1622                 "size to write %u, is_unsolicited_data %d, "
1623                 "outstanding_r2t %d, data_waiting %d, sess->exp_cmd_sn %u, "
1624                 "conn %p, rd_task %p)", cmnd, cmnd->scst_cmd, cmnd->scst_state,
1625                 atomic_read(&cmnd->ref_cnt), cmnd_itt(cmnd), cmnd->pdu.bhs.sn,
1626                 cmnd_opcode(cmnd), cmnd->r2t_length, cmnd_scsicode(cmnd),
1627                 cmnd_write_size(cmnd), cmnd->is_unsolicited_data,
1628                 cmnd->outstanding_r2t, cmnd->data_waiting,
1629                 cmnd->conn->session->exp_cmd_sn, cmnd->conn,
1630                 cmnd->conn->rd_task);
1631
1632 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
1633         TRACE_MGMT_DBG("net_ref_cnt %d", atomic_read(&cmnd->net_ref_cnt));
1634 #endif
1635
1636         cmnd->tm_aborted = 1;
1637
1638         return;
1639 }
1640
1641 /* Must be called from the read thread */
1642 static int cmnd_abort(struct iscsi_cmnd *req)
1643 {
1644         struct iscsi_session *session = req->conn->session;
1645         struct iscsi_task_mgt_hdr *req_hdr =
1646                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1647         struct iscsi_cmnd *cmnd;
1648         int err;
1649
1650         req_hdr->ref_cmd_sn = be32_to_cpu(req_hdr->ref_cmd_sn);
1651
1652         if (after(req_hdr->ref_cmd_sn, req_hdr->cmd_sn)) {
1653                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) > CmdSN(%u)",
1654                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn);
1655                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1656                 goto out;
1657         }
1658
1659         cmnd = cmnd_find_hash_get(session, req_hdr->rtt, ISCSI_RESERVED_TAG);
1660         if (cmnd) {
1661                 struct iscsi_conn *conn = cmnd->conn;
1662                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
1663
1664                 if (req_hdr->lun != hdr->lun) {
1665                         PRINT_ERROR("ABORT TASK: LUN mismatch: req LUN "
1666                                     "%llx, cmd LUN %llx, rtt %u",
1667                                     (long long unsigned int)req_hdr->lun,
1668                                     (long long unsigned int)hdr->lun,
1669                                     req_hdr->rtt);
1670                         err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1671                         goto out_put;
1672                 }
1673
1674                 if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
1675                         if (req_hdr->ref_cmd_sn != req_hdr->cmd_sn) {
1676                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != TM "
1677                                         "cmd CmdSN(%u) for immediate command "
1678                                         "%p", req_hdr->ref_cmd_sn,
1679                                         req_hdr->cmd_sn, cmnd);
1680                                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1681                                 goto out_put;
1682                         }
1683                 } else {
1684                         if (req_hdr->ref_cmd_sn != hdr->cmd_sn) {
1685                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != "
1686                                         "CmdSN(%u) for command %p",
1687                                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn,
1688                                         cmnd);
1689                                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1690                                 goto out_put;
1691                         }
1692                 }
1693
1694                 if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
1695                     (req_hdr->cmd_sn == hdr->cmd_sn)) {
1696                         PRINT_ERROR("ABORT TASK: SN mismatch: req SN %x, "
1697                                 "cmd SN %x, rtt %u", req_hdr->cmd_sn,
1698                                 hdr->cmd_sn, req_hdr->rtt);
1699                         err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1700                         goto out_put;
1701                 }
1702
1703                 spin_lock_bh(&conn->cmd_list_lock);
1704                 __cmnd_abort(cmnd);
1705                 spin_unlock_bh(&conn->cmd_list_lock);
1706
1707                 cmnd_put(cmnd);
1708                 err = 0;
1709         } else {
1710                 TRACE_MGMT_DBG("cmd RTT %x not found", req_hdr->rtt);
1711                 err = ISCSI_RESPONSE_UNKNOWN_TASK;
1712         }
1713
1714 out:
1715         return err;
1716
1717 out_put:
1718         cmnd_put(cmnd);
1719         goto out;
1720 }
1721
1722 /* Must be called from the read thread */
1723 static int target_abort(struct iscsi_cmnd *req, int all)
1724 {
1725         struct iscsi_target *target = req->conn->session->target;
1726         struct iscsi_task_mgt_hdr *req_hdr =
1727                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1728         struct iscsi_session *session;
1729         struct iscsi_conn *conn;
1730         struct iscsi_cmnd *cmnd;
1731
1732         mutex_lock(&target->target_mutex);
1733
1734         list_for_each_entry(session, &target->session_list,
1735                             session_list_entry) {
1736                 list_for_each_entry(conn, &session->conn_list,
1737                                     conn_list_entry) {
1738                         spin_lock_bh(&conn->cmd_list_lock);
1739                         list_for_each_entry(cmnd, &conn->cmd_list,
1740                                             cmd_list_entry) {
1741                                 if (cmnd == req)
1742                                         continue;
1743                                 if (all)
1744                                         __cmnd_abort(cmnd);
1745                                 else if (req_hdr->lun == cmnd_hdr(cmnd)->lun)
1746                                         __cmnd_abort(cmnd);
1747                         }
1748                         spin_unlock_bh(&conn->cmd_list_lock);
1749                 }
1750         }
1751
1752         mutex_unlock(&target->target_mutex);
1753         return 0;
1754 }
1755
1756 /* Must be called from the read thread */
1757 static void task_set_abort(struct iscsi_cmnd *req)
1758 {
1759         struct iscsi_session *session = req->conn->session;
1760         struct iscsi_task_mgt_hdr *req_hdr =
1761                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1762         struct iscsi_target *target = session->target;
1763         struct iscsi_conn *conn;
1764         struct iscsi_cmnd *cmnd;
1765
1766         mutex_lock(&target->target_mutex);
1767
1768         list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
1769                 spin_lock_bh(&conn->cmd_list_lock);
1770                 list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1771                         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
1772                         if (cmnd == req)
1773                                 continue;
1774                         if (req_hdr->lun != hdr->lun)
1775                                 continue;
1776                         if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
1777                             req_hdr->cmd_sn == hdr->cmd_sn)
1778                                 continue;
1779                         __cmnd_abort(cmnd);
1780                 }
1781                 spin_unlock_bh(&conn->cmd_list_lock);
1782         }
1783
1784         mutex_unlock(&target->target_mutex);
1785         return;
1786 }
1787
1788 /* Must be called from the read thread */
1789 void conn_abort(struct iscsi_conn *conn)
1790 {
1791         struct iscsi_cmnd *cmnd;
1792
1793         TRACE_MGMT_DBG("Aborting conn %p", conn);
1794
1795         iscsi_extracheck_is_rd_thread(conn);
1796
1797         spin_lock_bh(&conn->cmd_list_lock);
1798 again:
1799         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1800                 __cmnd_abort(cmnd);
1801                 if (cmnd->data_waiting) {
1802                         if (!cmnd_get_check(cmnd)) {
1803                                 spin_unlock_bh(&conn->cmd_list_lock);
1804
1805                                 /* ToDo: this is racy for MC/S */
1806                                 TRACE_MGMT_DBG("Restarting data waiting cmd "
1807                                         "%p", cmnd);
1808                                 iscsi_fail_waiting_cmnd(cmnd);
1809
1810                                 cmnd_put(cmnd);
1811
1812                                 /*
1813                                  * We are in the read thread, so we may not
1814                                  * worry that after cmnd release conn gets
1815                                  * released as well.
1816                                  */
1817                                 spin_lock_bh(&conn->cmd_list_lock);
1818                                 goto again;
1819                         }
1820                 }
1821         }
1822         spin_unlock_bh(&conn->cmd_list_lock);
1823
1824         return;
1825 }
1826
1827 static void execute_task_management(struct iscsi_cmnd *req)
1828 {
1829         struct iscsi_conn *conn = req->conn;
1830         struct iscsi_session *sess = conn->session;
1831         struct iscsi_task_mgt_hdr *req_hdr =
1832                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1833         int rc, status, function = req_hdr->function & ISCSI_FUNCTION_MASK;
1834         struct scst_rx_mgmt_params params;
1835
1836         TRACE((function == ISCSI_FUNCTION_ABORT_TASK) ?
1837                         TRACE_MGMT_MINOR : TRACE_MGMT,
1838                 "TM fn %d", function);
1839
1840         TRACE_MGMT_DBG("TM req %p, itt %x, rtt %x, sn %u, con %p", req,
1841                 cmnd_itt(req), req_hdr->rtt, req_hdr->cmd_sn, conn);
1842
1843         iscsi_extracheck_is_rd_thread(conn);
1844
1845         spin_lock(&sess->sn_lock);
1846         sess->tm_active++;
1847         sess->tm_sn = req_hdr->cmd_sn;
1848         if (sess->tm_rsp != NULL) {
1849                 struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
1850
1851                 TRACE(TRACE_MGMT_MINOR, "Dropping delayed TM rsp %p", tm_rsp);
1852
1853                 sess->tm_rsp = NULL;
1854                 sess->tm_active--;
1855
1856                 spin_unlock(&sess->sn_lock);
1857
1858                 sBUG_ON(sess->tm_active < 0);
1859
1860                 rsp_cmnd_release(tm_rsp);
1861         } else
1862                 spin_unlock(&sess->sn_lock);
1863
1864         memset(&params, 0, sizeof(params));
1865         params.atomic = SCST_NON_ATOMIC;
1866         params.tgt_priv = req;
1867
1868         if ((function != ISCSI_FUNCTION_ABORT_TASK) &&
1869             (req_hdr->rtt != ISCSI_RESERVED_TAG)) {
1870                 PRINT_ERROR("Invalid RTT %x (TM fn %x)", req_hdr->rtt,
1871                         function);
1872                 rc = -1;
1873                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1874                 goto reject;
1875         }
1876
1877         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
1878
1879         switch (function) {
1880         case ISCSI_FUNCTION_ABORT_TASK:
1881                 rc = -1;
1882                 status = cmnd_abort(req);
1883                 if (status == 0) {
1884                         params.fn = SCST_ABORT_TASK;
1885                         params.tag = req_hdr->rtt;
1886                         params.tag_set = 1;
1887                         params.lun = (uint8_t *)&req_hdr->lun;
1888                         params.lun_len = sizeof(req_hdr->lun);
1889                         params.lun_set = 1;
1890                         params.cmd_sn = req_hdr->cmd_sn;
1891                         params.cmd_sn_set = 1;
1892                         rc = scst_rx_mgmt_fn(conn->session->scst_sess,
1893                                 &params);
1894                         status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1895                 }
1896                 break;
1897         case ISCSI_FUNCTION_ABORT_TASK_SET:
1898                 task_set_abort(req);
1899                 params.fn = SCST_ABORT_TASK_SET;
1900                 params.lun = (uint8_t *)&req_hdr->lun;
1901                 params.lun_len = sizeof(req_hdr->lun);
1902                 params.lun_set = 1;
1903                 params.cmd_sn = req_hdr->cmd_sn;
1904                 params.cmd_sn_set = 1;
1905                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
1906                         &params);
1907                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1908                 break;
1909         case ISCSI_FUNCTION_CLEAR_TASK_SET:
1910                 task_set_abort(req);
1911                 params.fn = SCST_CLEAR_TASK_SET;
1912                 params.lun = (uint8_t *)&req_hdr->lun;
1913                 params.lun_len = sizeof(req_hdr->lun);
1914                 params.lun_set = 1;
1915                 params.cmd_sn = req_hdr->cmd_sn;
1916                 params.cmd_sn_set = 1;
1917                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
1918                         &params);
1919                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1920                 break;
1921         case ISCSI_FUNCTION_CLEAR_ACA:
1922                 params.fn = SCST_CLEAR_ACA;
1923                 params.lun = (uint8_t *)&req_hdr->lun;
1924                 params.lun_len = sizeof(req_hdr->lun);
1925                 params.lun_set = 1;
1926                 params.cmd_sn = req_hdr->cmd_sn;
1927                 params.cmd_sn_set = 1;
1928                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
1929                         &params);
1930                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1931                 break;
1932         case ISCSI_FUNCTION_TARGET_COLD_RESET:
1933         case ISCSI_FUNCTION_TARGET_WARM_RESET:
1934                 target_abort(req, 1);
1935                 params.fn = SCST_TARGET_RESET;
1936                 params.cmd_sn = req_hdr->cmd_sn;
1937                 params.cmd_sn_set = 1;
1938                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
1939                         &params);
1940                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1941                 break;
1942         case ISCSI_FUNCTION_LOGICAL_UNIT_RESET:
1943                 target_abort(req, 0);
1944                 params.fn = SCST_LUN_RESET;
1945                 params.lun = (uint8_t *)&req_hdr->lun;
1946                 params.lun_len = sizeof(req_hdr->lun);
1947                 params.lun_set = 1;
1948                 params.cmd_sn = req_hdr->cmd_sn;
1949                 params.cmd_sn_set = 1;
1950                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
1951                         &params);
1952                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1953                 break;
1954         case ISCSI_FUNCTION_TASK_REASSIGN:
1955                 rc = -1;
1956                 status = ISCSI_RESPONSE_ALLEGIANCE_REASSIGNMENT_UNSUPPORTED;
1957                 break;
1958         default:
1959                 PRINT_ERROR("Unknown TM function %d", function);
1960                 rc = -1;
1961                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1962                 break;
1963         }
1964
1965 reject:
1966         if (rc != 0)
1967                 iscsi_send_task_mgmt_resp(req, status);
1968
1969         return;
1970 }
1971
1972 static void noop_out_exec(struct iscsi_cmnd *req)
1973 {
1974         struct iscsi_cmnd *rsp;
1975         struct iscsi_nop_in_hdr *rsp_hdr;
1976
1977         TRACE_DBG("%p", req);
1978
1979         if (cmnd_itt(req) != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1980                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
1981
1982                 rsp_hdr = (struct iscsi_nop_in_hdr *)&rsp->pdu.bhs;
1983                 rsp_hdr->opcode = ISCSI_OP_NOOP_IN;
1984                 rsp_hdr->flags = ISCSI_FLG_FINAL;
1985                 rsp_hdr->itt = req->pdu.bhs.itt;
1986                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
1987
1988                 if (req->pdu.datasize)
1989                         sBUG_ON(req->sg == NULL);
1990                 else
1991                         sBUG_ON(req->sg != NULL);
1992
1993                 if (req->sg) {
1994                         rsp->sg = req->sg;
1995                         rsp->sg_cnt = req->sg_cnt;
1996                         rsp->bufflen = req->bufflen;
1997                 }
1998
1999                 /* We already checked it in check_segment_length() */
2000                 sBUG_ON(get_pgcnt(req->pdu.datasize, 0) > ISCSI_CONN_IOV_MAX);
2001
2002                 rsp->pdu.datasize = req->pdu.datasize;
2003                 iscsi_cmnd_init_write(rsp,
2004                         ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2005                 req_cmnd_release(req);
2006         } else
2007                 cmnd_put(req);
2008         return;
2009 }
2010
2011 static void logout_exec(struct iscsi_cmnd *req)
2012 {
2013         struct iscsi_logout_req_hdr *req_hdr;
2014         struct iscsi_cmnd *rsp;
2015         struct iscsi_logout_rsp_hdr *rsp_hdr;
2016
2017         PRINT_INFO("Logout received from initiator %s",
2018                 req->conn->session->initiator_name);
2019         TRACE_DBG("%p", req);
2020
2021         req_hdr = (struct iscsi_logout_req_hdr *)&req->pdu.bhs;
2022         rsp = iscsi_cmnd_create_rsp_cmnd(req);
2023         rsp_hdr = (struct iscsi_logout_rsp_hdr *)&rsp->pdu.bhs;
2024         rsp_hdr->opcode = ISCSI_OP_LOGOUT_RSP;
2025         rsp_hdr->flags = ISCSI_FLG_FINAL;
2026         rsp_hdr->itt = req_hdr->itt;
2027         rsp->should_close_conn = 1;
2028         iscsi_cmnd_init_write(rsp,
2029                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2030         req_cmnd_release(req);
2031         return;
2032 }
2033
2034 static void iscsi_cmnd_exec(struct iscsi_cmnd *cmnd)
2035 {
2036         TRACE_ENTRY();
2037
2038         TRACE_DBG("%p,%x,%u", cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn);
2039
2040         iscsi_extracheck_is_rd_thread(cmnd->conn);
2041
2042         if (unlikely(cmnd->tm_aborted)) {
2043                 TRACE_MGMT_DBG("cmnd %p (scst_cmd %p) aborted", cmnd,
2044                         cmnd->scst_cmd);
2045                 req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
2046                 goto out;
2047         }
2048
2049         if (unlikely(cmnd->rejected))
2050                 goto out_rejected;
2051
2052         switch (cmnd_opcode(cmnd)) {
2053         case ISCSI_OP_SCSI_CMD:
2054                 if (cmnd->r2t_length != 0) {
2055                         if (!cmnd->is_unsolicited_data) {
2056                                 send_r2t(cmnd);
2057                                 break;
2058                         }
2059                 } else
2060                         iscsi_restart_cmnd(cmnd);
2061                 break;
2062         case ISCSI_OP_NOOP_OUT:
2063                 noop_out_exec(cmnd);
2064                 break;
2065         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2066                 execute_task_management(cmnd);
2067                 break;
2068         case ISCSI_OP_LOGOUT_CMD:
2069                 logout_exec(cmnd);
2070                 break;
2071         default:
2072                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2073                 req_cmnd_release(cmnd);
2074                 break;
2075         }
2076 out:
2077         TRACE_EXIT();
2078         return;
2079
2080 out_rejected:
2081         TRACE_MGMT_DBG("Rejected cmd %p (reason %d)", cmnd,
2082                 cmnd->reject_reason);
2083         switch (cmnd->reject_reason) {
2084         default:
2085                 PRINT_ERROR("Unexpected reject reason %d",
2086                             cmnd->reject_reason);
2087                 /* go through */
2088         case ISCSI_REJECT_SCSI_CMD:
2089                 req_cmnd_release(cmnd);
2090                 break;
2091         }
2092         goto out;
2093 }
2094
2095 static void __cmnd_send_pdu(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd,
2096         u32 offset, u32 size)
2097 {
2098         TRACE_DBG("%p %u,%u,%u", cmnd, offset, size, cmnd->bufflen);
2099
2100         iscsi_extracheck_is_wr_thread(conn);
2101
2102         sBUG_ON(offset > cmnd->bufflen);
2103         sBUG_ON(offset + size > cmnd->bufflen);
2104
2105         conn->write_offset = offset;
2106         conn->write_size += size;
2107         return;
2108 }
2109
2110 static void cmnd_send_pdu(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
2111 {
2112         u32 size;
2113
2114         if (!cmnd->pdu.datasize)
2115                 return;
2116
2117         size = (cmnd->pdu.datasize + 3) & -4;
2118         sBUG_ON(cmnd->sg == NULL);
2119         sBUG_ON(cmnd->bufflen != size);
2120         __cmnd_send_pdu(conn, cmnd, 0, size);
2121         return;
2122 }
2123
2124 /*
2125  * Note: the code belows passes a kernel space pointer (&opt) to setsockopt()
2126  * while the declaration of setsockopt specifies that it expects a user space
2127  * pointer. This seems to work fine, and this approach is also used in some
2128  * other parts of the Linux kernel (see e.g. fs/ocfs2/cluster/tcp.c).
2129  */
2130 static void set_cork(struct socket *sock, int on)
2131 {
2132         int opt = on;
2133         mm_segment_t oldfs;
2134
2135         oldfs = get_fs();
2136         set_fs(get_ds());
2137         sock->ops->setsockopt(sock, SOL_TCP, TCP_CORK,
2138                               (void __force __user *)&opt, sizeof(opt));
2139         set_fs(oldfs);
2140         return;
2141 }
2142
2143 void cmnd_tx_start(struct iscsi_cmnd *cmnd)
2144 {
2145         struct iscsi_conn *conn = cmnd->conn;
2146
2147         TRACE_DBG("%p:%p:%x", conn, cmnd, cmnd_opcode(cmnd));
2148         iscsi_cmnd_set_length(&cmnd->pdu);
2149
2150         iscsi_extracheck_is_wr_thread(conn);
2151
2152         set_cork(conn->sock, 1);
2153
2154         conn->write_iop = conn->write_iov;
2155         conn->write_iop->iov_base = (void __force __user *)(&cmnd->pdu.bhs);
2156         conn->write_iop->iov_len = sizeof(cmnd->pdu.bhs);
2157         conn->write_iop_used = 1;
2158         conn->write_size = sizeof(cmnd->pdu.bhs);
2159
2160         switch (cmnd_opcode(cmnd)) {
2161         case ISCSI_OP_NOOP_IN:
2162                 cmnd_set_sn(cmnd, 1);
2163                 cmnd_send_pdu(conn, cmnd);
2164                 break;
2165         case ISCSI_OP_SCSI_RSP:
2166                 cmnd_set_sn(cmnd, 1);
2167                 cmnd_send_pdu(conn, cmnd);
2168                 break;
2169         case ISCSI_OP_SCSI_TASK_MGT_RSP:
2170                 cmnd_set_sn(cmnd, 1);
2171                 break;
2172         case ISCSI_OP_TEXT_RSP:
2173                 cmnd_set_sn(cmnd, 1);
2174                 break;
2175         case ISCSI_OP_SCSI_DATA_IN:
2176         {
2177                 struct iscsi_data_in_hdr *rsp =
2178                         (struct iscsi_data_in_hdr *)&cmnd->pdu.bhs;
2179                 u32 offset = cpu_to_be32(rsp->buffer_offset);
2180
2181                 cmnd_set_sn(cmnd, (rsp->flags & ISCSI_FLG_FINAL) ? 1 : 0);
2182                 __cmnd_send_pdu(conn, cmnd, offset, cmnd->pdu.datasize);
2183                 break;
2184         }
2185         case ISCSI_OP_LOGOUT_RSP:
2186                 cmnd_set_sn(cmnd, 1);
2187                 break;
2188         case ISCSI_OP_R2T:
2189                 cmnd->pdu.bhs.sn = cmnd_set_sn(cmnd, 0);
2190                 break;
2191         case ISCSI_OP_ASYNC_MSG:
2192                 cmnd_set_sn(cmnd, 1);
2193                 break;
2194         case ISCSI_OP_REJECT:
2195                 cmnd_set_sn(cmnd, 1);
2196                 cmnd_send_pdu(conn, cmnd);
2197                 break;
2198         default:
2199                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2200                 break;
2201         }
2202
2203         /* move this? */
2204         conn->write_size = (conn->write_size + 3) & -4;
2205         iscsi_dump_pdu(&cmnd->pdu);
2206         return;
2207 }
2208
2209 void cmnd_tx_end(struct iscsi_cmnd *cmnd)
2210 {
2211         struct iscsi_conn *conn = cmnd->conn;
2212
2213         TRACE_DBG("%p:%x (should_close_conn %d, should_close_all_conn %d)",
2214                 cmnd, cmnd_opcode(cmnd), cmnd->should_close_conn,
2215                 cmnd->should_close_all_conn);
2216
2217         switch (cmnd_opcode(cmnd)) {
2218         case ISCSI_OP_NOOP_IN:
2219         case ISCSI_OP_SCSI_RSP:
2220         case ISCSI_OP_SCSI_TASK_MGT_RSP:
2221         case ISCSI_OP_TEXT_RSP:
2222         case ISCSI_OP_R2T:
2223         case ISCSI_OP_ASYNC_MSG:
2224         case ISCSI_OP_REJECT:
2225         case ISCSI_OP_SCSI_DATA_IN:
2226         case ISCSI_OP_LOGOUT_RSP:
2227                 break;
2228         default:
2229                 PRINT_CRIT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2230                 sBUG();
2231                 break;
2232         }
2233
2234         if (unlikely(cmnd->should_close_conn)) {
2235                 if (cmnd->should_close_all_conn) {
2236                         PRINT_INFO("Closing all connections for target %x at "
2237                                 "initiator's %s request",
2238                                 cmnd->conn->session->target->tid,
2239                                 conn->session->initiator_name);
2240                         target_del_all_sess(cmnd->conn->session->target, false);
2241                 } else {
2242                         PRINT_INFO("Closing connection at initiator's %s "
2243                                 "request", conn->session->initiator_name);
2244                         mark_conn_closed(conn);
2245                 }
2246         }
2247
2248         set_cork(cmnd->conn->sock, 0);
2249         return;
2250 }
2251
2252 /*
2253  * Push the command for execution. This functions reorders the commands.
2254  * Called from the read thread.
2255  */
2256 static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd)
2257 {
2258         struct iscsi_session *session = cmnd->conn->session;
2259         struct list_head *entry;
2260         u32 cmd_sn;
2261
2262         TRACE_DBG("%p:%x %u,%u",
2263                 cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn,
2264                 session->exp_cmd_sn);
2265
2266         iscsi_extracheck_is_rd_thread(cmnd->conn);
2267
2268         sBUG_ON(cmnd->parent_req != NULL);
2269
2270         if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
2271                 TRACE_DBG("Immediate cmd %p (cmd_sn %u)", cmnd,
2272                         cmnd->pdu.bhs.sn);
2273                 iscsi_cmnd_exec(cmnd);
2274                 goto out;
2275         }
2276
2277         spin_lock(&session->sn_lock);
2278
2279         cmd_sn = cmnd->pdu.bhs.sn;
2280         if (cmd_sn == session->exp_cmd_sn) {
2281                 while (1) {
2282                         session->exp_cmd_sn = ++cmd_sn;
2283
2284                         if (unlikely(session->tm_active > 0)) {
2285                                 if (before(cmd_sn, session->tm_sn)) {
2286                                         struct iscsi_conn *conn = cmnd->conn;
2287
2288                                         spin_unlock(&session->sn_lock);
2289
2290                                         spin_lock_bh(&conn->cmd_list_lock);
2291                                         __cmnd_abort(cmnd);
2292                                         spin_unlock_bh(&conn->cmd_list_lock);
2293
2294                                         spin_lock(&session->sn_lock);
2295                                 }
2296                                 iscsi_check_send_delayed_tm_resp(session);
2297                         }
2298
2299                         spin_unlock(&session->sn_lock);
2300
2301                         iscsi_cmnd_exec(cmnd);
2302
2303                         if (list_empty(&session->pending_list))
2304                                 break;
2305                         cmnd = list_entry(session->pending_list.next,
2306                                           struct iscsi_cmnd,
2307                                           pending_list_entry);
2308                         if (cmnd->pdu.bhs.sn != cmd_sn)
2309                                 break;
2310
2311                         list_del(&cmnd->pending_list_entry);
2312                         cmnd->pending = 0;
2313
2314                         TRACE_DBG("Processing pending cmd %p (cmd_sn %u)",
2315                                 cmnd, cmd_sn);
2316
2317                         spin_lock(&session->sn_lock);
2318                 }
2319         } else {
2320                 int drop = 0;
2321
2322                 TRACE_DBG("Pending cmd %p (cmd_sn %u, exp_cmd_sn %u)",
2323                         cmnd, cmd_sn, session->exp_cmd_sn);
2324
2325                 /*
2326                  * iSCSI RFC 3720: "The target MUST silently ignore any
2327                  * non-immediate command outside of [from ExpCmdSN to MaxCmdSN
2328                  * inclusive] range". But we won't honor the MaxCmdSN
2329                  * requirement, because, since we adjust MaxCmdSN from the
2330                  * separate write thread, rarery it is possible that initiator
2331                  * can legally send command with CmdSN>MaxSN. But it won't
2332                  * hurt anything, in the worst case it will lead to
2333                  * additional QUEUE FULL status.
2334                  */
2335
2336                 if (unlikely(before(cmd_sn, session->exp_cmd_sn))) {
2337                         PRINT_ERROR("Unexpected cmd_sn (%u,%u)", cmd_sn,
2338                                 session->exp_cmd_sn);
2339                         drop = 1;
2340                 }
2341
2342 #if 0
2343                 if (unlikely(after(cmd_sn, session->exp_cmd_sn +
2344                                         iscsi_get_allowed_cmds(session)))) {
2345                         TRACE_MGMT_DBG("Too large cmd_sn %u (exp_cmd_sn %u, "
2346                                 "max_sn %u)", cmd_sn, session->exp_cmd_sn,
2347                                 iscsi_get_allowed_cmds(session));
2348                 }
2349 #endif
2350
2351                 spin_unlock(&session->sn_lock);
2352
2353                 if (unlikely(drop)) {
2354                         req_cmnd_release_force(cmnd,
2355                                                ISCSI_FORCE_RELEASE_WRITE);
2356                         goto out;
2357                 }
2358
2359                 if (unlikely(cmnd->tm_aborted)) {
2360                         struct iscsi_cmnd *tm_clone;
2361
2362                         TRACE_MGMT_DBG("Pending aborted cmnd %p, creating TM "
2363                                 "clone (scst cmd %p, state %d)", cmnd,
2364                                 cmnd->scst_cmd, cmnd->scst_state);
2365
2366                         tm_clone = cmnd_alloc(cmnd->conn, NULL);
2367                         if (tm_clone != NULL) {
2368                                 tm_clone->tm_aborted = 1;
2369                                 tm_clone->pdu = cmnd->pdu;
2370
2371                                 TRACE_MGMT_DBG("TM clone %p created",
2372                                                tm_clone);
2373
2374                                 iscsi_cmnd_exec(cmnd);
2375                                 cmnd = tm_clone;
2376                         } else
2377                                 PRINT_ERROR("%s", "Unable to create TM clone");
2378                 }
2379
2380                 list_for_each(entry, &session->pending_list) {
2381                         struct iscsi_cmnd *tmp =
2382                                 list_entry(entry, struct iscsi_cmnd,
2383                                            pending_list_entry);
2384                         if (before(cmd_sn, tmp->pdu.bhs.sn))
2385                                 break;
2386                 }
2387
2388                 list_add_tail(&cmnd->pending_list_entry, entry);
2389                 cmnd->pending = 1;
2390         }
2391 out:
2392         return;
2393 }
2394
2395 static int check_segment_length(struct iscsi_cmnd *cmnd)
2396 {
2397         struct iscsi_conn *conn = cmnd->conn;
2398         struct iscsi_session *session = conn->session;
2399
2400         if (unlikely(cmnd->pdu.datasize > session->sess_param.max_recv_data_length)) {
2401                 PRINT_ERROR("Initiator %s violated negotiated parameters: "
2402                         "data too long (ITT %x, datasize %u, "
2403                         "max_recv_data_length %u", session->initiator_name,
2404                         cmnd_itt(cmnd), cmnd->pdu.datasize,
2405                         session->sess_param.max_recv_data_length);
2406                 mark_conn_closed(conn);
2407                 return -EINVAL;
2408         }
2409         return 0;
2410 }
2411
2412 int cmnd_rx_start(struct iscsi_cmnd *cmnd)
2413 {
2414         struct iscsi_conn *conn = cmnd->conn;
2415         int res, rc;
2416
2417         iscsi_dump_pdu(&cmnd->pdu);
2418
2419         res = check_segment_length(cmnd);
2420         if (res != 0)
2421                 goto out;
2422
2423         switch (cmnd_opcode(cmnd)) {
2424         case ISCSI_OP_NOOP_OUT:
2425                 rc = noop_out_start(cmnd);
2426                 break;
2427         case ISCSI_OP_SCSI_CMD:
2428                 rc = cmnd_insert_hash(cmnd);
2429                 if (likely(rc == 0)) {
2430                         res = scsi_cmnd_start(cmnd);
2431                         if (unlikely(res != 0))
2432                                 goto out;
2433                 }
2434                 break;
2435         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2436                 rc = cmnd_insert_hash(cmnd);
2437                 break;
2438         case ISCSI_OP_SCSI_DATA_OUT:
2439                 res = data_out_start(conn, cmnd);
2440                 rc = 0; /* to avoid compiler warning */
2441                 if (unlikely(res != 0))
2442                         goto out;
2443                 break;
2444         case ISCSI_OP_LOGOUT_CMD:
2445                 rc = cmnd_insert_hash(cmnd);
2446                 break;
2447         case ISCSI_OP_TEXT_CMD:
2448         case ISCSI_OP_SNACK_CMD:
2449                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2450                 break;
2451         default:
2452                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2453                 break;
2454         }
2455
2456         if (unlikely(rc < 0)) {
2457                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
2458                 PRINT_ERROR("Error %d (iSCSI opcode %x, ITT %x, op %x)", rc,
2459                         cmnd_opcode(cmnd), cmnd_itt(cmnd),
2460                         (cmnd_opcode(cmnd) == ISCSI_OP_SCSI_CMD ?
2461                                 hdr->scb[0] : -1));
2462                 iscsi_cmnd_reject(cmnd, -rc);
2463         }
2464
2465 out:
2466         TRACE_EXIT_RES(res);
2467         return res;
2468 }
2469
2470 void cmnd_rx_end(struct iscsi_cmnd *cmnd)
2471 {
2472         TRACE_ENTRY();
2473
2474         TRACE_DBG("%p:%x", cmnd, cmnd_opcode(cmnd));
2475
2476         if (unlikely(cmnd->rejected))
2477                 goto out_rejected;
2478
2479 cont:
2480         switch (cmnd_opcode(cmnd)) {
2481         case ISCSI_OP_SCSI_CMD:
2482         case ISCSI_OP_NOOP_OUT:
2483         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2484         case ISCSI_OP_LOGOUT_CMD:
2485                 iscsi_session_push_cmnd(cmnd);
2486                 break;
2487         case ISCSI_OP_SCSI_DATA_OUT:
2488                 data_out_end(cmnd);
2489                 break;
2490         default:
2491                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2492                 req_cmnd_release(cmnd);
2493                 break;
2494         }
2495
2496 out:
2497         TRACE_EXIT();
2498         return;
2499
2500 out_rejected:
2501         switch (cmnd->reject_reason) {
2502         default:
2503                 PRINT_ERROR("Unexpected reject reason %d",
2504                             cmnd->reject_reason);
2505                 /* go through */
2506         case ISCSI_REJECT_CMD:
2507         case ISCSI_REJECT_DATA:
2508                 req_cmnd_release(cmnd);
2509                 break;
2510         case ISCSI_REJECT_SCSI_CMD:
2511                 goto cont;
2512         }
2513         goto out;
2514 }
2515
2516 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
2517 static int iscsi_alloc_data_buf(struct scst_cmd *cmd)
2518 {
2519         /*
2520          * sock->ops->sendpage() is async zero copy operation,
2521          * so we must be sure not to free and reuse
2522          * the command's buffer before the sending was completed
2523          * by the network layers. It is possible only if we
2524          * don't use SGV cache.
2525          */
2526         EXTRACHECKS_BUG_ON(scst_cmd_get_data_direction(cmd) != SCST_DATA_READ);
2527         scst_cmd_set_no_sgv(cmd);
2528         return 1;
2529 }
2530 #endif
2531
2532 static inline void iscsi_set_state_wake_up(struct iscsi_cmnd *req,
2533         int new_state)
2534 {
2535         /*
2536          * We use wait_event() to wait for the state change, but it checks its
2537          * condition without any protection, so without cmnd_get() it is
2538          * possible that req will die "immediately" after the state assignment
2539          * and wake_up() will operate on dead data. We use the ordered version
2540          * of cmnd_get(), because "get" must be done before the state
2541          * assignment.
2542          */
2543         cmnd_get_ordered(req);
2544         req->scst_state = new_state;
2545         wake_up(&req->scst_waitQ);
2546         cmnd_put(req);
2547         return;
2548 }
2549
2550 static void iscsi_preprocessing_done(struct scst_cmd *scst_cmd)
2551 {
2552         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
2553                                 scst_cmd_get_tgt_priv(scst_cmd);
2554
2555         TRACE_DBG("req %p", req);
2556
2557         EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_RX_CMD);
2558
2559         iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_AFTER_PREPROC);
2560         return;
2561 }
2562
2563 /*
2564  * No locks.
2565  *
2566  * IMPORTANT! Connection conn must be protected by additional conn_get()
2567  * upon entrance in this function, because otherwise it could be destroyed
2568  * inside as a result of iscsi_send(), which releases sent commands.
2569  */
2570 static void iscsi_try_local_processing(struct iscsi_conn *conn)
2571 {
2572         int local;
2573
2574         TRACE_ENTRY();
2575
2576         spin_lock_bh(&iscsi_wr_lock);
2577         switch (conn->wr_state) {
2578         case ISCSI_CONN_WR_STATE_IN_LIST:
2579                 list_del(&conn->wr_list_entry);
2580                 /* go through */
2581         case ISCSI_CONN_WR_STATE_IDLE:
2582 #ifdef CONFIG_SCST_EXTRACHECKS
2583                 conn->wr_task = current;
2584 #endif
2585                 conn->wr_state = ISCSI_CONN_WR_STATE_PROCESSING;
2586                 conn->wr_space_ready = 0;
2587                 local = 1;
2588                 break;
2589         default:
2590                 local = 0;
2591                 break;
2592         }
2593         spin_unlock_bh(&iscsi_wr_lock);
2594
2595         if (local) {
2596                 int rc = 1;
2597
2598                 if (test_write_ready(conn))
2599                         rc = iscsi_send(conn);
2600
2601                 spin_lock_bh(&iscsi_wr_lock);
2602 #ifdef CONFIG_SCST_EXTRACHECKS
2603                 conn->wr_task = NULL;
2604 #endif
2605                 if ((rc <= 0) || test_write_ready(conn)) {
2606                         list_add_tail(&conn->wr_list_entry, &iscsi_wr_list);
2607                         conn->wr_state = ISCSI_CONN_WR_STATE_IN_LIST;
2608                         wake_up(&iscsi_wr_waitQ);
2609                 } else
2610                         conn->wr_state = ISCSI_CONN_WR_STATE_IDLE;
2611                 spin_unlock_bh(&iscsi_wr_lock);
2612         }
2613
2614         TRACE_EXIT();
2615         return;
2616 }
2617
2618 static int iscsi_xmit_response(struct scst_cmd *scst_cmd)
2619 {
2620         int is_send_status = scst_cmd_get_is_send_status(scst_cmd);
2621         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
2622                                         scst_cmd_get_tgt_priv(scst_cmd);
2623         struct iscsi_conn *conn = req->conn;
2624         int status = scst_cmd_get_status(scst_cmd);
2625         u8 *sense = scst_cmd_get_sense_buffer(scst_cmd);
2626         int sense_len = scst_cmd_get_sense_buffer_len(scst_cmd);
2627         int old_state = req->scst_state;
2628
2629         if (scst_cmd_atomic(scst_cmd))
2630                 return SCST_TGT_RES_NEED_THREAD_CTX;
2631
2632         scst_cmd_set_tgt_priv(scst_cmd, NULL);
2633
2634         req->tm_aborted |= scst_cmd_aborted(scst_cmd) ? 1 : 0;
2635         if (unlikely(req->tm_aborted)) {
2636                 TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
2637                         req->scst_cmd);
2638
2639                 scst_set_delivery_status(req->scst_cmd,
2640                         SCST_CMD_DELIVERY_ABORTED);
2641
2642                 if (old_state == ISCSI_CMD_STATE_RESTARTED) {
2643                         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
2644                         req_cmnd_release_force(req, ISCSI_FORCE_RELEASE_WRITE);
2645                 } else
2646                         iscsi_set_state_wake_up(req,
2647                                                 ISCSI_CMD_STATE_PROCESSED);
2648
2649                 goto out;
2650         }
2651
2652         if (unlikely(old_state != ISCSI_CMD_STATE_RESTARTED)) {
2653                 TRACE_DBG("req %p on %d state", req, old_state);
2654
2655                 create_status_rsp(req, status, sense, sense_len);
2656
2657                 switch (old_state) {
2658                 case ISCSI_CMD_STATE_RX_CMD:
2659                 case ISCSI_CMD_STATE_AFTER_PREPROC:
2660                         break;
2661                 default:
2662                         sBUG();
2663                 }
2664
2665                 iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_PROCESSED);
2666                 goto out;
2667         }
2668
2669         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
2670
2671         req->bufflen = scst_cmd_get_resp_data_len(scst_cmd);
2672         req->sg = scst_cmd_get_sg(scst_cmd);
2673         req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
2674
2675         TRACE_DBG("req %p, is_send_status=%x, req->bufflen=%d, req->sg=%p, "
2676                 "req->sg_cnt %d", req, is_send_status, req->bufflen, req->sg,
2677                 req->sg_cnt);
2678
2679         if (unlikely((req->bufflen != 0) && !is_send_status)) {
2680                 PRINT_CRIT_ERROR("%s", "Sending DATA without STATUS is "
2681                         "unsupported");
2682                 scst_set_cmd_error(scst_cmd,
2683                         SCST_LOAD_SENSE(scst_sense_hardw_error));
2684                 sBUG();
2685         }
2686
2687         if (req->bufflen != 0) {
2688                 /*
2689                  * Check above makes sure that is_send_status is set,
2690                  * so status is valid here, but in future that could change.
2691                  * ToDo
2692                  */
2693                 if (status != SAM_STAT_CHECK_CONDITION) {
2694                         send_data_rsp(req, status, is_send_status);
2695                 } else {
2696                         struct iscsi_cmnd *rsp;
2697                         struct iscsi_scsi_rsp_hdr *rsp_hdr;
2698                         int resid;
2699                         send_data_rsp(req, 0, 0);
2700                         if (is_send_status) {
2701                                 rsp = create_status_rsp(req, status, sense,
2702                                         sense_len);
2703                                 rsp_hdr =
2704                                     (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
2705                                 resid = cmnd_read_size(req) - req->bufflen;
2706                                 if (resid > 0) {
2707                                         rsp_hdr->flags |=
2708                                                 ISCSI_FLG_RESIDUAL_UNDERFLOW;
2709                                         rsp_hdr->residual_count =
2710                                                 cpu_to_be32(resid);
2711                                 } else if (resid < 0) {
2712                                         rsp_hdr->flags |=
2713                                                 ISCSI_FLG_RESIDUAL_OVERFLOW;
2714                                         rsp_hdr->residual_count =
2715                                                 cpu_to_be32(-resid);
2716                                 }
2717                                 iscsi_cmnd_init_write(rsp,
2718                                         ISCSI_INIT_WRITE_REMOVE_HASH);
2719                         }
2720                 }
2721         } else if (is_send_status) {
2722                 struct iscsi_cmnd *rsp;
2723                 struct iscsi_scsi_rsp_hdr *rsp_hdr;
2724                 u32 resid;
2725                 rsp = create_status_rsp(req, status, sense, sense_len);
2726                 rsp_hdr = (struct iscsi_scsi_rsp_hdr *) &rsp->pdu.bhs;
2727                 resid = cmnd_read_size(req);
2728                 if (resid != 0) {
2729                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
2730                         rsp_hdr->residual_count = cpu_to_be32(resid);
2731                 }
2732                 iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH);
2733         }
2734 #ifdef CONFIG_SCST_EXTRACHECKS
2735         else
2736                 sBUG();
2737 #endif
2738
2739         /*
2740          * "_ordered" here to protect from reorder, which can lead to
2741          * preliminary connection destroy in req_cmnd_release(). Just in
2742          * case, actually, because reordering shouldn't go so far, but who
2743          * knows..
2744          */
2745         conn_get_ordered(conn);
2746         req_cmnd_release(req);
2747         iscsi_try_local_processing(conn);
2748         conn_put(conn);
2749
2750 out:
2751         return SCST_TGT_RES_SUCCESS;
2752 }
2753
2754 /* Called under sn_lock */
2755 static bool iscsi_is_delay_tm_resp(struct iscsi_cmnd *rsp)
2756 {
2757         bool res = 0;
2758         struct iscsi_task_mgt_hdr *req_hdr =
2759                 (struct iscsi_task_mgt_hdr *)&rsp->parent_req->pdu.bhs;
2760         int function = req_hdr->function & ISCSI_FUNCTION_MASK;
2761         struct iscsi_session *sess = rsp->conn->session;
2762
2763         TRACE_ENTRY();
2764
2765         /* This should be checked for immediate TM commands as well */
2766
2767         switch (function) {
2768         default:
2769                 if (before(sess->exp_cmd_sn, req_hdr->cmd_sn))
2770                         res = 1;
2771                 break;
2772         }
2773
2774         TRACE_EXIT_RES(res);
2775         return res;
2776 }
2777
2778 /* Called under sn_lock, but might drop it inside, then reaquire */
2779 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess)
2780 {
2781         struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
2782
2783         TRACE_ENTRY();
2784
2785         if (tm_rsp == NULL)
2786                 goto out;
2787
2788         if (iscsi_is_delay_tm_resp(tm_rsp))
2789                 goto out;
2790
2791         TRACE(TRACE_MGMT_MINOR, "Sending delayed rsp %p", tm_rsp);
2792
2793         sess->tm_rsp = NULL;
2794         sess->tm_active--;
2795
2796         spin_unlock(&sess->sn_lock);
2797
2798         sBUG_ON(sess->tm_active < 0);
2799
2800         iscsi_cmnd_init_write(tm_rsp,
2801                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2802
2803         spin_lock(&sess->sn_lock);
2804
2805 out:
2806         TRACE_EXIT();
2807         return;
2808 }
2809
2810 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status)
2811 {
2812         struct iscsi_cmnd *rsp;
2813         struct iscsi_task_mgt_hdr *req_hdr =
2814                                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2815         struct iscsi_task_rsp_hdr *rsp_hdr;
2816         struct iscsi_session *sess = req->conn->session;
2817         int fn = req_hdr->function & ISCSI_FUNCTION_MASK;
2818
2819         TRACE_ENTRY();
2820
2821         TRACE_MGMT_DBG("TM req %p finished", req);
2822         TRACE((req_hdr->function == ISCSI_FUNCTION_ABORT_TASK) ?
2823                          TRACE_MGMT_MINOR : TRACE_MGMT,
2824                 "TM fn %d finished, status %d", fn, status);
2825
2826         rsp = iscsi_cmnd_create_rsp_cmnd(req);
2827         rsp_hdr = (struct iscsi_task_rsp_hdr *)&rsp->pdu.bhs;
2828
2829         rsp_hdr->opcode = ISCSI_OP_SCSI_TASK_MGT_RSP;
2830         rsp_hdr->flags = ISCSI_FLG_FINAL;
2831         rsp_hdr->itt = req_hdr->itt;
2832         rsp_hdr->response = status;
2833
2834         if (fn == ISCSI_FUNCTION_TARGET_COLD_RESET) {
2835                 rsp->should_close_conn = 1;
2836                 rsp->should_close_all_conn = 1;
2837         }
2838
2839         sBUG_ON(sess->tm_rsp != NULL);
2840
2841         spin_lock(&sess->sn_lock);
2842         if (iscsi_is_delay_tm_resp(rsp)) {
2843                 TRACE(TRACE_MGMT_MINOR, "Delaying TM fn %x response %p "
2844                         "(req %p), because not all affected commands received "
2845                         "(TM cmd sn %u, exp sn %u)",
2846                         req_hdr->function & ISCSI_FUNCTION_MASK, rsp, req,
2847                         req_hdr->cmd_sn, sess->exp_cmd_sn);
2848                 sess->tm_rsp = rsp;
2849                 spin_unlock(&sess->sn_lock);
2850                 goto out_release;
2851         }
2852         sess->tm_active--;
2853         spin_unlock(&sess->sn_lock);
2854
2855         sBUG_ON(sess->tm_active < 0);
2856
2857         iscsi_cmnd_init_write(rsp,
2858                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2859
2860 out_release:
2861         req_cmnd_release(req);
2862
2863         TRACE_EXIT();
2864         return;
2865 }
2866
2867 static inline int iscsi_get_mgmt_response(int status)
2868 {
2869         switch (status) {
2870         case SCST_MGMT_STATUS_SUCCESS:
2871                 return ISCSI_RESPONSE_FUNCTION_COMPLETE;
2872
2873         case SCST_MGMT_STATUS_TASK_NOT_EXIST:
2874                 return ISCSI_RESPONSE_UNKNOWN_TASK;
2875
2876         case SCST_MGMT_STATUS_LUN_NOT_EXIST:
2877                 return ISCSI_RESPONSE_UNKNOWN_LUN;
2878
2879         case SCST_MGMT_STATUS_FN_NOT_SUPPORTED:
2880                 return ISCSI_RESPONSE_FUNCTION_UNSUPPORTED;
2881
2882         case SCST_MGMT_STATUS_REJECTED:
2883         case SCST_MGMT_STATUS_FAILED:
2884         default:
2885                 return ISCSI_RESPONSE_FUNCTION_REJECTED;
2886         }
2887 }
2888
2889 static void iscsi_task_mgmt_fn_done(struct scst_mgmt_cmd *scst_mcmd)
2890 {
2891         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
2892                                 scst_mgmt_cmd_get_tgt_priv(scst_mcmd);
2893         int status =
2894                 iscsi_get_mgmt_response(scst_mgmt_cmd_get_status(scst_mcmd));
2895
2896         TRACE_MGMT_DBG("req %p, scst_mcmd %p, fn %d, scst status %d",
2897                 req, scst_mcmd, scst_mgmt_cmd_get_fn(scst_mcmd),
2898                 scst_mgmt_cmd_get_status(scst_mcmd));
2899
2900         iscsi_send_task_mgmt_resp(req, status);
2901
2902         scst_mgmt_cmd_set_tgt_priv(scst_mcmd, NULL);
2903
2904         return;
2905 }
2906
2907 static int iscsi_target_detect(struct scst_tgt_template *templ)
2908 {
2909         /* Nothing to do */
2910         return 0;
2911 }
2912
2913 static int iscsi_target_release(struct scst_tgt *scst_tgt)
2914 {
2915         /* Nothing to do */
2916         return 0;
2917 }
2918
2919 struct scst_tgt_template iscsi_template = {
2920         .name = "iscsi",
2921         .sg_tablesize = 0xFFFF /* no limit */,
2922         .threads_num = 0,
2923         .no_clustering = 1,
2924         .xmit_response_atomic = 0,
2925         .detect = iscsi_target_detect,
2926         .release = iscsi_target_release,
2927         .xmit_response = iscsi_xmit_response,
2928 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
2929         .alloc_data_buf = iscsi_alloc_data_buf,
2930 #endif
2931         .preprocessing_done = iscsi_preprocessing_done,
2932         .pre_exec = iscsi_pre_exec,
2933         .task_mgmt_fn_done = iscsi_task_mgmt_fn_done,
2934 };
2935
2936 static __init int iscsi_run_threads(int count, char *name, int (*fn)(void *))
2937 {
2938         int res = 0;
2939         int i;
2940         struct iscsi_thread_t *thr;
2941
2942         for (i = 0; i < count; i++) {
2943                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
2944                 if (!thr) {
2945                         res = -ENOMEM;
2946                         PRINT_ERROR("Failed to allocate thr %d", res);
2947                         goto out;
2948                 }
2949                 thr->thr = kthread_run(fn, NULL, "%s%d", name, i);
2950                 if (IS_ERR(thr->thr)) {
2951                         res = PTR_ERR(thr->thr);
2952                         PRINT_ERROR("kthread_create() failed: %d", res);
2953                         kfree(thr);
2954                         goto out;
2955                 }
2956                 list_add(&thr->threads_list_entry, &iscsi_threads_list);
2957         }
2958
2959 out:
2960         return res;
2961 }
2962
2963 static void iscsi_stop_threads(void)
2964 {
2965         struct iscsi_thread_t *t, *tmp;
2966
2967         list_for_each_entry_safe(t, tmp, &iscsi_threads_list,
2968                                 threads_list_entry) {
2969                 int rc = kthread_stop(t->thr);
2970                 if (rc < 0)
2971                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
2972                 list_del(&t->threads_list_entry);
2973                 kfree(t);
2974         }
2975         return;
2976 }
2977
2978 static int __init iscsi_init(void)
2979 {
2980         int err = 0;
2981         int num;
2982
2983         PRINT_INFO("iSCSI SCST Target - version %s", ISCSI_VERSION_STRING);
2984
2985         dummy_page = alloc_pages(GFP_KERNEL, 0);
2986         if (dummy_page == NULL) {
2987                 PRINT_ERROR("%s", "Dummy page allocation failed");
2988                 goto out;
2989         }
2990
2991         sg_init_table(&dummy_sg, 1);
2992         sg_set_page(&dummy_sg, dummy_page, PAGE_SIZE, 0);
2993
2994 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
2995         err = net_set_get_put_page_callbacks(iscsi_get_page_callback,
2996                         iscsi_put_page_callback);
2997         if (err != 0) {
2998                 PRINT_INFO("Unable to set page callbackes: %d", err);
2999                 goto out_free_dummy;
3000         }
3001 #else
3002         PRINT_WARNING("%s",
3003                 "CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION "
3004                 "not enabled in your kernel. ISCSI-SCST will be working with "
3005                 "not the best performance. Refer README file for details.");
3006 #endif
3007
3008         ctr_major = register_chrdev(0, ctr_name, &ctr_fops);
3009         if (ctr_major < 0) {
3010                 PRINT_ERROR("failed to register the control device %d",
3011                             ctr_major);
3012                 err = ctr_major;
3013                 goto out_callb;
3014         }
3015
3016         err = event_init();
3017         if (err < 0)
3018                 goto out_reg;
3019
3020         iscsi_cmnd_cache = KMEM_CACHE(iscsi_cmnd, SCST_SLAB_FLAGS);
3021         if (!iscsi_cmnd_cache) {
3022                 err = -ENOMEM;
3023                 goto out_event;
3024         }
3025
3026         err = scst_register_target_template(&iscsi_template);
3027         if (err < 0)
3028                 goto out_kmem;
3029
3030         iscsi_template_registered = 1;
3031
3032         err = iscsi_procfs_init();
3033         if (err < 0)
3034                 goto out_reg_tmpl;
3035
3036         num = max(num_online_cpus(), 2);
3037
3038         err = iscsi_run_threads(num, "iscsird", istrd);
3039         if (err != 0)
3040                 goto out_thr;
3041
3042         err = iscsi_run_threads(num, "iscsiwr", istwr);
3043         if (err != 0)
3044                 goto out_thr;
3045
3046 out:
3047         return err;
3048
3049 out_thr:
3050         iscsi_procfs_exit();
3051         iscsi_stop_threads();
3052
3053 out_reg_tmpl:
3054         scst_unregister_target_template(&iscsi_template);
3055
3056 out_kmem:
3057         kmem_cache_destroy(iscsi_cmnd_cache);
3058
3059 out_event:
3060         event_exit();
3061
3062 out_reg:
3063         unregister_chrdev(ctr_major, ctr_name);
3064
3065 out_callb:
3066 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
3067         net_set_get_put_page_callbacks(NULL, NULL);
3068
3069 out_free_dummy:
3070 #endif
3071         __free_pages(dummy_page, 0);
3072         goto out;
3073 }
3074
3075 static void __exit iscsi_exit(void)
3076 {
3077         iscsi_stop_threads();
3078
3079         unregister_chrdev(ctr_major, ctr_name);
3080
3081         iscsi_procfs_exit();
3082         event_exit();
3083
3084         kmem_cache_destroy(iscsi_cmnd_cache);
3085
3086         scst_unregister_target_template(&iscsi_template);
3087
3088 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
3089         net_set_get_put_page_callbacks(NULL, NULL);
3090 #endif
3091
3092         __free_pages(dummy_page, 0);
3093         return;
3094 }
3095
3096 module_init(iscsi_init);
3097 module_exit(iscsi_exit);
3098
3099 MODULE_LICENSE("GPL");