89930fab7fa3f206797f199d6da102dd7b3f1c07
[mirror/scst/.git] / iscsi-scst / kernel / iscsi.c
1 /*
2  *  Copyright (C) 2002-2003 Ardis Technolgies <roman@ardistech.com>
3  *  Copyright (C) 2007 Vladislav Bolkhovitin
4  *  Copyright (C) 2007 CMS Distribution Limited
5  *
6  *  This program is free software; you can redistribute it and/or
7  *  modify it under the terms of the GNU General Public License
8  *  as published by the Free Software Foundation, version 2
9  *  of the License.
10  *
11  *  This program is distributed in the hope that it will be useful,
12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  *  GNU General Public License for more details.
15  */
16
17 #include <linux/module.h>
18 #include <linux/hash.h>
19 #include <linux/kthread.h>
20 #include <net/tcp.h>
21 #include <scsi/scsi.h>
22
23 #include "iscsi.h"
24 #include "digest.h"
25
26 #ifndef NET_PAGE_CALLBACKS_DEFINED
27 #warning Patch put_page_callback-<kernel-version>.patch not applied on your \
28         kernel. ISCSI-SCST will run in the performance degraded mode. Refer \
29         README file for details.
30 #endif
31
32 #define ISCSI_INIT_WRITE_WAKE           0x1
33 #define ISCSI_INIT_WRITE_REMOVE_HASH    0x2
34
35 static int ctr_major;
36 static char ctr_name[] = "iscsi-scst-ctl";
37 static int iscsi_template_registered;
38
39 #if defined(DEBUG) || defined(TRACING)
40 unsigned long iscsi_trace_flag = ISCSI_DEFAULT_LOG_FLAGS;
41 #endif
42
43 static struct kmem_cache *iscsi_cmnd_cache;
44
45 spinlock_t iscsi_rd_lock = SPIN_LOCK_UNLOCKED;
46 LIST_HEAD(iscsi_rd_list);
47 DECLARE_WAIT_QUEUE_HEAD(iscsi_rd_waitQ);
48
49 spinlock_t iscsi_wr_lock = SPIN_LOCK_UNLOCKED;
50 LIST_HEAD(iscsi_wr_list);
51 DECLARE_WAIT_QUEUE_HEAD(iscsi_wr_waitQ);
52
53 static char dummy_data[1024];
54
55 struct iscsi_thread_t {
56         struct task_struct *thr;
57         struct list_head threads_list_entry;
58 };
59
60 static LIST_HEAD(iscsi_threads_list);
61
62 static void cmnd_remove_hash(struct iscsi_cmnd *cmnd);
63 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status);
64 static void cmnd_prepare_get_rejected_cmd_data(struct iscsi_cmnd *cmnd);
65 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess);
66 static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd);
67
68 static inline u32 cmnd_write_size(struct iscsi_cmnd *cmnd)
69 {
70         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
71
72         if (hdr->flags & ISCSI_CMD_WRITE)
73                 return be32_to_cpu(hdr->data_length);
74         return 0;
75 }
76
77 static inline u32 cmnd_read_size(struct iscsi_cmnd *cmnd)
78 {
79         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
80
81         if (hdr->flags & ISCSI_CMD_READ) {
82                 struct iscsi_rlength_ahdr *ahdr =
83                         (struct iscsi_rlength_ahdr *)cmnd->pdu.ahs;
84
85                 if (!(hdr->flags & ISCSI_CMD_WRITE))
86                         return be32_to_cpu(hdr->data_length);
87                 if (ahdr && ahdr->ahstype == ISCSI_AHSTYPE_RLENGTH)
88                         return be32_to_cpu(ahdr->read_length);
89         }
90         return 0;
91 }
92
93 static inline void iscsi_restart_cmnd(struct iscsi_cmnd *cmnd)
94 {
95         EXTRACHECKS_BUG_ON(cmnd->data_waiting);
96
97         cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
98         scst_restart_cmd(cmnd->scst_cmd, SCST_PREPROCESS_STATUS_SUCCESS,
99                 SCST_CONTEXT_THREAD);
100 }
101
102 static inline void iscsi_restart_waiting_cmnd(struct iscsi_cmnd *cmnd)
103 {
104         /*
105          * There is no race with conn_abort(), since all functions
106          * called from single read thread
107          */
108         iscsi_extracheck_is_rd_thread(cmnd->conn);
109         cmnd->data_waiting = 0;
110
111         iscsi_restart_cmnd(cmnd);
112 }
113
114 static inline void iscsi_fail_waiting_cmnd(struct iscsi_cmnd *cmnd)
115 {
116         TRACE_MGMT_DBG("Failing data waiting cmd %p", cmnd);
117
118         /*
119          * There is no race with conn_abort(), since all functions
120          * called from single read thread
121          */
122         iscsi_extracheck_is_rd_thread(cmnd->conn);
123         cmnd->data_waiting = 0;
124
125         req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
126 }
127
128 struct iscsi_cmnd *cmnd_alloc(struct iscsi_conn *conn, struct iscsi_cmnd *parent)
129 {
130         struct iscsi_cmnd *cmnd;
131
132         /* ToDo: __GFP_NOFAIL?? */
133 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17)
134         cmnd = kmem_cache_alloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
135         memset(cmnd, 0, sizeof(*cmnd));
136 #else
137         cmnd = kmem_cache_zalloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
138 #endif
139
140         atomic_set(&cmnd->ref_cnt, 1);
141         cmnd->scst_state = ISCSI_CMD_STATE_NEW;
142         cmnd->conn = conn;
143         cmnd->parent_req = parent;
144         init_waitqueue_head(&cmnd->scst_waitQ);
145
146         if (parent == NULL) {
147                 conn_get(conn);
148
149 #ifdef NET_PAGE_CALLBACKS_DEFINED
150                 atomic_set(&cmnd->net_ref_cnt, 0);
151 #endif
152                 spin_lock_init(&cmnd->rsp_cmd_lock);
153                 INIT_LIST_HEAD(&cmnd->rsp_cmd_list);
154                 INIT_LIST_HEAD(&cmnd->rx_ddigest_cmd_list);
155
156                 spin_lock_bh(&conn->cmd_list_lock);
157                 list_add_tail(&cmnd->cmd_list_entry, &conn->cmd_list);
158                 spin_unlock_bh(&conn->cmd_list_lock);
159         }
160
161         TRACE_DBG("conn %p, parent %p, cmnd %p", conn, parent, cmnd);
162         return cmnd;
163 }
164
165 /* Frees a command. Also frees the additional header. */
166 void cmnd_free(struct iscsi_cmnd *cmnd)
167 {
168         TRACE_DBG("%p", cmnd);
169
170         if (unlikely(cmnd->tm_aborted)) {
171                 TRACE_MGMT_DBG("Free aborted cmd %p (scst cmd %p, state %d, "
172                         "parent_req %p)", cmnd, cmnd->scst_cmd, cmnd->scst_state,
173                         cmnd->parent_req);
174         }
175
176         /* Catch users from cmd_list or rsp_cmd_list */
177         EXTRACHECKS_BUG_ON(atomic_read(&cmnd->ref_cnt) != 0);
178
179         kfree(cmnd->pdu.ahs);
180
181         if (unlikely(cmnd->on_write_list || cmnd->on_written_list)) {
182                 struct iscsi_scsi_cmd_hdr *req = cmnd_hdr(cmnd);
183
184                 PRINT_CRIT_ERROR("cmnd %p still on some list?, %x, %x, %x, %x, %x, %x, %x",
185                         cmnd, req->opcode, req->scb[0], req->flags, req->itt,
186                         be32_to_cpu(req->data_length),
187                         req->cmd_sn, be32_to_cpu(cmnd->pdu.datasize));
188
189                 if (unlikely(cmnd->parent_req)) {
190                         struct iscsi_scsi_cmd_hdr *req =
191                                         cmnd_hdr(cmnd->parent_req);
192                         PRINT_CRIT_ERROR("%p %x %u", req, req->opcode, req->scb[0]);
193                 }
194                 sBUG();
195         }
196
197         kmem_cache_free(iscsi_cmnd_cache, cmnd);
198         return;
199 }
200
201 void cmnd_done(struct iscsi_cmnd *cmnd)
202 {
203         TRACE_DBG("%p", cmnd);
204
205         if (unlikely(cmnd->tm_aborted)) {
206                 TRACE_MGMT_DBG("Done aborted cmd %p (scst cmd %p, state %d, "
207                         "parent_req %p)", cmnd, cmnd->scst_cmd, cmnd->scst_state,
208                         cmnd->parent_req);
209         }
210
211         EXTRACHECKS_BUG_ON(cmnd->on_rx_digest_list);
212
213         if (cmnd->on_written_list) {
214                 struct iscsi_conn *conn = cmnd->conn;
215                 TRACE_DBG("Deleting cmd %p from conn %p written_list", cmnd,
216                         conn);
217                 spin_lock_bh(&conn->write_list_lock);
218                 list_del(&cmnd->write_list_entry);
219                 cmnd->on_written_list = 0;
220                 spin_unlock_bh(&conn->write_list_lock);
221         }
222
223         if (cmnd->parent_req == NULL) {
224                 struct iscsi_conn *conn = cmnd->conn;
225                 TRACE_DBG("Deleting req %p from conn %p", cmnd, conn);
226
227                 spin_lock_bh(&conn->cmd_list_lock);
228                 list_del(&cmnd->cmd_list_entry);
229                 spin_unlock_bh(&conn->cmd_list_lock);
230
231                 conn_put(conn);
232
233                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rsp_cmd_list));
234                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rx_ddigest_cmd_list));
235
236                 /* Order between above and below code is important! */
237
238                 if (cmnd->scst_cmd) {
239                         switch (cmnd->scst_state) {
240                         case ISCSI_CMD_STATE_PROCESSED:
241                                 TRACE_DBG("cmd %p PROCESSED", cmnd);
242                                 scst_tgt_cmd_done(cmnd->scst_cmd);
243                                 break;
244                         case ISCSI_CMD_STATE_AFTER_PREPROC:
245                         {
246                                 struct scst_cmd *scst_cmd = cmnd->scst_cmd;
247                                 TRACE_DBG("cmd %p AFTER_PREPROC", cmnd);
248                                 cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
249                                 cmnd->scst_cmd = NULL;
250                                 scst_restart_cmd(scst_cmd,
251                                         SCST_PREPROCESS_STATUS_ERROR_FATAL,
252                                         SCST_CONTEXT_THREAD);
253                                 break;
254                         }
255                         default:
256                                 PRINT_CRIT_ERROR("Unexpected cmnd scst state %d",
257                                                  cmnd->scst_state);
258                                 sBUG();
259                                 break;
260                         }
261                 }
262         } else {
263                 EXTRACHECKS_BUG_ON(cmnd->scst_cmd != NULL);
264                 TRACE_DBG("Deleting rsp %p from parent %p", cmnd,
265                         cmnd->parent_req);
266
267                 spin_lock_bh(&cmnd->parent_req->rsp_cmd_lock);
268                 list_del(&cmnd->rsp_cmd_list_entry);
269                 spin_unlock_bh(&cmnd->parent_req->rsp_cmd_lock);
270
271                 cmnd_put(cmnd->parent_req);
272         }
273
274         /* Order between above and below code is important! */
275
276         if (cmnd->own_sg) {
277                 TRACE_DBG("%s", "own_sg");
278                 scst_free(cmnd->sg, cmnd->sg_cnt);
279 #ifdef DEBUG
280                 cmnd->own_sg = 0;
281                 cmnd->sg = NULL;
282                 cmnd->sg_cnt = -1;
283 #endif
284         }
285
286         if (cmnd->dec_active_cmnds) {
287                 struct iscsi_session *sess = cmnd->conn->session;
288                 TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
289                         "new value %d)", cmnd, sess,
290                         atomic_read(&sess->active_cmds)-1);
291                 atomic_dec(&sess->active_cmds);
292 #ifdef EXTRACHECKS
293                 if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
294                         PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
295                                 atomic_read(&sess->active_cmds));
296                         sBUG();
297                 }
298 #endif
299         }
300
301         cmnd_free(cmnd);
302         return;
303 }
304
305 /*
306  * Corresponding conn may also gets destroyed atfer this function, except only
307  * if it's called from the read thread!
308  *
309  * It can't be called in parallel with iscsi_cmnds_init_write()!
310  */
311 void req_cmnd_release_force(struct iscsi_cmnd *req, int flags)
312 {
313         struct iscsi_cmnd *rsp, *t;
314         struct iscsi_conn *conn = req->conn;
315         LIST_HEAD(cmds_list);
316
317         TRACE_ENTRY();
318
319         TRACE_MGMT_DBG("%p", req);
320
321         sBUG_ON(req == conn->read_cmnd);
322
323         if (flags & ISCSI_FORCE_RELEASE_WRITE) {
324                 spin_lock_bh(&conn->write_list_lock);
325                 list_for_each_entry_safe(rsp, t, &conn->write_list,
326                                                 write_list_entry) {
327                         if (rsp->parent_req != req)
328                                 continue;
329
330                         cmd_del_from_write_list(rsp);
331
332                         list_add_tail(&rsp->write_list_entry, &cmds_list);
333                 }
334                 spin_unlock_bh(&conn->write_list_lock);
335
336                 list_for_each_entry_safe(rsp, t, &cmds_list, write_list_entry) {
337                         TRACE_MGMT_DBG("Putting write rsp %p", rsp);
338                         list_del(&rsp->write_list_entry);
339                         cmnd_put(rsp);
340                 }
341         }
342
343 again_rsp:
344         spin_lock_bh(&req->rsp_cmd_lock);
345         list_for_each_entry_reverse(rsp, &req->rsp_cmd_list, rsp_cmd_list_entry) {
346                 bool r;
347
348                 if (rsp->force_cleanup_done)
349                         continue;
350
351                 rsp->force_cleanup_done = 1;
352
353                 if (cmnd_get_check(rsp))
354                         continue;
355
356                 spin_unlock_bh(&req->rsp_cmd_lock);
357
358                 spin_lock_bh(&conn->write_list_lock);
359                 r = rsp->on_write_list || rsp->write_processing_started;
360                 spin_unlock_bh(&conn->write_list_lock);
361
362                 cmnd_put(rsp);
363
364                 if (r)
365                         goto again_rsp;
366
367                 /*
368                  * If both on_write_list and write_processing_started not set,
369                  * we can safely put() rsp.
370                  */
371                 TRACE_MGMT_DBG("Putting rsp %p", rsp);
372                 cmnd_put(rsp);
373                 goto again_rsp;
374         }
375         spin_unlock_bh(&req->rsp_cmd_lock);
376
377         req_cmnd_release(req);
378
379         TRACE_EXIT();
380         return;
381 }
382
383 /*
384  * Corresponding conn may also gets destroyed atfer this function, except only
385  * if it's called from the read thread!
386  */
387 void req_cmnd_release(struct iscsi_cmnd *req)
388 {
389         struct iscsi_cmnd *c, *t;
390
391         TRACE_ENTRY();
392
393         TRACE_DBG("%p", req);
394
395 #ifdef EXTRACHECKS
396         sBUG_ON(req->release_called);
397         req->release_called = 1;
398 #endif
399
400         if (unlikely(req->tm_aborted)) {
401                 TRACE_MGMT_DBG("Release aborted req cmd %p (scst cmd %p, "
402                         "state %d)", req, req->scst_cmd, req->scst_state);
403         }
404
405         sBUG_ON(req->parent_req != NULL);
406
407         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
408                                 rx_ddigest_cmd_list_entry) {
409                 cmd_del_from_rx_ddigest_list(c);
410                 cmnd_put(c);
411         }
412
413         if (req->hashed)
414                 cmnd_remove_hash(req);
415
416         if (req->dec_active_cmnds) {
417                 struct iscsi_session *sess = req->conn->session;
418                 TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
419                         "new value %d)", req, sess,
420                         atomic_read(&sess->active_cmds)-1);
421                 atomic_dec(&sess->active_cmds);
422                 req->dec_active_cmnds = 0;
423 #ifdef EXTRACHECKS
424                 if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
425                         PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
426                                 atomic_read(&sess->active_cmds));
427                         sBUG();
428                 }
429 #endif
430         }
431
432         cmnd_put(req);
433
434         TRACE_EXIT();
435         return;
436 }
437
438 /*
439  * Corresponding conn may also gets destroyed atfer this function, except only
440  * if it's called from the read thread!
441  */
442 void rsp_cmnd_release(struct iscsi_cmnd *cmnd)
443 {
444         TRACE_DBG("%p", cmnd);
445
446 #ifdef EXTRACHECKS
447         sBUG_ON(cmnd->release_called);
448         cmnd->release_called = 1;
449 #endif
450
451         sBUG_ON(cmnd->hashed);
452         sBUG_ON(cmnd->parent_req == NULL);
453
454         cmnd_put(cmnd);
455         return;
456 }
457
458 /**
459  * create a new command used as response.
460  *
461  * iscsi_cmnd_create_rsp_cmnd -
462  * @cmnd: ptr to request command
463  *
464  * @return    ptr to response command or NULL
465  */
466 static struct iscsi_cmnd *iscsi_cmnd_create_rsp_cmnd(struct iscsi_cmnd *parent)
467 {
468         struct iscsi_cmnd *rsp;
469
470         rsp = cmnd_alloc(parent->conn, parent);
471
472         spin_lock_bh(&parent->rsp_cmd_lock);
473         TRACE_DBG("Adding rsp %p to parent %p", rsp, parent);
474         list_add_tail(&rsp->rsp_cmd_list_entry, &parent->rsp_cmd_list);
475         spin_unlock_bh(&parent->rsp_cmd_lock);
476         cmnd_get(parent);
477         return rsp;
478 }
479
480 static inline struct iscsi_cmnd *get_rsp_cmnd(struct iscsi_cmnd *req)
481 {
482         struct iscsi_cmnd *res = NULL;
483
484         /* Currently this lock isn't needed, but just in case.. */
485         spin_lock_bh(&req->rsp_cmd_lock);
486         if (!list_empty(&req->rsp_cmd_list)) {
487                 res = list_entry(req->rsp_cmd_list.prev, struct iscsi_cmnd,
488                         rsp_cmd_list_entry);
489         }
490         spin_unlock_bh(&req->rsp_cmd_lock);
491
492         return res;
493 }
494
495 static void iscsi_cmnds_init_write(struct list_head *send, int flags)
496 {
497         struct iscsi_cmnd *rsp = list_entry(send->next, struct iscsi_cmnd,
498                                                 write_list_entry);
499         struct iscsi_conn *conn = rsp->conn;
500         struct list_head *pos, *next;
501
502         sBUG_ON(list_empty(send));
503
504         /*
505          * If we don't remove hashed req cmd from the hash list here, before
506          * submitting it for transmittion, we will have a race, when for
507          * some reason cmd's release is delayed after transmittion and
508          * initiator sends cmd with the same ITT => this command will be
509          * erroneously rejected as a duplicate.
510          */
511         if ((flags & ISCSI_INIT_WRITE_REMOVE_HASH) && rsp->parent_req->hashed &&
512             (rsp->parent_req->r2t_length == 0) &&
513             (rsp->parent_req->outstanding_r2t == 0))
514                 cmnd_remove_hash(rsp->parent_req);
515
516         if (!(conn->ddigest_type & DIGEST_NONE)) {
517                 list_for_each(pos, send) {
518                         rsp = list_entry(pos, struct iscsi_cmnd,
519                                                 write_list_entry);
520
521                         if (rsp->pdu.datasize != 0) {
522                                 TRACE_DBG("Doing data digest (%p:%x)", rsp,
523                                         cmnd_opcode(rsp));
524                                 digest_tx_data(rsp);
525                         }
526                 }
527         }
528
529         spin_lock_bh(&conn->write_list_lock);
530         list_for_each_safe(pos, next, send) {
531                 rsp = list_entry(pos, struct iscsi_cmnd, write_list_entry);
532
533                 TRACE_DBG("%p:%x", rsp, cmnd_opcode(rsp));
534
535                 sBUG_ON(conn != rsp->conn);
536
537                 list_del(&rsp->write_list_entry);
538                 cmd_add_on_write_list(conn, rsp);
539         }
540         spin_unlock_bh(&conn->write_list_lock);
541
542         if (flags & ISCSI_INIT_WRITE_WAKE)
543                 iscsi_make_conn_wr_active(conn);
544 }
545
546 static void iscsi_cmnd_init_write(struct iscsi_cmnd *rsp, int flags)
547 {
548         LIST_HEAD(head);
549
550         if (unlikely(rsp->on_write_list)) {
551                 PRINT_CRIT_ERROR("cmd already on write list (%x %x %x %x %u %u "
552                         "%u %u %u %u %u %d %d",
553                         cmnd_itt(rsp), cmnd_ttt(rsp), cmnd_opcode(rsp),
554                         cmnd_scsicode(rsp), rsp->r2t_sn,
555                         rsp->r2t_length, rsp->is_unsolicited_data,
556                         rsp->target_task_tag, rsp->outstanding_r2t,
557                         rsp->hdigest, rsp->ddigest,
558                         list_empty(&rsp->rsp_cmd_list), rsp->hashed);
559                 sBUG();
560         }
561         list_add(&rsp->write_list_entry, &head);
562         iscsi_cmnds_init_write(&head, flags);
563 }
564
565 static void iscsi_set_datasize(struct iscsi_cmnd *cmnd, u32 offset, u32 size)
566 {
567         cmnd->pdu.datasize = size;
568
569         if (size & 3) {
570                 u32 last_off = offset + size;
571                 int idx = last_off >> PAGE_SHIFT;
572                 u8 *p = (u8*)page_address(sg_page(&cmnd->sg[idx])) +
573                         (last_off & ~PAGE_MASK);
574                 int i = 4 - (size & 3);
575                 while (i--)
576                     *p++ = 0;
577         }
578 }
579
580 static void send_data_rsp(struct iscsi_cmnd *req, u8 status, int send_status)
581 {
582         struct iscsi_cmnd *rsp;
583         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
584         struct iscsi_data_in_hdr *rsp_hdr;
585         u32 pdusize, expsize, scsisize, size, offset, sn;
586         LIST_HEAD(send);
587
588         TRACE_DBG("req %p", req);
589
590         pdusize = req->conn->session->sess_param.max_xmit_data_length;
591         expsize = cmnd_read_size(req);
592         size = min(expsize, (u32)req->bufflen);
593         offset = 0;
594         sn = 0;
595
596         while (1) {
597                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
598                 TRACE_DBG("rsp %p", rsp);
599                 rsp->sg = req->sg;
600                 rsp->sg_cnt = req->sg_cnt;
601                 rsp->bufflen = req->bufflen;
602                 rsp_hdr = (struct iscsi_data_in_hdr *)&rsp->pdu.bhs;
603
604                 rsp_hdr->opcode = ISCSI_OP_SCSI_DATA_IN;
605                 rsp_hdr->itt = req_hdr->itt;
606                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
607                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
608                 rsp_hdr->data_sn = cpu_to_be32(sn);
609
610                 if (size <= pdusize) {
611                         TRACE_DBG("offset %d, size %d", offset, size);
612                         iscsi_set_datasize(rsp, offset, size);
613                         if (send_status) {
614                                 TRACE_DBG("status %x", status);
615                                 rsp_hdr->flags =
616                                         ISCSI_FLG_FINAL | ISCSI_FLG_STATUS;
617                                 rsp_hdr->cmd_status = status;
618                         }
619                         scsisize = req->bufflen;
620                         if (scsisize < expsize) {
621                                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
622                                 size = expsize - scsisize;
623                         } else if (scsisize > expsize) {
624                                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
625                                 size = scsisize - expsize;
626                         } else
627                                 size = 0;
628                         rsp_hdr->residual_count = cpu_to_be32(size);
629                         list_add_tail(&rsp->write_list_entry, &send);
630                         break;
631                 }
632
633                 TRACE_DBG("pdusize %d, offset %d, size %d", pdusize, offset,
634                         size);
635
636                 iscsi_set_datasize(rsp, offset, pdusize);
637
638                 size -= pdusize;
639                 offset += pdusize;
640                 sn++;
641
642                 list_add_tail(&rsp->write_list_entry, &send);
643         }
644         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_REMOVE_HASH);
645 }
646
647 static struct iscsi_cmnd *create_status_rsp(struct iscsi_cmnd *req, int status,
648         const u8 *sense_buf, int sense_len)
649 {
650         struct iscsi_cmnd *rsp;
651         struct iscsi_scsi_rsp_hdr *rsp_hdr;
652         struct iscsi_sense_data *sense;
653         struct scatterlist *sg;
654
655         rsp = iscsi_cmnd_create_rsp_cmnd(req);
656         TRACE_DBG("%p", rsp);
657
658         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
659         rsp_hdr->opcode = ISCSI_OP_SCSI_RSP;
660         rsp_hdr->flags = ISCSI_FLG_FINAL;
661         rsp_hdr->response = ISCSI_RESPONSE_COMMAND_COMPLETED;
662         rsp_hdr->cmd_status = status;
663         rsp_hdr->itt = cmnd_hdr(req)->itt;
664
665         if (SCST_SENSE_VALID(sense_buf)) {
666                 TRACE_DBG("%s", "SENSE VALID");
667                 /* ToDo: __GFP_NOFAIL ?? */
668                 sg = rsp->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
669                                         &rsp->sg_cnt);
670                 if (sg == NULL) {
671                         /* ToDo(); */
672                 }
673                 rsp->own_sg = 1;
674                 sense = (struct iscsi_sense_data *)page_address(sg_page(&sg[0]));
675                 sense->length = cpu_to_be16(sense_len);
676                 memcpy(sense->data, sense_buf, sense_len);
677                 rsp->pdu.datasize = sizeof(struct iscsi_sense_data) + sense_len;
678                 rsp->bufflen = (rsp->pdu.datasize + 3) & -4;
679                 if (rsp->bufflen - rsp->pdu.datasize) {
680                     int i = rsp->pdu.datasize;
681                     u8 *p = (u8 *)sense + i;
682
683                     while (i < rsp->bufflen) {
684                         *p++ = 0;
685                         i++;
686                     }
687                 }
688         } else {
689                 rsp->pdu.datasize = 0;
690                 rsp->bufflen = 0;
691         }
692
693         return rsp;
694 }
695
696 static struct iscsi_cmnd *create_sense_rsp(struct iscsi_cmnd *req,
697         u8 sense_key, u8 asc, u8 ascq)
698 {
699         u8 sense[14];
700         memset(sense, 0, sizeof(sense));
701         sense[0] = 0xf0;
702         sense[2] = sense_key;
703         sense[7] = 6;   // Additional sense length
704         sense[12] = asc;
705         sense[13] = ascq;
706         return create_status_rsp(req, SAM_STAT_CHECK_CONDITION, sense,
707                 sizeof(sense));
708 }
709
710 static void iscsi_cmnd_reject(struct iscsi_cmnd *req, int reason)
711 {
712         struct iscsi_cmnd *rsp;
713         struct iscsi_reject_hdr *rsp_hdr;
714         struct scatterlist *sg;
715         char *addr;
716
717         TRACE_MGMT_DBG("Reject: req %p, reason %x", req, reason);
718
719         sBUG_ON(req->rejected);
720         req->rejected = 1;
721         req->reject_reason = ISCSI_REJECT_CMD;
722
723         rsp = iscsi_cmnd_create_rsp_cmnd(req);
724         rsp_hdr = (struct iscsi_reject_hdr *)&rsp->pdu.bhs;
725
726         rsp_hdr->opcode = ISCSI_OP_REJECT;
727         rsp_hdr->ffffffff = ISCSI_RESERVED_TAG;
728         rsp_hdr->reason = reason;
729
730         /* ToDo: __GFP_NOFAIL ?? */
731         sg = rsp->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
732                                 &rsp->sg_cnt);
733         if (sg == NULL) {
734                 /* ToDo(); */
735         }
736         rsp->own_sg = 1;
737         addr = page_address(sg_page(&sg[0]));
738         clear_page(addr);
739         memcpy(addr, &req->pdu.bhs, sizeof(struct iscsi_hdr));
740         rsp->bufflen = rsp->pdu.datasize = sizeof(struct iscsi_hdr);
741
742         iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH |
743                                          ISCSI_INIT_WRITE_WAKE);
744
745         cmnd_prepare_get_rejected_cmd_data(req);
746 }
747
748 static inline int iscsi_get_allowed_cmds(struct iscsi_session *sess)
749 {
750         int res = max(-1, (int)sess->max_queued_cmnds -
751                                 atomic_read(&sess->active_cmds)-1);
752         TRACE_DBG("allowed cmds %d (sess %p, active_cmds %d)", res,
753                 sess, atomic_read(&sess->active_cmds));
754         return res;
755 }
756
757 static u32 cmnd_set_sn(struct iscsi_cmnd *cmnd, int set_stat_sn)
758 {
759         struct iscsi_conn *conn = cmnd->conn;
760         struct iscsi_session *sess = conn->session;
761         u32 res;
762
763         spin_lock(&sess->sn_lock);
764
765         if (set_stat_sn)
766                 cmnd->pdu.bhs.sn = cpu_to_be32(conn->stat_sn++);
767         cmnd->pdu.bhs.exp_sn = cpu_to_be32(sess->exp_cmd_sn);
768         cmnd->pdu.bhs.max_sn = cpu_to_be32(sess->exp_cmd_sn +
769                                  iscsi_get_allowed_cmds(sess));
770
771         res = cpu_to_be32(conn->stat_sn);
772
773         spin_unlock(&sess->sn_lock);
774         return res;
775 }
776
777 /* Called under sn_lock */
778 static void __update_stat_sn(struct iscsi_cmnd *cmnd)
779 {
780         struct iscsi_conn *conn = cmnd->conn;
781         u32 exp_stat_sn;
782
783         cmnd->pdu.bhs.exp_sn = exp_stat_sn = be32_to_cpu(cmnd->pdu.bhs.exp_sn);
784         TRACE_DBG("%x,%x", cmnd_opcode(cmnd), exp_stat_sn);
785         if ((int)(exp_stat_sn - conn->exp_stat_sn) > 0 &&
786             (int)(exp_stat_sn - conn->stat_sn) <= 0) {
787                 // free pdu resources
788                 cmnd->conn->exp_stat_sn = exp_stat_sn;
789         }
790 }
791
792 static inline void update_stat_sn(struct iscsi_cmnd *cmnd)
793 {
794         spin_lock(&cmnd->conn->session->sn_lock);
795         __update_stat_sn(cmnd);
796         spin_unlock(&cmnd->conn->session->sn_lock);
797 }
798
799 /* Called under sn_lock */
800 static int check_cmd_sn(struct iscsi_cmnd *cmnd)
801 {
802         struct iscsi_session *session = cmnd->conn->session;
803         u32 cmd_sn;
804
805         cmnd->pdu.bhs.sn = cmd_sn = be32_to_cpu(cmnd->pdu.bhs.sn);
806         TRACE_DBG("%d(%d)", cmd_sn, session->exp_cmd_sn);
807         if (likely((s32)(cmd_sn - session->exp_cmd_sn) >= 0))
808                 return 0;
809         PRINT_ERROR("sequence error (%x,%x)", cmd_sn, session->exp_cmd_sn);
810         return -ISCSI_REASON_PROTOCOL_ERROR;
811 }
812
813 static inline struct iscsi_cmnd *__cmnd_find_hash(struct iscsi_session *session,
814         u32 itt, u32 ttt)
815 {
816         struct list_head *head;
817         struct iscsi_cmnd *cmnd;
818
819         head = &session->cmnd_hash[cmnd_hashfn(itt)];
820
821         list_for_each_entry(cmnd, head, hash_list_entry) {
822                 if (cmnd->pdu.bhs.itt == itt) {
823                         if ((ttt != ISCSI_RESERVED_TAG) && (ttt != cmnd->target_task_tag))
824                                 continue;
825                         return cmnd;
826                 }
827         }
828         return NULL;
829 }
830
831 static struct iscsi_cmnd *cmnd_find_hash(struct iscsi_session *session,
832         u32 itt, u32 ttt)
833 {
834         struct iscsi_cmnd *cmnd;
835
836         spin_lock(&session->cmnd_hash_lock);
837         cmnd = __cmnd_find_hash(session, itt, ttt);
838         spin_unlock(&session->cmnd_hash_lock);
839
840         return cmnd;
841 }
842
843 static struct iscsi_cmnd *cmnd_find_hash_get(struct iscsi_session *session,
844         u32 itt, u32 ttt)
845 {
846         struct iscsi_cmnd *cmnd;
847
848         spin_lock(&session->cmnd_hash_lock);
849         cmnd = __cmnd_find_hash(session, itt, ttt);
850         if (cmnd != NULL) {
851                 if (unlikely(cmnd_get_check(cmnd)))
852                         cmnd = NULL;
853         }
854         spin_unlock(&session->cmnd_hash_lock);
855
856         return cmnd;
857 }
858
859 static int cmnd_insert_hash(struct iscsi_cmnd *cmnd)
860 {
861         struct iscsi_session *session = cmnd->conn->session;
862         struct iscsi_cmnd *tmp;
863         struct list_head *head;
864         int err = 0;
865         u32 itt = cmnd->pdu.bhs.itt;
866
867         TRACE_DBG("%p:%x", cmnd, itt);
868         if (unlikely(itt == ISCSI_RESERVED_TAG)) {
869                 PRINT_ERROR("%s", "ITT is RESERVED_TAG");
870                 PRINT_BUFFER("Incorrect BHS", &cmnd->pdu.bhs,
871                         sizeof(cmnd->pdu.bhs));
872                 err = -ISCSI_REASON_PROTOCOL_ERROR;
873                 goto out;
874         }
875
876         spin_lock(&session->cmnd_hash_lock);
877
878         head = &session->cmnd_hash[cmnd_hashfn(cmnd->pdu.bhs.itt)];
879
880         tmp = __cmnd_find_hash(session, itt, ISCSI_RESERVED_TAG);
881         if (likely(!tmp)) {
882                 list_add_tail(&cmnd->hash_list_entry, head);
883                 cmnd->hashed = 1;
884         } else {
885                 PRINT_ERROR("Task %x in progress, cmnd %p", itt, cmnd);
886                 err = -ISCSI_REASON_TASK_IN_PROGRESS;
887         }
888
889         spin_unlock(&session->cmnd_hash_lock);
890
891         if (likely(!err)) {
892                 spin_lock(&session->sn_lock);
893                 __update_stat_sn(cmnd);
894                 err = check_cmd_sn(cmnd);
895                 spin_unlock(&session->sn_lock);
896         }
897
898 out:
899         return err;
900 }
901
902 static void cmnd_remove_hash(struct iscsi_cmnd *cmnd)
903 {
904         struct iscsi_session *session = cmnd->conn->session;
905         struct iscsi_cmnd *tmp;
906
907         spin_lock(&session->cmnd_hash_lock);
908
909         tmp = __cmnd_find_hash(session, cmnd->pdu.bhs.itt, ISCSI_RESERVED_TAG);
910
911         if (likely(tmp && tmp == cmnd)) {
912                 list_del(&cmnd->hash_list_entry);
913                 cmnd->hashed = 0;
914         } else {
915                 PRINT_ERROR("%p:%x not found", cmnd, cmnd_itt(cmnd));
916         }
917
918         spin_unlock(&session->cmnd_hash_lock);
919 }
920
921 static void cmnd_prepare_get_rejected_cmd_data(struct iscsi_cmnd *cmnd)
922 {
923         struct iscsi_conn *conn = cmnd->conn;
924         struct scatterlist *sg = cmnd->sg;
925         char *addr;
926         u32 size;
927         int i;
928
929         TRACE_MGMT_DBG("Skipping (%p, %x %x %x %u, %p, scst state %d)", cmnd,
930                 cmnd_itt(cmnd), cmnd_opcode(cmnd), cmnd_hdr(cmnd)->scb[0],
931                 cmnd->pdu.datasize, cmnd->scst_cmd, cmnd->scst_state);
932
933         iscsi_extracheck_is_rd_thread(conn);
934
935         if (!(size = cmnd->pdu.datasize))
936                 return;
937
938         if (sg == NULL) {
939                 /* ToDo: __GFP_NOFAIL ?? */
940                 sg = cmnd->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
941                                         &cmnd->sg_cnt);
942                 if (sg == NULL) {
943                         /* ToDo(); */
944                 }
945                 cmnd->own_sg = 1;
946                 cmnd->bufflen = PAGE_SIZE;
947         }
948
949         addr = page_address(sg_page(&sg[0]));
950         sBUG_ON(addr == NULL);
951         size = (size + 3) & -4;
952         conn->read_size = size;
953         for (i = 0; size > PAGE_SIZE; i++, size -= cmnd->bufflen) {
954                 sBUG_ON(i >= ISCSI_CONN_IOV_MAX);
955                 conn->read_iov[i].iov_base = addr;
956                 conn->read_iov[i].iov_len = cmnd->bufflen;
957         }
958         conn->read_iov[i].iov_base = addr;
959         conn->read_iov[i].iov_len = size;
960         conn->read_msg.msg_iov = conn->read_iov;
961         conn->read_msg.msg_iovlen = ++i;
962
963         return;
964 }
965
966 static void cmnd_reject_scsi_cmd(struct iscsi_cmnd *req)
967 {
968         struct iscsi_cmnd *rsp;
969         struct iscsi_scsi_rsp_hdr *rsp_hdr;
970         u32 size;
971
972         TRACE_DBG("%p", req);
973
974         sBUG_ON(req->rejected);
975         req->rejected = 1;
976         req->reject_reason = ISCSI_REJECT_SCSI_CMD;
977
978         rsp = get_rsp_cmnd(req);
979         if (rsp == NULL) {
980                 /* That can be true for aborted commands */
981                 goto out_reject;
982         }
983
984         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
985
986         sBUG_ON(cmnd_opcode(rsp) != ISCSI_OP_SCSI_RSP);
987
988         size = cmnd_write_size(req);
989         if (size) {
990                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
991                 rsp_hdr->residual_count = cpu_to_be32(size);
992         }
993         size = cmnd_read_size(req);
994         if (size) {
995                 if (cmnd_hdr(req)->flags & ISCSI_CMD_WRITE) {
996                         rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_UNDERFLOW;
997                         rsp_hdr->bi_residual_count = cpu_to_be32(size);
998                 } else {
999                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
1000                         rsp_hdr->residual_count = cpu_to_be32(size);
1001                 }
1002         }
1003
1004         iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH |
1005                                          ISCSI_INIT_WRITE_WAKE);
1006
1007 out_reject:
1008         cmnd_prepare_get_rejected_cmd_data(req);
1009         return;
1010 }
1011
1012 static int cmnd_prepare_recv_pdu(struct iscsi_conn *conn,
1013         struct iscsi_cmnd *cmd, u32 offset, u32 size)
1014 {
1015         struct scatterlist *sg = cmd->sg;
1016         int bufflen = cmd->bufflen;
1017         int idx, i;
1018         char *addr;
1019         int res = 0;
1020
1021         TRACE_DBG("%p %u,%u", cmd->sg, offset, size);
1022
1023         iscsi_extracheck_is_rd_thread(conn);
1024
1025         if (unlikely((offset >= bufflen) ||
1026                      (offset + size > bufflen))) {
1027                 PRINT_ERROR("Wrong ltn (%u %u %u)", offset, size, bufflen);
1028                 mark_conn_closed(conn);
1029                 res = -EIO;
1030                 goto out;
1031         }
1032
1033         offset += sg[0].offset;
1034         idx = offset >> PAGE_SHIFT;
1035         offset &= ~PAGE_MASK;
1036
1037         conn->read_msg.msg_iov = conn->read_iov;
1038         conn->read_size = size = (size + 3) & -4;
1039
1040         i = 0;
1041         while (1) {
1042                 addr = page_address(sg_page(&sg[idx]));
1043                 sBUG_ON(addr == NULL);
1044                 conn->read_iov[i].iov_base = addr + offset;
1045                 if (offset + size <= PAGE_SIZE) {
1046                         TRACE_DBG("idx=%d, offset=%u, size=%d, addr=%p",
1047                                 idx, offset, size, addr);
1048                         conn->read_iov[i].iov_len = size;
1049                         conn->read_msg.msg_iovlen = ++i;
1050                         break;
1051                 }
1052                 conn->read_iov[i].iov_len = PAGE_SIZE - offset;
1053                 TRACE_DBG("idx=%d, offset=%u, size=%d, iov_len=%zd, addr=%p",
1054                         idx, offset, size, conn->read_iov[i].iov_len, addr);
1055                 size -= conn->read_iov[i].iov_len;
1056                 offset = 0;
1057                 if (unlikely(++i >= ISCSI_CONN_IOV_MAX)) {
1058                         PRINT_ERROR("Initiator %s violated negotiated "
1059                                 "parameters by sending too much data (size "
1060                                 "left %d)", conn->session->initiator_name, size);
1061                         mark_conn_closed(conn);
1062                         res = -EINVAL;
1063                         break;
1064                 }
1065                 idx++;
1066         }
1067         TRACE_DBG("msg_iov=%p, msg_iovlen=%zd",
1068                 conn->read_msg.msg_iov, conn->read_msg.msg_iovlen);
1069
1070 out:
1071         return res;
1072 }
1073
1074 static void send_r2t(struct iscsi_cmnd *req)
1075 {
1076         struct iscsi_session *session = req->conn->session;
1077         struct iscsi_cmnd *rsp;
1078         struct iscsi_r2t_hdr *rsp_hdr;
1079         u32 offset, burst;
1080         LIST_HEAD(send);
1081
1082         if (unlikely(req->tm_aborted)) {
1083                 TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted on R2T "
1084                         "(r2t_length %d, outstanding_r2t %d)", req,
1085                         req->scst_cmd, req->r2t_length, req->outstanding_r2t);
1086                 if (req->outstanding_r2t == 0)
1087                         iscsi_fail_waiting_cmnd(req);
1088                 goto out;
1089         }
1090
1091         /*
1092          * There is no race with data_out_start() and conn_abort(), since
1093          * all functions called from single read thread
1094          */
1095         iscsi_extracheck_is_rd_thread(req->conn);
1096
1097         burst = session->sess_param.max_burst_length;
1098         offset = be32_to_cpu(cmnd_hdr(req)->data_length) - req->r2t_length;
1099
1100         do {
1101                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
1102                 rsp->pdu.bhs.ttt = req->target_task_tag;
1103                 rsp_hdr = (struct iscsi_r2t_hdr *)&rsp->pdu.bhs;
1104                 rsp_hdr->opcode = ISCSI_OP_R2T;
1105                 rsp_hdr->flags = ISCSI_FLG_FINAL;
1106                 rsp_hdr->lun = cmnd_hdr(req)->lun;
1107                 rsp_hdr->itt = cmnd_hdr(req)->itt;
1108                 rsp_hdr->r2t_sn = cpu_to_be32(req->r2t_sn++);
1109                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
1110                 if (req->r2t_length > burst) {
1111                         rsp_hdr->data_length = cpu_to_be32(burst);
1112                         req->r2t_length -= burst;
1113                         offset += burst;
1114                 } else {
1115                         rsp_hdr->data_length = cpu_to_be32(req->r2t_length);
1116                         req->r2t_length = 0;
1117                 }
1118
1119                 TRACE_WRITE("%x %u %u %u %u", cmnd_itt(req),
1120                         be32_to_cpu(rsp_hdr->data_length),
1121                         be32_to_cpu(rsp_hdr->buffer_offset),
1122                         be32_to_cpu(rsp_hdr->r2t_sn), req->outstanding_r2t);
1123
1124                 list_add_tail(&rsp->write_list_entry, &send);
1125
1126                 if (++req->outstanding_r2t >= session->sess_param.max_outstanding_r2t)
1127                         break;
1128
1129         } while (req->r2t_length != 0);
1130
1131         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_WAKE);
1132
1133 out:
1134         return;
1135 }
1136
1137 static int iscsi_pre_exec(struct scst_cmd *scst_cmd)
1138 {
1139         int res = SCST_PREPROCESS_STATUS_SUCCESS;
1140         struct iscsi_cmnd *req = (struct iscsi_cmnd*)
1141                 scst_cmd_get_tgt_priv(scst_cmd);
1142         struct iscsi_cmnd *c, *t;
1143
1144         TRACE_ENTRY();
1145
1146         EXTRACHECKS_BUG_ON(scst_cmd_atomic(scst_cmd));
1147
1148         if (scst_cmd_get_data_direction(scst_cmd) == SCST_DATA_READ) {
1149                 if (!(req->conn->ddigest_type & DIGEST_NONE))
1150                         scst_set_long_xmit(scst_cmd);
1151 #ifndef NET_PAGE_CALLBACKS_DEFINED
1152                 else if (cmnd_hdr(req)->data_length > 8*1024)
1153                         scst_set_long_xmit(scst_cmd);
1154 #endif
1155                 EXTRACHECKS_BUG_ON(!list_empty(&req->rx_ddigest_cmd_list));
1156                 goto out;
1157         }
1158
1159         /* If data digest isn't used this list will be empty */
1160         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
1161                                 rx_ddigest_cmd_list_entry) {
1162                 TRACE_DBG("Checking digest of RX ddigest cmd %p", c);
1163                 if (digest_rx_data(c) != 0) {
1164                         scst_set_cmd_error(scst_cmd,
1165                                 SCST_LOAD_SENSE(iscsi_sense_crc_error));
1166                         res = SCST_PREPROCESS_STATUS_ERROR_SENSE_SET;
1167                         /*
1168                          * The rest of rx_ddigest_cmd_list will be freed
1169                          * in req_cmnd_release()
1170                          */
1171                         goto out;
1172                 }
1173                 cmd_del_from_rx_ddigest_list(c);
1174                 cmnd_put(c);
1175         }
1176
1177 out:
1178         TRACE_EXIT_RES(res);
1179         return res;
1180 }
1181
1182 static int noop_out_start(struct iscsi_cmnd *cmnd)
1183 {
1184         struct iscsi_conn *conn = cmnd->conn;
1185         u32 size, tmp;
1186         int i, err = 0;
1187
1188         TRACE_DBG("%p", cmnd);
1189
1190         iscsi_extracheck_is_rd_thread(conn);
1191
1192         if (unlikely(cmnd_ttt(cmnd) != cpu_to_be32(ISCSI_RESERVED_TAG))) {
1193                 /*
1194                  * We don't request a NOP-Out by sending a NOP-In.
1195                  * See 10.18.2 in the draft 20.
1196                  */
1197                 PRINT_ERROR("Initiator sent command with not RESERVED tag and "
1198                         "TTT %x", cmnd_itt(cmnd));
1199                 err = -ISCSI_REASON_PROTOCOL_ERROR;
1200                 goto out;
1201         }
1202
1203         if (cmnd_itt(cmnd) == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1204                 if (unlikely(!(cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE)))
1205                         PRINT_ERROR("%s", "Initiator sent RESERVED tag for "
1206                                 "non-immediate command");
1207                 spin_lock(&conn->session->sn_lock);
1208                 __update_stat_sn(cmnd);
1209                 err = check_cmd_sn(cmnd);
1210                 spin_unlock(&conn->session->sn_lock);
1211                 if (unlikely(err))
1212                         goto out;
1213         } else if (unlikely((err = cmnd_insert_hash(cmnd)) < 0)) {
1214                 PRINT_ERROR("Can't insert in hash: ignore this request %x",
1215                         cmnd_itt(cmnd));
1216                 goto out;
1217         }
1218
1219         if ((size = cmnd->pdu.datasize)) {
1220                 size = (size + 3) & -4;
1221                 conn->read_msg.msg_iov = conn->read_iov;
1222                 if (cmnd->pdu.bhs.itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1223                         struct scatterlist *sg;
1224
1225                         /* ToDo: __GFP_NOFAIL ?? */
1226                         cmnd->sg = sg = scst_alloc(size,
1227                                 GFP_KERNEL|__GFP_NOFAIL, &cmnd->sg_cnt);
1228                         if (sg == NULL) {
1229                                 /* ToDo(); */
1230                         }
1231                         if (cmnd->sg_cnt > ISCSI_CONN_IOV_MAX) {
1232                                 /* ToDo(); */
1233                         }
1234                         cmnd->own_sg = 1;
1235                         cmnd->bufflen = size;
1236
1237                         for (i = 0; i < cmnd->sg_cnt; i++) {
1238                                 conn->read_iov[i].iov_base =
1239                                         page_address(sg_page(&sg[i]));
1240                                 tmp = min_t(u32, size, PAGE_SIZE);
1241                                 conn->read_iov[i].iov_len = tmp;
1242                                 conn->read_size += tmp;
1243                                 size -= tmp;
1244                         }
1245                 } else {
1246                         /*
1247                          * There are no problems with the safety from concurrent
1248                          * accesses to dummy_data, since for ISCSI_RESERVED_TAG
1249                          * the data only read and then discarded.
1250                          */
1251                         for (i = 0; i < ISCSI_CONN_IOV_MAX; i++) {
1252                                 conn->read_iov[i].iov_base = dummy_data;
1253                                 tmp = min_t(u32, size, sizeof(dummy_data));
1254                                 conn->read_iov[i].iov_len = tmp;
1255                                 conn->read_size += tmp;
1256                                 size -= tmp;
1257                         }
1258                 }
1259                 sBUG_ON(size == 0);
1260                 conn->read_msg.msg_iovlen = i;
1261                 TRACE_DBG("msg_iov=%p, msg_iovlen=%zd", conn->read_msg.msg_iov,
1262                         conn->read_msg.msg_iovlen);
1263         }
1264 out:
1265         return err;
1266 }
1267
1268 static inline u32 get_next_ttt(struct iscsi_conn *conn)
1269 {
1270         u32 ttt;
1271         struct iscsi_session *session = conn->session;
1272
1273         iscsi_extracheck_is_rd_thread(conn);
1274
1275         if (session->next_ttt == ISCSI_RESERVED_TAG)
1276                 session->next_ttt++;
1277         ttt = session->next_ttt++;
1278
1279         return cpu_to_be32(ttt);
1280 }
1281
1282 static int scsi_cmnd_start(struct iscsi_cmnd *req)
1283 {
1284         struct iscsi_conn *conn = req->conn;
1285         struct iscsi_session *session = conn->session;
1286         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1287         struct scst_cmd *scst_cmd;
1288         scst_data_direction dir;
1289         int res = 0;
1290
1291         TRACE_ENTRY();
1292
1293         TRACE_DBG("scsi command: %02x", req_hdr->scb[0]);
1294
1295         TRACE_DBG("Incrementing active_cmds (cmd %p, sess %p, "
1296                 "new value %d)", req, session,
1297                 atomic_read(&session->active_cmds)+1);
1298         atomic_inc(&session->active_cmds);
1299         req->dec_active_cmnds = 1;
1300
1301         scst_cmd = scst_rx_cmd(session->scst_sess,
1302                 (uint8_t*)&req_hdr->lun, sizeof(req_hdr->lun),
1303                 req_hdr->scb, sizeof(req_hdr->scb), SCST_NON_ATOMIC);
1304         if (scst_cmd == NULL) {
1305                 create_status_rsp(req, SAM_STAT_BUSY, NULL, 0);
1306                 cmnd_reject_scsi_cmd(req);
1307                 goto out;
1308         }
1309
1310         req->scst_cmd = scst_cmd;
1311         scst_cmd_set_tag(scst_cmd, req_hdr->itt);
1312         scst_cmd_set_tgt_priv(scst_cmd, req);
1313 #ifndef NET_PAGE_CALLBACKS_DEFINED
1314         scst_cmd_set_data_buf_tgt_alloc(scst_cmd);
1315 #endif
1316
1317         if (req_hdr->flags & ISCSI_CMD_READ)
1318                 dir = SCST_DATA_READ;
1319         else if (req_hdr->flags & ISCSI_CMD_WRITE)
1320                 dir = SCST_DATA_WRITE;
1321         else
1322                 dir = SCST_DATA_NONE;
1323         scst_cmd_set_expected(scst_cmd, dir, be32_to_cpu(req_hdr->data_length));
1324
1325         switch (req_hdr->flags & ISCSI_CMD_ATTR_MASK) {
1326         case ISCSI_CMD_SIMPLE:
1327                 scst_cmd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1328                 break;
1329         case ISCSI_CMD_HEAD_OF_QUEUE:
1330                 scst_cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1331                 break;
1332         case ISCSI_CMD_ORDERED:
1333                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1334                 break;
1335         case ISCSI_CMD_ACA:
1336                 scst_cmd->queue_type = SCST_CMD_QUEUE_ACA;
1337                 break;
1338         case ISCSI_CMD_UNTAGGED:
1339                 scst_cmd->queue_type = SCST_CMD_QUEUE_UNTAGGED;
1340                 break;
1341         default:
1342                 PRINT_ERROR("Unknown task code %x, use ORDERED instead",
1343                         req_hdr->flags & ISCSI_CMD_ATTR_MASK);
1344                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1345                 break;
1346         }
1347
1348         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
1349         scst_cmd_set_tgt_sn(scst_cmd, req_hdr->cmd_sn);
1350
1351         TRACE_DBG("START Command (tag %d, queue_type %d)",
1352                 req_hdr->itt, scst_cmd->queue_type);
1353         req->scst_state = ISCSI_CMD_STATE_RX_CMD;
1354         scst_cmd_init_stage1_done(scst_cmd, SCST_CONTEXT_DIRECT, 0);
1355
1356         wait_event(req->scst_waitQ, (req->scst_state != ISCSI_CMD_STATE_RX_CMD));
1357
1358         if (unlikely(req->scst_state != ISCSI_CMD_STATE_AFTER_PREPROC)) {
1359                 TRACE_DBG("req %p is in %x state", req, req->scst_state);
1360                 if (req->scst_state == ISCSI_CMD_STATE_PROCESSED) {
1361                         cmnd_reject_scsi_cmd(req);
1362                         goto out;
1363                 }
1364                 if (unlikely(req->tm_aborted)) {
1365                         TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
1366                                 req->scst_cmd);
1367                         cmnd_prepare_get_rejected_cmd_data(req);
1368                         goto out;
1369                 }
1370                 sBUG();
1371         }
1372
1373         dir = scst_cmd_get_data_direction(scst_cmd);
1374         if (dir != SCST_DATA_WRITE) {
1375                 if (unlikely(!(req_hdr->flags & ISCSI_CMD_FINAL) ||
1376                              req->pdu.datasize)) {
1377                         PRINT_ERROR("Unexpected unsolicited data (ITT %x "
1378                                 "CDB %x", cmnd_itt(req), req_hdr->scb[0]);
1379                         create_sense_rsp(req, ABORTED_COMMAND, 0xc, 0xc);
1380                         cmnd_reject_scsi_cmd(req);
1381                         goto out;
1382                 }
1383         }
1384
1385         if (dir == SCST_DATA_WRITE) {
1386                 req->is_unsolicited_data = !(req_hdr->flags & ISCSI_CMD_FINAL);
1387                 req->r2t_length = be32_to_cpu(req_hdr->data_length) - req->pdu.datasize;
1388                 if (req->r2t_length > 0)
1389                         req->data_waiting = 1;
1390         }
1391         req->target_task_tag = get_next_ttt(conn);
1392         req->sg = scst_cmd_get_sg(scst_cmd);
1393         req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
1394         req->bufflen = scst_cmd_get_bufflen(scst_cmd);
1395         if (unlikely(req->r2t_length > req->bufflen)) {
1396                 PRINT_ERROR("req->r2t_length %d > req->bufflen %d",
1397                         req->r2t_length, req->bufflen);
1398                 req->r2t_length = req->bufflen;
1399         }
1400
1401         TRACE_DBG("req=%p, dir=%d, is_unsolicited_data=%d, "
1402                 "r2t_length=%d, bufflen=%d", req, dir,
1403                 req->is_unsolicited_data, req->r2t_length, req->bufflen);
1404
1405         if (unlikely(!session->sess_param.immediate_data &&
1406                      req->pdu.datasize)) {
1407                 PRINT_ERROR("Initiator %s violated negotiated paremeters: "
1408                         "forbidden immediate data sent (ITT %x, op  %x)",
1409                         session->initiator_name, cmnd_itt(req), req_hdr->scb[0]);
1410                 res = -EINVAL;
1411                 goto out;
1412         }
1413
1414         if (unlikely(session->sess_param.initial_r2t &&
1415                      !(req_hdr->flags & ISCSI_CMD_FINAL))) {
1416                 PRINT_ERROR("Initiator %s violated negotiated paremeters: "
1417                         "initial R2T is required (ITT %x, op  %x)",
1418                         session->initiator_name, cmnd_itt(req), req_hdr->scb[0]);
1419                 res = -EINVAL;
1420                 goto out;
1421         }
1422
1423         if (req->pdu.datasize) {
1424                 if (unlikely(dir != SCST_DATA_WRITE)) {
1425                         PRINT_ERROR("pdu.datasize(%d) >0, but dir(%x) isn't WRITE",
1426                                 req->pdu.datasize, dir);
1427                         create_sense_rsp(req, ABORTED_COMMAND, 0xc, 0xc);
1428                         cmnd_reject_scsi_cmd(req);
1429                 } else
1430                         res = cmnd_prepare_recv_pdu(conn, req, 0, req->pdu.datasize);
1431         }
1432 out:
1433         /* Aborted commands will be freed in cmnd_rx_end() */
1434         TRACE_EXIT_RES(res);
1435         return res;
1436 }
1437
1438 static int data_out_start(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
1439 {
1440         struct iscsi_data_out_hdr *req_hdr = (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1441         struct iscsi_cmnd *orig_req = NULL;
1442         u32 offset = be32_to_cpu(req_hdr->buffer_offset);
1443         int res = 0;
1444
1445         TRACE_ENTRY();
1446
1447         /*
1448          * There is no race with send_r2t() and conn_abort(), since
1449          * all functions called from single read thread
1450          */
1451         iscsi_extracheck_is_rd_thread(cmnd->conn);
1452
1453         update_stat_sn(cmnd);
1454
1455         cmnd->cmd_req = orig_req = cmnd_find_hash(conn->session, req_hdr->itt,
1456                                         req_hdr->ttt);
1457         if (unlikely(orig_req == NULL)) {
1458                 /* It might happen if req was aborted and then freed */
1459                 TRACE(TRACE_MGMT_MINOR, "Unable to find scsi task %x %x",
1460                         cmnd_itt(cmnd), cmnd_ttt(cmnd));
1461                 goto out_reject;
1462         }
1463
1464         if (orig_req->is_unsolicited_data) {
1465                 if (unlikely(orig_req->r2t_length < cmnd->pdu.datasize)) {
1466                         PRINT_ERROR("Data size (%d) > R2T length (%d)",
1467                                 cmnd->pdu.datasize, orig_req->r2t_length);
1468                         mark_conn_closed(conn);
1469                         res = -EINVAL;
1470                         goto out;
1471                 }
1472                 orig_req->r2t_length -= cmnd->pdu.datasize;
1473         }
1474
1475         /* Check unsolicited burst data */
1476         if (unlikely((req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) &&
1477                      (orig_req->pdu.bhs.flags & ISCSI_FLG_FINAL))) {
1478                 PRINT_ERROR("Unexpected data from %x %x",
1479                         cmnd_itt(cmnd), cmnd_ttt(cmnd));
1480                 mark_conn_closed(conn);
1481                 res = -EINVAL;
1482                 goto out;
1483         }
1484
1485         TRACE_WRITE("%u %p %p %u %u", req_hdr->ttt, cmnd, orig_req,
1486                 offset, cmnd->pdu.datasize);
1487
1488         res = cmnd_prepare_recv_pdu(conn, orig_req, offset, cmnd->pdu.datasize);
1489
1490 out:
1491         TRACE_EXIT_RES(res);
1492         return res;
1493
1494 out_reject:
1495         sBUG_ON(cmnd->rejected);
1496         cmnd->rejected = 1;
1497         cmnd->reject_reason = ISCSI_REJECT_DATA;
1498         cmnd_prepare_get_rejected_cmd_data(cmnd);
1499         goto out;
1500 }
1501
1502 static void data_out_end(struct iscsi_cmnd *cmnd)
1503 {
1504         struct iscsi_data_out_hdr *req_hdr = (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1505         struct iscsi_cmnd *req;
1506
1507         sBUG_ON(cmnd == NULL);
1508         req = cmnd->cmd_req;
1509         sBUG_ON(req == NULL);
1510
1511         TRACE_DBG("cmnd %p, req %p", cmnd, req);
1512
1513         iscsi_extracheck_is_rd_thread(cmnd->conn);
1514
1515         if (!(cmnd->conn->ddigest_type & DIGEST_NONE) &&
1516             !cmnd->ddigest_checked) {
1517                 cmd_add_on_rx_ddigest_list(req, cmnd);
1518                 cmnd_get(cmnd);
1519         }
1520
1521         if (req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1522                 TRACE_DBG("ISCSI_RESERVED_TAG, FINAL %x",
1523                         req_hdr->flags & ISCSI_FLG_FINAL);
1524
1525                 if (req_hdr->flags & ISCSI_FLG_FINAL) {
1526                         req->is_unsolicited_data = 0;
1527                         if (req->pending)
1528                                 goto out_put;
1529                 } else
1530                         goto out_put;
1531         } else {
1532                 TRACE_DBG("FINAL %x, outstanding_r2t %d, r2t_length %d",
1533                         req_hdr->flags & ISCSI_FLG_FINAL,
1534                         req->outstanding_r2t, req->r2t_length);
1535
1536                 if (req_hdr->flags & ISCSI_FLG_FINAL) {
1537                         if (unlikely(req->is_unsolicited_data)) {
1538                                 PRINT_ERROR("Unexpected unsolicited data "
1539                                         "(r2t_length %u, outstanding_r2t %d)",
1540                                         req->r2t_length, req->is_unsolicited_data);
1541                                 mark_conn_closed(req->conn);
1542                                 goto out_put;
1543                         }
1544                         req->outstanding_r2t--;
1545                 } else
1546                         goto out_put;
1547         }
1548
1549         if (req->r2t_length != 0) {
1550                 if (!req->is_unsolicited_data)
1551                         send_r2t(req);
1552         } else
1553                 iscsi_restart_waiting_cmnd(req);
1554
1555 out_put:
1556         cmnd_put(cmnd);
1557         return;
1558 }
1559
1560 static void __cmnd_abort(struct iscsi_cmnd *cmnd)
1561 {
1562         /*
1563          * Here, if cmnd is data_waiting, we should iscsi_fail_waiting_cmnd()
1564          * it. But, since this function can be called from any thread, not only
1565          * from the read one, we at the moment can't do that, because of
1566          * absence of appropriate locking protection. But this isn't a stuff
1567          * for 0.9.6. So, currently a misbehaving initiator, not sending
1568          * data in R2T state for a sharing between targets device, for which
1569          * for some reason an aborting TM command, e.g. TARGET RESET, from
1570          * another initiator is issued, can block response for this TM command
1571          * virtually forever and by this make the issuing initiator eventually
1572          * put the device offline.
1573          *
1574          * ToDo in the next version, possibly a simple connection mutex, taken
1575          * by the read thread before starting any processing and by this
1576          * function, should be sufficient.
1577          */
1578
1579         TRACE_MGMT_DBG("Aborting cmd %p, scst_cmd %p (scst state %x, "
1580                 "ref_cnt %d, itt %x, sn %u, op %x, r2t_len %x, CDB op %x, "
1581                 "size to write %u, is_unsolicited_data %d, "
1582                 "outstanding_r2t %d, data_waiting %d, sess->exp_cmd_sn %u, "
1583                 "conn %p, rd_task %p)", cmnd, cmnd->scst_cmd, cmnd->scst_state,
1584                 atomic_read(&cmnd->ref_cnt), cmnd_itt(cmnd), cmnd->pdu.bhs.sn,
1585                 cmnd_opcode(cmnd), cmnd->r2t_length, cmnd_scsicode(cmnd),
1586                 cmnd_write_size(cmnd), cmnd->is_unsolicited_data,
1587                 cmnd->outstanding_r2t, cmnd->data_waiting,
1588                 cmnd->conn->session->exp_cmd_sn, cmnd->conn,
1589                 cmnd->conn->rd_task);
1590
1591 #ifdef NET_PAGE_CALLBACKS_DEFINED
1592         TRACE_MGMT_DBG("net_ref_cnt %d", atomic_read(&cmnd->net_ref_cnt));
1593 #endif
1594
1595         cmnd->tm_aborted = 1;
1596
1597         return;
1598 }
1599
1600 /* Must be called from the read thread */
1601 static int cmnd_abort(struct iscsi_cmnd *req)
1602 {
1603         struct iscsi_session *session = req->conn->session;
1604         struct iscsi_task_mgt_hdr *req_hdr =
1605                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1606         struct iscsi_cmnd *cmnd;
1607         int err;
1608
1609         req_hdr->ref_cmd_sn = be32_to_cpu(req_hdr->ref_cmd_sn);
1610
1611         if (after(req_hdr->ref_cmd_sn, req_hdr->cmd_sn)) {
1612                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) > CmdSN(%u)",
1613                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn);
1614                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1615                 goto out;
1616         }
1617
1618         if ((cmnd = cmnd_find_hash_get(session, req_hdr->rtt, ISCSI_RESERVED_TAG))) {
1619                 struct iscsi_conn *conn = cmnd->conn;
1620                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
1621
1622                 if (req_hdr->lun != hdr->lun) {
1623                          PRINT_ERROR("ABORT TASK: LUN mismatch: req LUN "
1624                                      "%Lx, cmd LUN %Lx, rtt %u", req_hdr->lun,
1625                                      hdr->lun, req_hdr->rtt);
1626                         err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1627                         goto out_put;
1628                 }
1629
1630                 if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
1631                         if (req_hdr->ref_cmd_sn != req_hdr->cmd_sn) {
1632                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != TM cmd "
1633                                         "CmdSN(%u) for immediate command %p",
1634                                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn,
1635                                         cmnd);
1636                                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1637                                 goto out_put;
1638                         }
1639                 } else {
1640                         if (req_hdr->ref_cmd_sn != hdr->cmd_sn) {
1641                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != "
1642                                         "CmdSN(%u) for command %p",
1643                                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn,
1644                                         cmnd);
1645                                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1646                                 goto out_put;
1647                         }
1648                 }
1649
1650                 if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
1651                     (req_hdr->cmd_sn == hdr->cmd_sn)) {
1652                         PRINT_ERROR("ABORT TASK: SN mismatch: req SN %x, "
1653                                 "cmd SN %x, rtt %u", req_hdr->cmd_sn,
1654                                 hdr->cmd_sn, req_hdr->rtt);
1655                         err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1656                         goto out_put;
1657                 }
1658
1659                 spin_lock_bh(&conn->cmd_list_lock);
1660                 __cmnd_abort(cmnd);
1661                 spin_unlock_bh(&conn->cmd_list_lock);
1662
1663                 cmnd_put(cmnd);
1664                 err = 0;
1665         } else {
1666                 TRACE_MGMT_DBG("cmd RTT %x not found", req_hdr->rtt);
1667                 err = ISCSI_RESPONSE_UNKNOWN_TASK;
1668         }
1669
1670 out:
1671         return err;
1672
1673 out_put:
1674         cmnd_put(cmnd);
1675         goto out;
1676 }
1677
1678 /* Must be called from the read thread */
1679 static int target_abort(struct iscsi_cmnd *req, int all)
1680 {
1681         struct iscsi_target *target = req->conn->session->target;
1682         struct iscsi_task_mgt_hdr *req_hdr =
1683                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1684         struct iscsi_session *session;
1685         struct iscsi_conn *conn;
1686         struct iscsi_cmnd *cmnd;
1687
1688         mutex_lock(&target->target_mutex);
1689
1690         list_for_each_entry(session, &target->session_list, session_list_entry) {
1691                 list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
1692                         spin_lock_bh(&conn->cmd_list_lock);
1693                         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1694                                 if (cmnd == req)
1695                                         continue;
1696                                 if (all)
1697                                         __cmnd_abort(cmnd);
1698                                 else if (req_hdr->lun == cmnd_hdr(cmnd)->lun)
1699                                         __cmnd_abort(cmnd);
1700                         }
1701                         spin_unlock_bh(&conn->cmd_list_lock);
1702                 }
1703         }
1704
1705         mutex_unlock(&target->target_mutex);
1706         return 0;
1707 }
1708
1709 /* Must be called from the read thread */
1710 static void task_set_abort(struct iscsi_cmnd *req)
1711 {
1712         struct iscsi_session *session = req->conn->session;
1713         struct iscsi_task_mgt_hdr *req_hdr =
1714                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1715         struct iscsi_target *target = session->target;
1716         struct iscsi_conn *conn;
1717         struct iscsi_cmnd *cmnd;
1718
1719         mutex_lock(&target->target_mutex);
1720
1721         list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
1722                 spin_lock_bh(&conn->cmd_list_lock);
1723                 list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1724                         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
1725                         if (cmnd == req)
1726                                 continue;
1727                         if (req_hdr->lun != hdr->lun)
1728                                 continue;
1729                         if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
1730                             req_hdr->cmd_sn == hdr->cmd_sn)
1731                                 continue;
1732                         __cmnd_abort(cmnd);
1733                 }
1734                 spin_unlock_bh(&conn->cmd_list_lock);
1735         }
1736
1737         mutex_unlock(&target->target_mutex);
1738         return;
1739 }
1740
1741 /* Must be called from the read thread */
1742 void conn_abort(struct iscsi_conn *conn)
1743 {
1744         struct iscsi_cmnd *cmnd;
1745
1746         TRACE_MGMT_DBG("Aborting conn %p", conn);
1747
1748         iscsi_extracheck_is_rd_thread(conn);
1749
1750         spin_lock_bh(&conn->cmd_list_lock);
1751 again:
1752         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1753                 __cmnd_abort(cmnd);
1754                 if (cmnd->data_waiting) {
1755                         if (!cmnd_get_check(cmnd)) {
1756                                 spin_unlock_bh(&conn->cmd_list_lock);
1757
1758                                 /* ToDo: this is racy for MC/S */
1759                                 TRACE_MGMT_DBG("Restarting data waiting cmd %p",
1760                                         cmnd);
1761                                 iscsi_fail_waiting_cmnd(cmnd);
1762
1763                                 cmnd_put(cmnd);
1764
1765                                 /*
1766                                  * We are in the read thread, so we may not
1767                                  * worry that after cmnd release conn gets
1768                                  * released as well.
1769                                  */
1770                                 spin_lock_bh(&conn->cmd_list_lock);
1771                                 goto again;
1772                         }
1773                 }
1774         }
1775         spin_unlock_bh(&conn->cmd_list_lock);
1776
1777         return;
1778 }
1779
1780 static void execute_task_management(struct iscsi_cmnd *req)
1781 {
1782         struct iscsi_conn *conn = req->conn;
1783         struct iscsi_session *sess = conn->session;
1784         struct iscsi_task_mgt_hdr *req_hdr =
1785                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1786         int rc, status, function = req_hdr->function & ISCSI_FUNCTION_MASK;
1787         struct scst_rx_mgmt_params params;
1788
1789         TRACE((function == ISCSI_FUNCTION_ABORT_TASK) ? TRACE_MGMT_MINOR : TRACE_MGMT,
1790                 "TM fn %d", function);
1791
1792         TRACE_MGMT_DBG("TM req %p, itt %x, rtt %x, sn %u, con %p", req,
1793                 cmnd_itt(req), req_hdr->rtt, req_hdr->cmd_sn, conn);
1794
1795         iscsi_extracheck_is_rd_thread(conn);
1796
1797         spin_lock(&sess->sn_lock);
1798         sess->tm_active++;
1799         sess->tm_sn = req_hdr->cmd_sn;
1800         if (sess->tm_rsp != NULL) {
1801                 struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
1802
1803                 TRACE(TRACE_MGMT_MINOR, "Dropping delayed TM rsp %p", tm_rsp);
1804
1805                 sess->tm_rsp = NULL;
1806                 sess->tm_active--;
1807
1808                 spin_unlock(&sess->sn_lock);
1809
1810                 sBUG_ON(sess->tm_active < 0);
1811
1812                 rsp_cmnd_release(tm_rsp);
1813         } else
1814                 spin_unlock(&sess->sn_lock);
1815
1816         memset(&params, 0, sizeof(params));
1817         params.atomic = SCST_NON_ATOMIC;
1818         params.tgt_priv = req;
1819
1820         if ((function != ISCSI_FUNCTION_ABORT_TASK) &&
1821             (req_hdr->rtt != ISCSI_RESERVED_TAG)) {
1822                 PRINT_ERROR("Invalid RTT %x (TM fn %x)", req_hdr->rtt,
1823                         function);
1824                 rc = -1;
1825                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1826                 goto reject;
1827         }
1828
1829         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
1830
1831         switch (function) {
1832         case ISCSI_FUNCTION_ABORT_TASK:
1833                 rc = -1;
1834                 status = cmnd_abort(req);
1835                 if (status == 0) {
1836                         params.fn = SCST_ABORT_TASK;
1837                         params.tag = req_hdr->rtt;
1838                         params.tag_set = 1;
1839                         params.lun = (uint8_t *)&req_hdr->lun;
1840                         params.lun_len = sizeof(req_hdr->lun);
1841                         params.lun_set = 1;
1842                         params.cmd_sn = req_hdr->cmd_sn;
1843                         params.cmd_sn_set = 1;
1844                         rc = scst_rx_mgmt_fn(conn->session->scst_sess,
1845                                 &params);
1846                         status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1847                 }
1848                 break;
1849         case ISCSI_FUNCTION_ABORT_TASK_SET:
1850                 task_set_abort(req);
1851                 params.fn = SCST_ABORT_TASK_SET;
1852                 params.lun = (uint8_t *)&req_hdr->lun;
1853                 params.lun_len = sizeof(req_hdr->lun);
1854                 params.lun_set = 1;
1855                 params.cmd_sn = req_hdr->cmd_sn;
1856                 params.cmd_sn_set = 1;
1857                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
1858                         &params);
1859                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1860                 break;
1861         case ISCSI_FUNCTION_CLEAR_TASK_SET:
1862                 task_set_abort(req);
1863                 params.fn = SCST_CLEAR_TASK_SET;
1864                 params.lun = (uint8_t *)&req_hdr->lun;
1865                 params.lun_len = sizeof(req_hdr->lun);
1866                 params.lun_set = 1;
1867                 params.cmd_sn = req_hdr->cmd_sn;
1868                 params.cmd_sn_set = 1;
1869                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
1870                         &params);
1871                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1872                 break;
1873         case ISCSI_FUNCTION_CLEAR_ACA:
1874                 params.fn = SCST_CLEAR_ACA;
1875                 params.lun = (uint8_t *)&req_hdr->lun;
1876                 params.lun_len = sizeof(req_hdr->lun);
1877                 params.lun_set = 1;
1878                 params.cmd_sn = req_hdr->cmd_sn;
1879                 params.cmd_sn_set = 1;
1880                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
1881                         &params);
1882                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1883                 break;
1884         case ISCSI_FUNCTION_TARGET_COLD_RESET:
1885         case ISCSI_FUNCTION_TARGET_WARM_RESET:
1886                 target_abort(req, 1);
1887                 params.fn = SCST_TARGET_RESET;
1888                 params.cmd_sn = req_hdr->cmd_sn;
1889                 params.cmd_sn_set = 1;
1890                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
1891                         &params);
1892                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1893                 break;
1894         case ISCSI_FUNCTION_LOGICAL_UNIT_RESET:
1895                 target_abort(req, 0);
1896                 params.fn = SCST_LUN_RESET;
1897                 params.lun = (uint8_t *)&req_hdr->lun;
1898                 params.lun_len = sizeof(req_hdr->lun);
1899                 params.lun_set = 1;
1900                 params.cmd_sn = req_hdr->cmd_sn;
1901                 params.cmd_sn_set = 1;
1902                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
1903                         &params);
1904                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1905                 break;
1906         case ISCSI_FUNCTION_TASK_REASSIGN:
1907                 rc = -1;
1908                 status = ISCSI_RESPONSE_FUNCTION_UNSUPPORTED;
1909                 break;
1910         default:
1911                 PRINT_ERROR("Unknown TM function %d", function);
1912                 rc = -1;
1913                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1914                 break;
1915         }
1916
1917 reject:
1918         if (rc != 0)
1919                 iscsi_send_task_mgmt_resp(req, status);
1920
1921         return;
1922 }
1923
1924 static void noop_out_exec(struct iscsi_cmnd *req)
1925 {
1926         struct iscsi_cmnd *rsp;
1927         struct iscsi_nop_in_hdr *rsp_hdr;
1928
1929         TRACE_DBG("%p", req);
1930
1931         if (cmnd_itt(req) != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1932                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
1933
1934                 rsp_hdr = (struct iscsi_nop_in_hdr *)&rsp->pdu.bhs;
1935                 rsp_hdr->opcode = ISCSI_OP_NOOP_IN;
1936                 rsp_hdr->flags = ISCSI_FLG_FINAL;
1937                 rsp_hdr->itt = req->pdu.bhs.itt;
1938                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
1939
1940                 if (req->pdu.datasize)
1941                         sBUG_ON(req->sg == NULL);
1942                 else
1943                         sBUG_ON(req->sg != NULL);
1944
1945                 if (req->sg) {
1946                         rsp->sg = req->sg;
1947                         rsp->sg_cnt = req->sg_cnt;
1948                         rsp->bufflen = req->bufflen;
1949                 }
1950
1951                 sBUG_ON(get_pgcnt(req->pdu.datasize, 0) > ISCSI_CONN_IOV_MAX);
1952
1953                 rsp->pdu.datasize = req->pdu.datasize;
1954                 iscsi_cmnd_init_write(rsp,
1955                         ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
1956                 req_cmnd_release(req);
1957         } else
1958                 cmnd_put(req);
1959 }
1960
1961 static void logout_exec(struct iscsi_cmnd *req)
1962 {
1963         struct iscsi_logout_req_hdr *req_hdr;
1964         struct iscsi_cmnd *rsp;
1965         struct iscsi_logout_rsp_hdr *rsp_hdr;
1966
1967         PRINT_INFO("Logout received from initiator %s",
1968                 req->conn->session->initiator_name);
1969         TRACE_DBG("%p", req);
1970
1971         req_hdr = (struct iscsi_logout_req_hdr *)&req->pdu.bhs;
1972         rsp = iscsi_cmnd_create_rsp_cmnd(req);
1973         rsp_hdr = (struct iscsi_logout_rsp_hdr *)&rsp->pdu.bhs;
1974         rsp_hdr->opcode = ISCSI_OP_LOGOUT_RSP;
1975         rsp_hdr->flags = ISCSI_FLG_FINAL;
1976         rsp_hdr->itt = req_hdr->itt;
1977         rsp->should_close_conn = 1;
1978         iscsi_cmnd_init_write(rsp,
1979                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
1980         req_cmnd_release(req);
1981 }
1982
1983 static void iscsi_cmnd_exec(struct iscsi_cmnd *cmnd)
1984 {
1985         TRACE_ENTRY();
1986
1987         TRACE_DBG("%p,%x,%u", cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn);
1988
1989         iscsi_extracheck_is_rd_thread(cmnd->conn);
1990
1991         if (unlikely(cmnd->tm_aborted)) {
1992                 TRACE_MGMT_DBG("cmnd %p (scst_cmd %p) aborted", cmnd,
1993                         cmnd->scst_cmd);
1994                 req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
1995                 goto out;
1996         }
1997
1998         if (unlikely(cmnd->rejected))
1999                 goto out_rejected;
2000
2001         switch (cmnd_opcode(cmnd)) {
2002         case ISCSI_OP_SCSI_CMD:
2003                 if (cmnd->r2t_length != 0) {
2004                         if (!cmnd->is_unsolicited_data) {
2005                                 send_r2t(cmnd);
2006                                 break;
2007                         }
2008                 } else
2009                         iscsi_restart_cmnd(cmnd);
2010                 break;
2011         case ISCSI_OP_NOOP_OUT:
2012                 noop_out_exec(cmnd);
2013                 break;
2014         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2015                 execute_task_management(cmnd);
2016                 break;
2017         case ISCSI_OP_LOGOUT_CMD:
2018                 logout_exec(cmnd);
2019                 break;
2020         default:
2021                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2022                 req_cmnd_release(cmnd);
2023                 break;
2024         }
2025 out:
2026         TRACE_EXIT();
2027         return;
2028
2029 out_rejected:
2030         TRACE_MGMT_DBG("Rejected cmd %p (reason %d)", cmnd,
2031                 cmnd->reject_reason);
2032         switch (cmnd->reject_reason) {
2033         default:
2034                 PRINT_ERROR("Unexpected reject reason %d", cmnd->reject_reason);
2035                 /* go through */
2036         case ISCSI_REJECT_SCSI_CMD:
2037                 req_cmnd_release(cmnd);
2038                 break;
2039         }
2040         goto out;
2041 }
2042
2043 static void __cmnd_send_pdu(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd,
2044         u32 offset, u32 size)
2045 {
2046         TRACE_DBG("%p %u,%u,%u", cmnd, offset, size, cmnd->bufflen);
2047
2048         iscsi_extracheck_is_wr_thread(conn);
2049
2050         sBUG_ON(offset > cmnd->bufflen);
2051         sBUG_ON(offset + size > cmnd->bufflen);
2052
2053         conn->write_offset = offset;
2054         conn->write_size += size;
2055 }
2056
2057 static void cmnd_send_pdu(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
2058 {
2059         u32 size;
2060
2061         if (!cmnd->pdu.datasize)
2062                 return;
2063
2064         size = (cmnd->pdu.datasize + 3) & -4;
2065         sBUG_ON(cmnd->sg == NULL);
2066         sBUG_ON(cmnd->bufflen != size);
2067         __cmnd_send_pdu(conn, cmnd, 0, size);
2068 }
2069
2070 static void set_cork(struct socket *sock, int on)
2071 {
2072         int opt = on;
2073         mm_segment_t oldfs;
2074
2075         oldfs = get_fs();
2076         set_fs(get_ds());
2077         sock->ops->setsockopt(sock, SOL_TCP, TCP_CORK, (void *)&opt, sizeof(opt));
2078         set_fs(oldfs);
2079 }
2080
2081 void cmnd_tx_start(struct iscsi_cmnd *cmnd)
2082 {
2083         struct iscsi_conn *conn = cmnd->conn;
2084
2085         TRACE_DBG("%p:%p:%x", conn, cmnd, cmnd_opcode(cmnd));
2086         iscsi_cmnd_set_length(&cmnd->pdu);
2087
2088         iscsi_extracheck_is_wr_thread(conn);
2089
2090         set_cork(conn->sock, 1);
2091
2092         conn->write_iop = conn->write_iov;
2093         conn->write_iop->iov_base = &cmnd->pdu.bhs;
2094         conn->write_iop->iov_len = sizeof(cmnd->pdu.bhs);
2095         conn->write_iop_used = 1;
2096         conn->write_size = sizeof(cmnd->pdu.bhs);
2097
2098         switch (cmnd_opcode(cmnd)) {
2099         case ISCSI_OP_NOOP_IN:
2100                 cmnd_set_sn(cmnd, 1);
2101                 cmnd_send_pdu(conn, cmnd);
2102                 break;
2103         case ISCSI_OP_SCSI_RSP:
2104                 cmnd_set_sn(cmnd, 1);
2105                 cmnd_send_pdu(conn, cmnd);
2106                 break;
2107         case ISCSI_OP_SCSI_TASK_MGT_RSP:
2108                 cmnd_set_sn(cmnd, 1);
2109                 break;
2110         case ISCSI_OP_TEXT_RSP:
2111                 cmnd_set_sn(cmnd, 1);
2112                 break;
2113         case ISCSI_OP_SCSI_DATA_IN:
2114         {
2115                 struct iscsi_data_in_hdr *rsp = (struct iscsi_data_in_hdr *)&cmnd->pdu.bhs;
2116                 u32 offset = cpu_to_be32(rsp->buffer_offset);
2117
2118                 cmnd_set_sn(cmnd, (rsp->flags & ISCSI_FLG_FINAL) ? 1 : 0);
2119                 __cmnd_send_pdu(conn, cmnd, offset, cmnd->pdu.datasize);
2120                 break;
2121         }
2122         case ISCSI_OP_LOGOUT_RSP:
2123                 cmnd_set_sn(cmnd, 1);
2124                 break;
2125         case ISCSI_OP_R2T:
2126                 cmnd->pdu.bhs.sn = cmnd_set_sn(cmnd, 0);
2127                 break;
2128         case ISCSI_OP_ASYNC_MSG:
2129                 cmnd_set_sn(cmnd, 1);
2130                 break;
2131         case ISCSI_OP_REJECT:
2132                 cmnd_set_sn(cmnd, 1);
2133                 cmnd_send_pdu(conn, cmnd);
2134                 break;
2135         default:
2136                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2137                 break;
2138         }
2139
2140         // move this?
2141         conn->write_size = (conn->write_size + 3) & -4;
2142         iscsi_dump_pdu(&cmnd->pdu);
2143 }
2144
2145 void cmnd_tx_end(struct iscsi_cmnd *cmnd)
2146 {
2147         struct iscsi_conn *conn = cmnd->conn;
2148
2149         TRACE_DBG("%p:%x (should_close_conn %d)", cmnd, cmnd_opcode(cmnd),
2150                 cmnd->should_close_conn);
2151
2152         switch (cmnd_opcode(cmnd)) {
2153         case ISCSI_OP_NOOP_IN:
2154         case ISCSI_OP_SCSI_RSP:
2155         case ISCSI_OP_SCSI_TASK_MGT_RSP:
2156         case ISCSI_OP_TEXT_RSP:
2157         case ISCSI_OP_R2T:
2158         case ISCSI_OP_ASYNC_MSG:
2159         case ISCSI_OP_REJECT:
2160         case ISCSI_OP_SCSI_DATA_IN:
2161         case ISCSI_OP_LOGOUT_RSP:
2162                 break;
2163         default:
2164                 PRINT_CRIT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2165                 sBUG();
2166                 break;
2167         }
2168
2169         if (cmnd->should_close_conn) {
2170                 PRINT_INFO("Closing connection at initiator %s request",
2171                         conn->session->initiator_name);
2172                 mark_conn_closed(conn);
2173         }
2174
2175         set_cork(cmnd->conn->sock, 0);
2176 }
2177
2178 /*
2179  * Push the command for execution. This functions reorders the commands.
2180  * Called from the read thread.
2181  */
2182 static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd)
2183 {
2184         struct iscsi_session *session = cmnd->conn->session;
2185         struct list_head *entry;
2186         u32 cmd_sn;
2187
2188         TRACE_DBG("%p:%x %u,%u",
2189                 cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn, session->exp_cmd_sn);
2190
2191         iscsi_extracheck_is_rd_thread(cmnd->conn);
2192
2193         sBUG_ON(cmnd->parent_req != NULL);
2194
2195         if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
2196                 TRACE_DBG("Immediate cmd %p (cmd_sn %u)", cmnd,
2197                         cmnd->pdu.bhs.sn);
2198                 iscsi_cmnd_exec(cmnd);
2199                 goto out;
2200         }
2201
2202         spin_lock(&session->sn_lock);
2203
2204         cmd_sn = cmnd->pdu.bhs.sn;
2205         if (cmd_sn == session->exp_cmd_sn) {
2206                 while (1) {
2207                         session->exp_cmd_sn = ++cmd_sn;
2208
2209                         if (unlikely(session->tm_active > 0)) {
2210                                 if (before(cmd_sn, session->tm_sn)) {
2211                                         struct iscsi_conn *conn = cmnd->conn;
2212
2213                                         spin_unlock(&session->sn_lock);
2214
2215                                         spin_lock_bh(&conn->cmd_list_lock);
2216                                         __cmnd_abort(cmnd);
2217                                         spin_unlock_bh(&conn->cmd_list_lock);
2218
2219                                         spin_lock(&session->sn_lock);
2220                                 }
2221                                 iscsi_check_send_delayed_tm_resp(session);
2222                         }
2223
2224                         spin_unlock(&session->sn_lock);
2225
2226                         iscsi_cmnd_exec(cmnd);
2227
2228                         if (list_empty(&session->pending_list))
2229                                 break;
2230                         cmnd = list_entry(session->pending_list.next, struct iscsi_cmnd,
2231                                                 pending_list_entry);
2232                         if (cmnd->pdu.bhs.sn != cmd_sn)
2233                                 break;
2234
2235                         list_del(&cmnd->pending_list_entry);
2236                         cmnd->pending = 0;
2237
2238                         TRACE_DBG("Processing pending cmd %p (cmd_sn %u)",
2239                                 cmnd, cmd_sn);
2240
2241                         spin_lock(&session->sn_lock);
2242                 }
2243         } else {
2244                 int drop = 0;
2245
2246                 TRACE_DBG("Pending cmd %p (cmd_sn %u, exp_cmd_sn %u)",
2247                         cmnd, cmd_sn, session->exp_cmd_sn);
2248
2249                 /*
2250                  * iSCSI RFC 3720: "The target MUST silently ignore any
2251                  * non-immediate command outside of [from ExpCmdSN to MaxCmdSN
2252                  * inclusive] range". But we won't honor the MaxCmdSN
2253                  * requirement, because, since we adjust MaxCmdSN from the
2254                  * separate write thread, rarery it is possible that initiator
2255                  * can legally send command with CmdSN>MaxSN. But it won't
2256                  * hurt anything, in the worst case it will lead to
2257                  * additional QUEUE FULL status.
2258                  */
2259
2260                 if (unlikely(before(cmd_sn, session->exp_cmd_sn))) {
2261                         PRINT_ERROR("Unexpected cmd_sn (%u,%u)", cmd_sn,
2262                                 session->exp_cmd_sn);
2263                         drop = 1;
2264                 }
2265
2266 #if 0
2267                 if (unlikely(after(cmd_sn, session->exp_cmd_sn +
2268                                         iscsi_get_allowed_cmds(session)))) {
2269                         TRACE_MGMT_DBG("Too large cmd_sn %u (exp_cmd_sn %u, "
2270                                 "max_sn %u)", cmd_sn, session->exp_cmd_sn,
2271                                 iscsi_get_allowed_cmds(session));
2272                 }
2273 #endif
2274
2275                 spin_unlock(&session->sn_lock);
2276
2277                 if (unlikely(drop)) {
2278                         req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
2279                         goto out;
2280                 }
2281
2282                 if (unlikely(cmnd->tm_aborted)) {
2283                         struct iscsi_cmnd *tm_clone;
2284
2285                         TRACE_MGMT_DBG("Pending aborted cmnd %p, creating TM "
2286                                 "clone (scst cmd %p, state %d)", cmnd,
2287                                 cmnd->scst_cmd, cmnd->scst_state);
2288
2289                         tm_clone = cmnd_alloc(cmnd->conn, NULL);
2290                         if (tm_clone != NULL) {
2291                                 tm_clone->tm_aborted = 1;
2292                                 tm_clone->pdu = cmnd->pdu;
2293
2294                                 TRACE_MGMT_DBG("TM clone %p created", tm_clone);
2295
2296                                 iscsi_cmnd_exec(cmnd);
2297                                 cmnd = tm_clone;
2298                         } else
2299                                 PRINT_ERROR("%s", "Unable to create TM clone");
2300                 }
2301
2302                 list_for_each(entry, &session->pending_list) {
2303                         struct iscsi_cmnd *tmp = list_entry(entry, struct iscsi_cmnd,
2304                                                         pending_list_entry);
2305                         if (before(cmd_sn, tmp->pdu.bhs.sn))
2306                                 break;
2307                 }
2308
2309                 list_add_tail(&cmnd->pending_list_entry, entry);
2310                 cmnd->pending = 1;
2311         }
2312 out:
2313         return;
2314 }
2315
2316 static int check_segment_length(struct iscsi_cmnd *cmnd)
2317 {
2318         struct iscsi_conn *conn = cmnd->conn;
2319         struct iscsi_session *session = conn->session;
2320
2321         if (unlikely(cmnd->pdu.datasize > session->sess_param.max_recv_data_length)) {
2322                 PRINT_ERROR("Initiator %s violated negotiated parameters: "
2323                         "data too long (ITT %x, datasize %u, "
2324                         "max_recv_data_length %u", session->initiator_name,
2325                         cmnd_itt(cmnd), cmnd->pdu.datasize,
2326                         session->sess_param.max_recv_data_length);
2327                 mark_conn_closed(conn);
2328                 return -EINVAL;
2329         }
2330         return 0;
2331 }
2332
2333 int cmnd_rx_start(struct iscsi_cmnd *cmnd)
2334 {
2335         struct iscsi_conn *conn = cmnd->conn;
2336         int res, rc;
2337
2338         iscsi_dump_pdu(&cmnd->pdu);
2339
2340         res = check_segment_length(cmnd);
2341         if (res != 0)
2342                 goto out;
2343
2344         switch (cmnd_opcode(cmnd)) {
2345         case ISCSI_OP_NOOP_OUT:
2346                 rc = noop_out_start(cmnd);
2347                 break;
2348         case ISCSI_OP_SCSI_CMD:
2349                 rc = cmnd_insert_hash(cmnd);
2350                 if (likely(rc == 0)) {
2351                         res = scsi_cmnd_start(cmnd);
2352                         if (unlikely(res != 0))
2353                                 goto out;
2354                 }
2355                 break;
2356         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2357                 rc = cmnd_insert_hash(cmnd);
2358                 break;
2359         case ISCSI_OP_SCSI_DATA_OUT:
2360                 res = data_out_start(conn, cmnd);
2361                 rc = 0; /* to avoid compiler warning */
2362                 if (unlikely(res != 0))
2363                         goto out;
2364                 break;
2365         case ISCSI_OP_LOGOUT_CMD:
2366                 rc = cmnd_insert_hash(cmnd);
2367                 break;
2368         case ISCSI_OP_TEXT_CMD:
2369         case ISCSI_OP_SNACK_CMD:
2370                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2371                 break;
2372         default:
2373                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2374                 break;
2375         }
2376
2377         if (unlikely(rc < 0)) {
2378                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
2379                 PRINT_ERROR("Error %d (iSCSI opcode %x, ITT %x, op %x)", rc,
2380                         cmnd_opcode(cmnd), cmnd_itt(cmnd),
2381                         (cmnd_opcode(cmnd) == ISCSI_OP_SCSI_CMD ?
2382                                 hdr->scb[0] : -1));
2383                 iscsi_cmnd_reject(cmnd, -rc);
2384         }
2385
2386 out:
2387         TRACE_EXIT_RES(res);
2388         return res;
2389 }
2390
2391 void cmnd_rx_end(struct iscsi_cmnd *cmnd)
2392 {
2393         TRACE_ENTRY();
2394
2395         TRACE_DBG("%p:%x", cmnd, cmnd_opcode(cmnd));
2396
2397         if (unlikely(cmnd->rejected))
2398                 goto out_rejected;
2399
2400 cont:
2401         switch (cmnd_opcode(cmnd)) {
2402         case ISCSI_OP_SCSI_CMD:
2403         case ISCSI_OP_NOOP_OUT:
2404         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2405         case ISCSI_OP_LOGOUT_CMD:
2406                 iscsi_session_push_cmnd(cmnd);
2407                 break;
2408         case ISCSI_OP_SCSI_DATA_OUT:
2409                 data_out_end(cmnd);
2410                 break;
2411         default:
2412                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2413                 req_cmnd_release(cmnd);
2414                 break;
2415         }
2416
2417 out:
2418         TRACE_EXIT();
2419         return;
2420
2421 out_rejected:
2422         switch (cmnd->reject_reason) {
2423         default:
2424                 PRINT_ERROR("Unexpected reject reason %d", cmnd->reject_reason);
2425                 /* go through */
2426         case ISCSI_REJECT_CMD:
2427         case ISCSI_REJECT_DATA:
2428                 req_cmnd_release(cmnd);
2429                 break;
2430         case ISCSI_REJECT_SCSI_CMD:
2431                 goto cont;
2432         }
2433         goto out;
2434 }
2435
2436 #ifndef NET_PAGE_CALLBACKS_DEFINED
2437 static int iscsi_alloc_data_buf(struct scst_cmd *cmd)
2438 {
2439         if (scst_cmd_get_data_direction(cmd) == SCST_DATA_READ) {
2440                 /*
2441                  * sock->ops->sendpage() is async zero copy operation,
2442                  * so we must be sure not to free and reuse
2443                  * the command's buffer before the sending was completed
2444                  * by the network layers. It is possible only if we
2445                  * don't use SGV cache.
2446                  */
2447                 scst_cmd_set_no_sgv(cmd);
2448         }
2449         return 1;
2450 }
2451 #endif
2452
2453 static inline void iscsi_set_state_wake_up(struct iscsi_cmnd *req,
2454         int new_state)
2455 {
2456         /*
2457          * We use wait_event() to wait for the state change, but it checks its
2458          * condition without any protection, so without cmnd_get() it is
2459          * possible that req will die "immediately" after the state assignment
2460          * and wake_up() will operate on dead data.
2461          */
2462         cmnd_get_ordered(req);
2463         req->scst_state = new_state;
2464         wake_up(&req->scst_waitQ);
2465         cmnd_put(req);
2466         return;
2467 }
2468
2469 static void iscsi_preprocessing_done(struct scst_cmd *scst_cmd)
2470 {
2471         struct iscsi_cmnd *req = (struct iscsi_cmnd*)
2472                                 scst_cmd_get_tgt_priv(scst_cmd);
2473
2474         TRACE_DBG("req %p", req);
2475
2476         EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_RX_CMD);
2477
2478         iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_AFTER_PREPROC);
2479         return;
2480 }
2481
2482 /*
2483  * No locks.
2484  *
2485  * IMPORTANT! Connection conn must be protected by additional conn_get()
2486  * upon entrance in this function, because otherwise it could be destroyed
2487  * inside as a result of iscsi_send(), which releases sent commands.
2488  */
2489 static void iscsi_try_local_processing(struct iscsi_conn *conn,
2490         bool single_only)
2491 {
2492         int local;
2493
2494         TRACE_ENTRY();
2495
2496         spin_lock_bh(&iscsi_wr_lock);
2497         switch (conn->wr_state) {
2498         case ISCSI_CONN_WR_STATE_IN_LIST:
2499                 list_del(&conn->wr_list_entry);
2500                 /* go through */
2501         case ISCSI_CONN_WR_STATE_IDLE:
2502 #ifdef EXTRACHECKS
2503                 conn->wr_task = current;
2504 #endif
2505                 conn->wr_state = ISCSI_CONN_WR_STATE_PROCESSING;
2506                 conn->wr_space_ready = 0;
2507                 local = 1;
2508                 break;
2509         default:
2510                 local = 0;
2511                 break;
2512         }
2513         spin_unlock_bh(&iscsi_wr_lock);
2514
2515         if (local) {
2516                 int rc = 1;
2517                 while (test_write_ready(conn)) {
2518                         rc = iscsi_send(conn);
2519                         if ((rc <= 0) || single_only) {
2520                                 break;
2521                         }
2522                 }
2523
2524                 spin_lock_bh(&iscsi_wr_lock);
2525 #ifdef EXTRACHECKS
2526                 conn->wr_task = NULL;
2527 #endif
2528                 if ((rc <= 0) || test_write_ready(conn)) {
2529                         list_add_tail(&conn->wr_list_entry, &iscsi_wr_list);
2530                         conn->wr_state = ISCSI_CONN_WR_STATE_IN_LIST;
2531                         wake_up(&iscsi_wr_waitQ);
2532                 } else
2533                         conn->wr_state = ISCSI_CONN_WR_STATE_IDLE;
2534                 spin_unlock_bh(&iscsi_wr_lock);
2535         }
2536
2537         TRACE_EXIT();
2538         return;
2539 }
2540
2541 static int iscsi_xmit_response(struct scst_cmd *scst_cmd)
2542 {
2543         int is_send_status = scst_cmd_get_is_send_status(scst_cmd);
2544         struct iscsi_cmnd *req = (struct iscsi_cmnd*)
2545                                         scst_cmd_get_tgt_priv(scst_cmd);
2546         struct iscsi_conn *conn = req->conn;
2547         int status = scst_cmd_get_status(scst_cmd);
2548         u8 *sense = scst_cmd_get_sense_buffer(scst_cmd);
2549         int sense_len = scst_cmd_get_sense_buffer_len(scst_cmd);
2550         int old_state = req->scst_state;
2551 #if 0 /* temp. ToDo */
2552         bool single_only = !scst_get_long_xmit(scst_cmd);
2553 #else
2554         bool single_only = 0;
2555 #endif
2556
2557         if (scst_cmd_atomic(scst_cmd))
2558                 return SCST_TGT_RES_NEED_THREAD_CTX;
2559
2560         scst_cmd_set_tgt_priv(scst_cmd, NULL);
2561
2562         req->tm_aborted |= scst_cmd_aborted(scst_cmd) ? 1 : 0;
2563         if (unlikely(req->tm_aborted)) {
2564                 TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
2565                         req->scst_cmd);
2566
2567                 scst_set_delivery_status(req->scst_cmd,
2568                         SCST_CMD_DELIVERY_ABORTED);
2569
2570                 if (old_state == ISCSI_CMD_STATE_RESTARTED) {
2571                         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
2572                         req_cmnd_release_force(req, ISCSI_FORCE_RELEASE_WRITE);
2573                 } else
2574                         iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_PROCESSED);
2575
2576                 goto out;
2577         }
2578
2579         if (unlikely(old_state != ISCSI_CMD_STATE_RESTARTED)) {
2580                 TRACE_DBG("req %p on %d state", req, old_state);
2581
2582                 create_status_rsp(req, status, sense, sense_len);
2583
2584                 switch (old_state) {
2585                 case ISCSI_CMD_STATE_RX_CMD:
2586                 case ISCSI_CMD_STATE_AFTER_PREPROC:
2587                         break;
2588                 default:
2589                         sBUG();
2590                 }
2591
2592                 iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_PROCESSED);
2593                 goto out;
2594         }
2595
2596         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
2597
2598         req->bufflen = scst_cmd_get_resp_data_len(scst_cmd);
2599         req->sg = scst_cmd_get_sg(scst_cmd);
2600         req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
2601
2602         TRACE_DBG("req %p, is_send_status=%x, req->bufflen=%d, req->sg=%p, "
2603                 "req->sg_cnt %d", req, is_send_status, req->bufflen, req->sg,
2604                 req->sg_cnt);
2605
2606         if (unlikely((req->bufflen != 0) && !is_send_status)) {
2607                 PRINT_CRIT_ERROR("%s", "Sending DATA without STATUS is unsupported");
2608                 scst_set_cmd_error(scst_cmd,
2609                         SCST_LOAD_SENSE(scst_sense_hardw_error));
2610                 sBUG();
2611         }
2612
2613         if (req->bufflen != 0) {
2614                 /*
2615                  * Check above makes sure that is_send_status is set,
2616                  * so status is valid here, but in future that could change.
2617                  * ToDo
2618                  */
2619                 if (status != SAM_STAT_CHECK_CONDITION) {
2620                         send_data_rsp(req, status, is_send_status);
2621                 } else {
2622                         struct iscsi_cmnd *rsp;
2623                         struct iscsi_scsi_rsp_hdr *rsp_hdr;
2624                         int resid;
2625                         send_data_rsp(req, 0, 0);
2626                         if (is_send_status) {
2627                                 rsp = create_status_rsp(req, status, sense,
2628                                         sense_len);
2629                                 rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
2630                                 resid = cmnd_read_size(req) - req->bufflen;
2631                                 if (resid > 0) {
2632                                         rsp_hdr->flags |=
2633                                                 ISCSI_FLG_RESIDUAL_UNDERFLOW;
2634                                         rsp_hdr->residual_count = cpu_to_be32(resid);
2635                                 } else if (resid < 0) {
2636                                         rsp_hdr->flags |=
2637                                                 ISCSI_FLG_RESIDUAL_OVERFLOW;
2638                                         rsp_hdr->residual_count = cpu_to_be32(-resid);
2639                                 }
2640                                 iscsi_cmnd_init_write(rsp,
2641                                         ISCSI_INIT_WRITE_REMOVE_HASH);
2642                         }
2643                 }
2644         } else if (is_send_status) {
2645                 struct iscsi_cmnd *rsp;
2646                 struct iscsi_scsi_rsp_hdr *rsp_hdr;
2647                 u32 resid;
2648                 rsp = create_status_rsp(req, status, sense, sense_len);
2649                 rsp_hdr = (struct iscsi_scsi_rsp_hdr *) &rsp->pdu.bhs;
2650                 resid = cmnd_read_size(req);
2651                 if (resid != 0) {
2652                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
2653                         rsp_hdr->residual_count = cpu_to_be32(resid);
2654                 }
2655                 iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH);
2656         }
2657 #ifdef EXTRACHECKS
2658         else
2659                 sBUG();
2660 #endif
2661
2662         conn_get_ordered(conn);
2663         req_cmnd_release(req);
2664         iscsi_try_local_processing(conn, single_only);
2665         conn_put(conn);
2666
2667 out:
2668         return SCST_TGT_RES_SUCCESS;
2669 }
2670
2671 /* Called under sn_lock */
2672 static bool iscsi_is_delay_tm_resp(struct iscsi_cmnd *rsp)
2673 {
2674         bool res = 0;
2675         struct iscsi_task_mgt_hdr *req_hdr =
2676                 (struct iscsi_task_mgt_hdr *)&rsp->parent_req->pdu.bhs;
2677         int function = req_hdr->function & ISCSI_FUNCTION_MASK;
2678         struct iscsi_session *sess = rsp->conn->session;
2679
2680         TRACE_ENTRY();
2681
2682         /* This should be checked for immediate TM commands as well */
2683
2684         switch (function) {
2685         default:
2686                 if (before(sess->exp_cmd_sn, req_hdr->cmd_sn))
2687                         res = 1;
2688                 break;
2689         }
2690
2691         TRACE_EXIT_RES(res);
2692         return res;
2693 }
2694
2695 /* Called under sn_lock, but might drop it inside, then reaquire */
2696 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess)
2697 {
2698         struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
2699
2700         TRACE_ENTRY();
2701
2702         if (tm_rsp == NULL)
2703                 goto out;
2704
2705         if (iscsi_is_delay_tm_resp(tm_rsp))
2706                 goto out;
2707
2708         TRACE(TRACE_MGMT_MINOR, "Sending delayed rsp %p", tm_rsp);
2709
2710         sess->tm_rsp = NULL;
2711         sess->tm_active--;
2712
2713         spin_unlock(&sess->sn_lock);
2714
2715         sBUG_ON(sess->tm_active < 0);
2716
2717         iscsi_cmnd_init_write(tm_rsp,
2718                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2719
2720         spin_lock(&sess->sn_lock);
2721
2722 out:
2723         TRACE_EXIT();
2724         return;
2725 }
2726
2727 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status)
2728 {
2729         struct iscsi_cmnd *rsp;
2730         struct iscsi_task_mgt_hdr *req_hdr =
2731                                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2732         struct iscsi_task_rsp_hdr *rsp_hdr;
2733         struct iscsi_session *sess = req->conn->session;
2734         int fn = req_hdr->function & ISCSI_FUNCTION_MASK;
2735
2736         TRACE_ENTRY();
2737
2738         TRACE_MGMT_DBG("TM req %p finished", req);
2739         TRACE((req_hdr->function == ISCSI_FUNCTION_ABORT_TASK) ?
2740                          TRACE_MGMT_MINOR : TRACE_MGMT,
2741                 "TM fn %d finished, status %d", fn, status);
2742
2743         rsp = iscsi_cmnd_create_rsp_cmnd(req);
2744         rsp_hdr = (struct iscsi_task_rsp_hdr *)&rsp->pdu.bhs;
2745
2746         rsp_hdr->opcode = ISCSI_OP_SCSI_TASK_MGT_RSP;
2747         rsp_hdr->flags = ISCSI_FLG_FINAL;
2748         rsp_hdr->itt = req_hdr->itt;
2749         rsp_hdr->response = status;
2750
2751         if (fn == ISCSI_FUNCTION_TARGET_COLD_RESET)
2752                 rsp->should_close_conn = 1;
2753
2754         sBUG_ON(sess->tm_rsp != NULL);
2755
2756         spin_lock(&sess->sn_lock);
2757         if (iscsi_is_delay_tm_resp(rsp)) {
2758                 TRACE(TRACE_MGMT_MINOR, "Delaying TM fn %x response %p "
2759                         "(req %p), because not all affected commands received "
2760                         "(TM cmd sn %u, exp sn %u)",
2761                         req_hdr->function & ISCSI_FUNCTION_MASK, rsp, req,
2762                         req_hdr->cmd_sn, sess->exp_cmd_sn);
2763                 sess->tm_rsp = rsp;
2764                 spin_unlock(&sess->sn_lock);
2765                 goto out_release;
2766         }
2767         sess->tm_active--;
2768         spin_unlock(&sess->sn_lock);
2769
2770         sBUG_ON(sess->tm_active < 0);
2771
2772         iscsi_cmnd_init_write(rsp,
2773                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2774
2775 out_release:
2776         req_cmnd_release(req);
2777
2778         TRACE_EXIT();
2779         return;
2780 }
2781
2782 static inline int iscsi_get_mgmt_response(int status)
2783 {
2784         switch (status) {
2785         case SCST_MGMT_STATUS_SUCCESS:
2786                 return ISCSI_RESPONSE_FUNCTION_COMPLETE;
2787
2788         case SCST_MGMT_STATUS_TASK_NOT_EXIST:
2789                 return ISCSI_RESPONSE_UNKNOWN_TASK;
2790
2791         case SCST_MGMT_STATUS_LUN_NOT_EXIST:
2792                 return ISCSI_RESPONSE_UNKNOWN_LUN;
2793
2794         case SCST_MGMT_STATUS_FN_NOT_SUPPORTED:
2795                 return ISCSI_RESPONSE_FUNCTION_UNSUPPORTED;
2796
2797         case SCST_MGMT_STATUS_REJECTED:
2798         case SCST_MGMT_STATUS_FAILED:
2799         default:
2800                 return ISCSI_RESPONSE_FUNCTION_REJECTED;
2801         }
2802 }
2803
2804 static void iscsi_task_mgmt_fn_done(struct scst_mgmt_cmd *scst_mcmd)
2805 {
2806         struct iscsi_cmnd *req = (struct iscsi_cmnd*)
2807                                 scst_mgmt_cmd_get_tgt_priv(scst_mcmd);
2808         int status = iscsi_get_mgmt_response(scst_mgmt_cmd_get_status(scst_mcmd));
2809
2810         TRACE_MGMT_DBG("req %p, scst_mcmd %p, fn %d, scst status %d",
2811                 req, scst_mcmd, scst_mgmt_cmd_get_fn(scst_mcmd),
2812                 scst_mgmt_cmd_get_status(scst_mcmd));
2813
2814         iscsi_send_task_mgmt_resp(req, status);
2815
2816         scst_mgmt_cmd_set_tgt_priv(scst_mcmd, NULL);
2817
2818         return;
2819 }
2820
2821 static int iscsi_target_detect(struct scst_tgt_template *templ)
2822 {
2823         /* Nothing to do */
2824         return 0;
2825 }
2826
2827 static int iscsi_target_release(struct scst_tgt *scst_tgt)
2828 {
2829         /* Nothing to do */
2830         return 0;
2831 }
2832
2833 struct scst_tgt_template iscsi_template = {
2834         .name = "iscsi",
2835         .sg_tablesize = ISCSI_CONN_IOV_MAX,
2836         .threads_num = 0,
2837         .no_clustering = 1,
2838         .xmit_response_atomic = 0,
2839         .detect = iscsi_target_detect,
2840         .release = iscsi_target_release,
2841         .xmit_response = iscsi_xmit_response,
2842 #ifndef NET_PAGE_CALLBACKS_DEFINED
2843         .alloc_data_buf = iscsi_alloc_data_buf,
2844 #endif
2845         .preprocessing_done = iscsi_preprocessing_done,
2846         .pre_exec = iscsi_pre_exec,
2847         .task_mgmt_fn_done = iscsi_task_mgmt_fn_done,
2848 };
2849
2850 static __init int iscsi_run_threads(int count, char *name, int (*fn)(void *))
2851 {
2852         int res = 0;
2853         int i;
2854         struct iscsi_thread_t *thr;
2855
2856         for (i = 0; i < count; i++) {
2857                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
2858                 if (!thr) {
2859                         res = -ENOMEM;
2860                         PRINT_ERROR("Failed to allocate thr %d", res);
2861                         goto out;
2862                 }
2863                 thr->thr = kthread_run(fn, NULL, "%s%d", name, i);
2864                 if (IS_ERR(thr->thr)) {
2865                         res = PTR_ERR(thr->thr);
2866                         PRINT_ERROR("kthread_create() failed: %d", res);
2867                         kfree(thr);
2868                         goto out;
2869                 }
2870                 list_add(&thr->threads_list_entry, &iscsi_threads_list);
2871         }
2872
2873 out:
2874         return res;
2875 }
2876
2877 static void iscsi_stop_threads(void)
2878 {
2879         struct iscsi_thread_t *t, *tmp;
2880
2881         list_for_each_entry_safe(t, tmp, &iscsi_threads_list,
2882                                 threads_list_entry) {
2883                 int rc = kthread_stop(t->thr);
2884                 if (rc < 0) {
2885                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
2886                 }
2887                 list_del(&t->threads_list_entry);
2888                 kfree(t);
2889         }
2890 }
2891
2892 static int __init iscsi_init(void)
2893 {
2894         int err;
2895         int num;
2896
2897         PRINT_INFO("iSCSI SCST Target - version %s", ISCSI_VERSION_STRING);
2898
2899 #ifdef NET_PAGE_CALLBACKS_DEFINED
2900         err = net_set_get_put_page_callbacks(iscsi_get_page_callback,
2901                         iscsi_put_page_callback);
2902         if (err != 0) {
2903                 PRINT_INFO("Unable to set page callbackes: %d", err);
2904                 goto out;
2905         }
2906 #else
2907         PRINT_INFO("%s", "Patch put_page_callback-<kernel-version>.patch "
2908                 "not applied on your kernel. Running in the performance "
2909                 "degraded mode. Refer README file for details");
2910 #endif
2911
2912         BUILD_BUG_ON(MAX_DATA_SEG_LEN != (ISCSI_CONN_IOV_MAX<<PAGE_SHIFT));
2913
2914         if ((ctr_major = register_chrdev(0, ctr_name, &ctr_fops)) < 0) {
2915                 PRINT_ERROR("failed to register the control device %d", ctr_major);
2916                 err = ctr_major;
2917                 goto out_callb;
2918         }
2919
2920         if ((err = event_init()) < 0)
2921                 goto out_reg;
2922
2923         iscsi_cmnd_cache = KMEM_CACHE(iscsi_cmnd, SCST_SLAB_FLAGS);
2924         if (!iscsi_cmnd_cache) {
2925                 err = -ENOMEM;
2926                 goto out_event;
2927         }
2928
2929         err = scst_register_target_template(&iscsi_template);
2930         if (err < 0)
2931                 goto out_kmem;
2932
2933         iscsi_template_registered = 1;
2934
2935         if ((err = iscsi_procfs_init()) < 0)
2936                 goto out_reg_tmpl;
2937
2938         num = max(num_online_cpus(), 2);
2939
2940         err = iscsi_run_threads(num, "iscsird", istrd);
2941         if (err != 0)
2942                 goto out_thr;
2943
2944         err = iscsi_run_threads(num, "iscsiwr", istwr);
2945         if (err != 0)
2946                 goto out_thr;
2947
2948 out:
2949         return err;
2950
2951 out_thr:
2952         iscsi_procfs_exit();
2953         iscsi_stop_threads();
2954
2955 out_reg_tmpl:
2956         scst_unregister_target_template(&iscsi_template);
2957
2958 out_kmem:
2959         kmem_cache_destroy(iscsi_cmnd_cache);
2960
2961 out_event:
2962         event_exit();
2963
2964 out_reg:
2965         unregister_chrdev(ctr_major, ctr_name);
2966
2967 out_callb:
2968 #ifdef NET_PAGE_CALLBACKS_DEFINED
2969         net_set_get_put_page_callbacks(NULL, NULL);
2970 #endif
2971         goto out;
2972 }
2973
2974 static void __exit iscsi_exit(void)
2975 {
2976         iscsi_stop_threads();
2977
2978         unregister_chrdev(ctr_major, ctr_name);
2979
2980         iscsi_procfs_exit();
2981         event_exit();
2982
2983         kmem_cache_destroy(iscsi_cmnd_cache);
2984
2985         scst_unregister_target_template(&iscsi_template);
2986
2987 #ifdef NET_PAGE_CALLBACKS_DEFINED
2988         net_set_get_put_page_callbacks(NULL, NULL);
2989 #endif
2990 }
2991
2992 module_init(iscsi_init);
2993 module_exit(iscsi_exit);
2994
2995 MODULE_LICENSE("GPL");