- Update to the latest IET r145
[mirror/scst/.git] / iscsi-scst / kernel / iscsi.c
1 /*
2  *  Copyright (C) 2002-2003 Ardis Technolgies <roman@ardistech.com>
3  *  Copyright (C) 2007 Vladislav Bolkhovitin
4  *  Copyright (C) 2007 CMS Distribution Limited
5  * 
6  *  This program is free software; you can redistribute it and/or
7  *  modify it under the terms of the GNU General Public License
8  *  as published by the Free Software Foundation, version 2
9  *  of the License.
10  * 
11  *  This program is distributed in the hope that it will be useful,
12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  *  GNU General Public License for more details.
15  */
16
17 #include <linux/module.h>
18 #include <linux/hash.h>
19 #include <linux/kthread.h>
20 #include <net/tcp.h>
21 #include <scsi/scsi.h>
22
23 #include "iscsi.h"
24 #include "digest.h"
25
26 #ifndef NET_PAGE_CALLBACKS_DEFINED
27 #warning Patch put_page_callback-<kernel-version>.patch not applied on your \
28         kernel. ISCSI-SCST will run in the performance degraded mode. Refer \
29         README file for details.
30 #endif
31
32 #define ISCSI_INIT_WRITE_WAKE           0x1
33 #define ISCSI_INIT_WRITE_REMOVE_HASH    0x2
34
35 static int ctr_major;
36 static char ctr_name[] = "iscsi-scst-ctl";
37 static int iscsi_template_registered;
38
39 #if defined(DEBUG) || defined(TRACING)
40 unsigned long iscsi_trace_flag = ISCSI_DEFAULT_LOG_FLAGS;
41 #endif
42
43 static struct kmem_cache *iscsi_cmnd_cache;
44
45 spinlock_t iscsi_rd_lock = SPIN_LOCK_UNLOCKED;
46 LIST_HEAD(iscsi_rd_list);
47 DECLARE_WAIT_QUEUE_HEAD(iscsi_rd_waitQ);
48
49 spinlock_t iscsi_wr_lock = SPIN_LOCK_UNLOCKED;
50 LIST_HEAD(iscsi_wr_list);
51 DECLARE_WAIT_QUEUE_HEAD(iscsi_wr_waitQ);
52
53 static char dummy_data[1024];
54
55 struct iscsi_thread_t {
56         struct task_struct *thr;
57         struct list_head threads_list_entry;
58 };
59
60 static LIST_HEAD(iscsi_threads_list);
61
62 static void cmnd_remove_hash(struct iscsi_cmnd *cmnd);
63 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status);
64 static void cmnd_prepare_skip_pdu(struct iscsi_cmnd *cmnd);
65 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess);
66
67 static inline u32 cmnd_write_size(struct iscsi_cmnd *cmnd)
68 {
69         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
70
71         if (hdr->flags & ISCSI_CMD_WRITE)
72                 return be32_to_cpu(hdr->data_length);
73         return 0;
74 }
75
76 static inline u32 cmnd_read_size(struct iscsi_cmnd *cmnd)
77 {
78         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
79
80         if (hdr->flags & ISCSI_CMD_READ) {
81                 struct iscsi_rlength_ahdr *ahdr =
82                         (struct iscsi_rlength_ahdr *)cmnd->pdu.ahs;
83
84                 if (!(hdr->flags & ISCSI_CMD_WRITE))
85                         return be32_to_cpu(hdr->data_length);
86                 if (ahdr && ahdr->ahstype == ISCSI_AHSTYPE_RLENGTH)
87                         return be32_to_cpu(ahdr->read_length);
88         }
89         return 0;
90 }
91
92 static inline void iscsi_restart_cmnd(struct iscsi_cmnd *cmnd)
93 {
94         cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
95         scst_restart_cmd(cmnd->scst_cmd, SCST_PREPROCESS_STATUS_SUCCESS,
96                 SCST_CONTEXT_THREAD);
97 }
98
99 struct iscsi_cmnd *cmnd_alloc(struct iscsi_conn *conn, struct iscsi_cmnd *parent)
100 {
101         struct iscsi_cmnd *cmnd;
102
103         /* ToDo: __GFP_NOFAIL?? */
104 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,17)
105         cmnd = kmem_cache_alloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
106         memset(cmnd, 0, sizeof(*cmnd));
107 #else
108         cmnd = kmem_cache_zalloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
109 #endif
110
111         atomic_set(&cmnd->ref_cnt, 1);
112         cmnd->scst_state = ISCSI_CMD_STATE_NEW;
113         cmnd->conn = conn;
114         cmnd->parent_req = parent;
115         init_waitqueue_head(&cmnd->scst_waitQ);
116
117         if (parent == NULL) {
118                 conn_get(conn);
119
120 #ifdef NET_PAGE_CALLBACKS_DEFINED
121                 atomic_set(&cmnd->net_ref_cnt, 0);
122 #endif          
123                 cmnd->target = conn->target;
124                 spin_lock_init(&cmnd->rsp_cmd_lock);
125                 INIT_LIST_HEAD(&cmnd->rsp_cmd_list);
126                 INIT_LIST_HEAD(&cmnd->rx_ddigest_cmd_list);
127
128                 spin_lock_bh(&conn->cmd_list_lock);
129                 list_add_tail(&cmnd->cmd_list_entry, &conn->cmd_list);
130                 spin_unlock_bh(&conn->cmd_list_lock);
131         }
132
133         TRACE_DBG("conn %p, parent %p, cmnd %p", conn, parent, cmnd);
134         return cmnd;
135 }
136
137 /* Frees a command. Also frees the additional header. */
138 void cmnd_free(struct iscsi_cmnd *cmnd)
139 {
140         TRACE_DBG("%p", cmnd);
141
142         /* Catch users from cmd_list or rsp_cmd_list */
143         EXTRACHECKS_BUG_ON(atomic_read(&cmnd->ref_cnt) != 0);
144
145         kfree(cmnd->pdu.ahs);
146
147         if (unlikely(cmnd->on_write_list)) {
148                 struct iscsi_scsi_cmd_hdr *req = cmnd_hdr(cmnd);
149
150                 PRINT_ERROR("cmnd %p still on some list?, %x, %x, %x, %x, %x, %x, %x",
151                         cmnd, req->opcode, req->scb[0], req->flags, req->itt,
152                         be32_to_cpu(req->data_length),
153                         req->cmd_sn, be32_to_cpu(cmnd->pdu.datasize));
154
155                 if (cmnd->parent_req) {
156                         struct iscsi_scsi_cmd_hdr *req =
157                                         cmnd_hdr(cmnd->parent_req);
158                         PRINT_ERROR("%p %x %u", req, req->opcode, req->scb[0]);
159                 }
160                 sBUG();
161         }
162
163         kmem_cache_free(iscsi_cmnd_cache, cmnd);
164         return;
165 }
166
167 void cmnd_done(struct iscsi_cmnd *cmnd)
168 {
169         TRACE_DBG("%p", cmnd);
170
171         if (unlikely(cmnd->tmfabort)) {
172                 TRACE_MGMT_DBG("Done aborted cmd %p (scst cmd %p, state %d)",
173                         cmnd, cmnd->scst_cmd, cmnd->scst_state);
174         }
175
176         if (cmnd->parent_req == NULL) {
177                 struct iscsi_conn *conn = cmnd->conn;
178                 TRACE_DBG("Deleting req %p from conn %p", cmnd, conn);
179                 spin_lock_bh(&conn->cmd_list_lock);
180                 list_del(&cmnd->cmd_list_entry);
181                 spin_unlock_bh(&conn->cmd_list_lock);
182
183                 conn_put(conn);
184
185                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rsp_cmd_list));
186                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rx_ddigest_cmd_list));
187
188                 /* Order between above and below code is important! */
189
190                 if (cmnd->scst_cmd) {
191                         switch(cmnd->scst_state) {
192                         case ISCSI_CMD_STATE_AFTER_PREPROC:
193                                 TRACE_DBG("%s", "AFTER_PREPROC");
194                                 cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
195                                 scst_restart_cmd(cmnd->scst_cmd,
196                                         SCST_PREPROCESS_STATUS_ERROR_FATAL,
197                                         SCST_CONTEXT_THREAD);
198                                 break;
199                         case ISCSI_CMD_STATE_PROCESSED:
200                                 TRACE_DBG("%s", "PROCESSED");
201                                 scst_tgt_cmd_done(cmnd->scst_cmd);
202                                 break;
203                         default:
204                                 PRINT_ERROR("Unexpected cmnd scst state %d",
205                                         cmnd->scst_state);
206                                 sBUG();
207                                 break;
208                         }
209                 }
210         } else {
211                 EXTRACHECKS_BUG_ON(cmnd->scst_cmd != NULL);
212
213                 spin_lock_bh(&cmnd->parent_req->rsp_cmd_lock);
214                 TRACE_DBG("Deleting rsp %p from parent %p", cmnd,
215                         cmnd->parent_req);
216                 list_del(&cmnd->rsp_cmd_list_entry);
217                 spin_unlock_bh(&cmnd->parent_req->rsp_cmd_lock);
218
219                 cmnd_put(cmnd->parent_req);
220         }
221
222         /* Order between above and below code is important! */
223
224         if (cmnd->own_sg) {
225                 TRACE_DBG("%s", "own_sg");
226                 scst_free(cmnd->sg, cmnd->sg_cnt);
227 #ifdef DEBUG
228                 cmnd->own_sg = 0;
229                 cmnd->sg = NULL;
230                 cmnd->sg_cnt = -1;
231 #endif
232         }
233
234         if (cmnd->dec_active_cmnds) {
235                 struct iscsi_session *sess = cmnd->conn->session;
236                 TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
237                         "new value %d)", cmnd, sess,
238                         atomic_read(&sess->active_cmds)-1);
239                 atomic_dec(&sess->active_cmds);
240         }
241
242         cmnd_free(cmnd);
243         return;
244 }
245
246 /*
247  * Corresponding conn may also gets destroyed atfer this function, except only
248  * if it's called from the read thread! 
249  */
250 void req_cmnd_release_force(struct iscsi_cmnd *req, int flags)
251 {
252         struct iscsi_cmnd *rsp;
253         struct iscsi_conn *conn = req->conn;
254
255         TRACE_ENTRY();
256
257         TRACE_DBG("%p", req);
258
259         if (flags & ISCSI_FORCE_RELEASE_WRITE) {
260 again_write:
261                 spin_lock(&conn->write_list_lock);
262                 list_for_each_entry(rsp, &conn->write_list, write_list_entry) {
263                         if (rsp->parent_req != req)
264                                 continue;
265
266                         cmd_del_from_write_list(rsp);
267
268                         spin_unlock(&conn->write_list_lock);
269
270                         cmnd_put(rsp);
271                         goto again_write;
272                 }
273                 spin_unlock(&conn->write_list_lock);
274         }
275
276 again_rsp:
277         spin_lock_bh(&req->rsp_cmd_lock);
278         list_for_each_entry(rsp, &req->rsp_cmd_list, rsp_cmd_list_entry) {
279                 int f;
280
281                 if (rsp->on_write_list || rsp->write_processing_started ||
282                     rsp->force_cleanup_done)
283                         continue;
284
285                 spin_unlock_bh(&req->rsp_cmd_lock);
286
287                 /* 
288                  * Recheck is necessary to not take write_list_lock under
289                  * rsp_cmd_lock.
290                  */
291                 spin_lock(&conn->write_list_lock);
292                 f = rsp->on_write_list || rsp->write_processing_started ||
293                         rsp->force_cleanup_done;
294                 spin_unlock(&conn->write_list_lock);
295                 if (f)
296                         goto again_rsp;
297
298                 rsp->force_cleanup_done = 1;
299                 cmnd_put(rsp);
300
301                 goto again_rsp;
302         }
303         spin_unlock_bh(&req->rsp_cmd_lock);
304
305         req_cmnd_release(req);
306
307         TRACE_EXIT();
308         return;
309 }
310
311 /*
312  * Corresponding conn may also gets destroyed atfer this function, except only
313  * if it's called from the read thread! 
314  */
315 void req_cmnd_release(struct iscsi_cmnd *req)
316 {
317         struct iscsi_cmnd *c, *t;
318
319         TRACE_ENTRY();
320
321         TRACE_DBG("%p", req);
322
323 #ifdef EXTRACHECKS
324         sBUG_ON(req->release_called);
325         req->release_called = 1;
326 #endif
327
328         if (unlikely(req->tmfabort)) {
329                 TRACE_MGMT_DBG("Release aborted req cmd %p (scst cmd %p, "
330                         "state %d)", req, req->scst_cmd, req->scst_state);
331         }
332
333         sBUG_ON(req->parent_req != NULL);
334
335         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
336                                 rx_ddigest_cmd_list_entry) {
337                 TRACE_DBG("Deleting RX ddigest cmd %p from digest "
338                         "list of req %p", c, req);
339                 list_del(&c->rx_ddigest_cmd_list_entry);
340                 cmnd_put(c);
341         }
342
343         if (req->hashed)
344                 cmnd_remove_hash(req);
345
346         if (req->dec_active_cmnds) {
347                 struct iscsi_session *sess = req->conn->session;
348                 TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
349                         "new value %d)", req, sess,
350                         atomic_read(&sess->active_cmds)-1);
351                 atomic_dec(&sess->active_cmds);
352                 req->dec_active_cmnds = 0;
353         }
354
355         cmnd_put(req);
356
357         TRACE_EXIT();
358         return;
359 }
360
361 /*
362  * Corresponding conn may also gets destroyed atfer this function, except only
363  * if it's called from the read thread! 
364  */
365 void rsp_cmnd_release(struct iscsi_cmnd *cmnd)
366 {
367         TRACE_DBG("%p", cmnd);
368
369 #ifdef EXTRACHECKS
370         sBUG_ON(cmnd->release_called);
371         cmnd->release_called = 1;
372 #endif
373
374         sBUG_ON(cmnd->hashed);
375         sBUG_ON(cmnd->parent_req == NULL);
376
377         if (unlikely(cmnd->tmfabort)) {
378                 TRACE_MGMT_DBG("Release aborted rsp cmd %p (parent req %p, "
379                         "scst cmd %p, state %d)", cmnd, cmnd->parent_req,
380                         cmnd->parent_req->scst_cmd,
381                         cmnd->parent_req->scst_state);
382         }
383
384         cmnd_put(cmnd);
385         return;
386 }
387
388 /**
389  * create a new command used as response.
390  *
391  * iscsi_cmnd_create_rsp_cmnd - 
392  * @cmnd: ptr to request command
393  *
394  * @return    ptr to response command or NULL
395  */
396 static struct iscsi_cmnd *iscsi_cmnd_create_rsp_cmnd(struct iscsi_cmnd *parent)
397 {
398         struct iscsi_cmnd *rsp = cmnd_alloc(parent->conn, parent);
399
400         spin_lock_bh(&parent->rsp_cmd_lock);
401         TRACE_DBG("Adding rsp %p to parent %p", rsp, parent);
402         list_add_tail(&rsp->rsp_cmd_list_entry, &parent->rsp_cmd_list);
403         spin_unlock_bh(&parent->rsp_cmd_lock);
404         cmnd_get(parent);
405         return rsp;
406 }
407
408 static inline struct iscsi_cmnd *get_rsp_cmnd(struct iscsi_cmnd *req)
409 {
410         struct iscsi_cmnd *res;
411
412         /* Currently this lock isn't needed, but just in case.. */
413         spin_lock_bh(&req->rsp_cmd_lock);
414         res = list_entry(req->rsp_cmd_list.prev, struct iscsi_cmnd,
415                 rsp_cmd_list_entry);
416         spin_unlock_bh(&req->rsp_cmd_lock);
417
418         return res;
419 }
420
421 static void iscsi_cmnds_init_write(struct list_head *send, int flags)
422 {
423         struct iscsi_cmnd *rsp = list_entry(send->next, struct iscsi_cmnd, 
424                                                 write_list_entry);
425         struct iscsi_conn *conn = rsp->conn;
426         struct list_head *pos, *next;
427
428         /*
429          * If we don't remove hashed req cmd from the hash list here, before
430          * submitting it for transmittion, we will have a race, when for
431          * some reason cmd's release is delayed after transmittion and
432          * initiator sends cmd with the same ITT => this command will be
433          * erroneously rejected as a duplicate.
434          */
435         if ((flags & ISCSI_INIT_WRITE_REMOVE_HASH) && rsp->parent_req->hashed &&
436             (rsp->parent_req->outstanding_r2t == 0))
437                 cmnd_remove_hash(rsp->parent_req);
438
439         list_for_each_safe(pos, next, send) {
440                 rsp = list_entry(pos, struct iscsi_cmnd, write_list_entry);
441
442                 TRACE_DBG("%p:%x", rsp, cmnd_opcode(rsp));
443
444                 sBUG_ON(conn != rsp->conn);
445
446                 if (!(conn->ddigest_type & DIGEST_NONE) &&
447                     (rsp->pdu.datasize != 0))
448                         digest_tx_data(rsp);
449
450                 list_del(&rsp->write_list_entry);
451
452                 spin_lock(&conn->write_list_lock);
453                 cmd_add_on_write_list(conn, rsp);
454                 spin_unlock(&conn->write_list_lock);
455         }
456         
457
458         if (flags & ISCSI_INIT_WRITE_WAKE)
459                 iscsi_make_conn_wr_active(conn);
460 }
461
462 static void iscsi_cmnd_init_write(struct iscsi_cmnd *rsp, int flags)
463 {
464         LIST_HEAD(head);
465
466         if (unlikely(rsp->on_write_list)) {
467                 PRINT_ERROR("cmd already on write list (%x %x %x %x %u %u "
468                         "%u %u %u %u %u %d %d",
469                         cmnd_itt(rsp), cmnd_ttt(rsp), cmnd_opcode(rsp),
470                         cmnd_scsicode(rsp), rsp->r2t_sn,
471                         rsp->r2t_length, rsp->is_unsolicited_data,
472                         rsp->target_task_tag, rsp->outstanding_r2t,
473                         rsp->hdigest, rsp->ddigest,
474                         list_empty(&rsp->rsp_cmd_list), rsp->hashed);
475                 sBUG();
476         }
477         list_add(&rsp->write_list_entry, &head);
478         iscsi_cmnds_init_write(&head, flags);
479 }
480
481 static void iscsi_set_datasize(struct iscsi_cmnd *cmnd, u32 offset, u32 size)
482 {
483         cmnd->pdu.datasize = size;
484
485         if (cmnd->pdu.datasize & 3) {
486                 int idx = (offset + cmnd->pdu.datasize) >> PAGE_SHIFT;
487                 u8 *p = (u8 *)page_address(cmnd->sg[idx].page) + 
488                         ((offset + cmnd->pdu.datasize) & ~PAGE_MASK);
489                 int i = 4 - (cmnd->pdu.datasize & 3);
490                 while (i--) 
491                     *p++ = 0;
492         }
493 }
494
495 static void send_data_rsp(struct iscsi_cmnd *req, u8 status, int send_status)
496 {
497         struct iscsi_cmnd *rsp;
498         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
499         struct iscsi_data_in_hdr *rsp_hdr;
500         u32 pdusize, expsize, scsisize, size, offset, sn;
501         LIST_HEAD(send);
502
503         TRACE_DBG("req %p", req);
504         pdusize = req->conn->session->sess_param.max_xmit_data_length;
505         expsize = cmnd_read_size(req);
506         size = min(expsize, (u32)req->bufflen);
507         offset = 0;
508         sn = 0;
509
510         while (1) {
511                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
512                 TRACE_DBG("rsp %p", rsp);
513                 rsp->sg = req->sg;
514                 rsp->bufflen = req->bufflen;
515                 rsp_hdr = (struct iscsi_data_in_hdr *)&rsp->pdu.bhs;
516
517                 rsp_hdr->opcode = ISCSI_OP_SCSI_DATA_IN;
518                 rsp_hdr->itt = req_hdr->itt;
519                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
520                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
521                 rsp_hdr->data_sn = cpu_to_be32(sn);
522
523                 if (size <= pdusize) {
524                         iscsi_set_datasize(rsp, offset, size);
525                         if (send_status) {
526                                 TRACE_DBG("status %x", status);
527                                 rsp_hdr->flags =
528                                         ISCSI_FLG_FINAL | ISCSI_FLG_STATUS;
529                                 rsp_hdr->cmd_status = status;
530                         }
531                         scsisize = req->bufflen;
532                         if (scsisize < expsize) {
533                                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
534                                 size = expsize - scsisize;
535                         } else if (scsisize > expsize) {
536                                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
537                                 size = scsisize - expsize;
538                         } else
539                                 size = 0;
540                         rsp_hdr->residual_count = cpu_to_be32(size);
541                         list_add_tail(&rsp->write_list_entry, &send);
542                         break;
543                 }
544
545                 iscsi_set_datasize(rsp, offset, pdusize);
546
547                 size -= pdusize;
548                 offset += pdusize;
549                 sn++;
550
551                 list_add_tail(&rsp->write_list_entry, &send);
552         }
553         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_REMOVE_HASH);
554 }
555
556 static struct iscsi_cmnd *create_status_rsp(struct iscsi_cmnd *req, int status,
557         const u8 *sense_buf, int sense_len)
558 {
559         struct iscsi_cmnd *rsp;
560         struct iscsi_scsi_rsp_hdr *rsp_hdr;
561         struct iscsi_sense_data *sense;
562         struct scatterlist *sg;
563
564         rsp = iscsi_cmnd_create_rsp_cmnd(req);
565         TRACE_DBG("%p", rsp);
566
567         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
568         rsp_hdr->opcode = ISCSI_OP_SCSI_RSP;
569         rsp_hdr->flags = ISCSI_FLG_FINAL;
570         rsp_hdr->response = ISCSI_RESPONSE_COMMAND_COMPLETED;
571         rsp_hdr->cmd_status = status;
572         rsp_hdr->itt = cmnd_hdr(req)->itt;
573
574         if (status == SAM_STAT_CHECK_CONDITION) {
575                 TRACE_DBG("%s", "CHECK_CONDITION");
576                 /* ToDo: __GFP_NOFAIL ?? */
577                 sg = rsp->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
578                                         &rsp->sg_cnt);
579                 if (sg == NULL) {
580                         /* ToDo(); */
581                 }
582                 rsp->own_sg = 1;
583                 sense = (struct iscsi_sense_data *)page_address(sg[0].page);
584                 sense->length = cpu_to_be16(sense_len);
585                 memcpy(sense->data, sense_buf, sense_len);
586                 rsp->pdu.datasize = sizeof(struct iscsi_sense_data) + sense_len;
587                 rsp->bufflen = (rsp->pdu.datasize + 3) & -4;
588                 if (rsp->bufflen - rsp->pdu.datasize) {
589                     int i = rsp->pdu.datasize;
590                     u8 *p = (u8 *)sense + i;
591                     
592                     while (i < rsp->bufflen) {
593                         *p ++ = 0;
594                         i++;
595                     }
596                 }
597         } else {
598                 rsp->pdu.datasize = 0;
599                 rsp->bufflen = 0;
600         }
601
602         return rsp;
603 }
604
605 static struct iscsi_cmnd *create_sense_rsp(struct iscsi_cmnd *req,
606         u8 sense_key, u8 asc, u8 ascq)
607 {
608         u8 sense[14];
609         memset(sense, 0, sizeof(sense));
610         sense[0] = 0xf0;
611         sense[2] = sense_key;
612         sense[7] = 6;   // Additional sense length
613         sense[12] = asc;
614         sense[13] = ascq;
615         return create_status_rsp(req, SAM_STAT_CHECK_CONDITION, sense,
616                 sizeof(sense));
617 }
618
619 static void iscsi_cmnd_reject(struct iscsi_cmnd *req, int reason)
620 {
621         struct iscsi_cmnd *rsp;
622         struct iscsi_reject_hdr *rsp_hdr;
623         struct scatterlist *sg;
624         char *addr;
625
626         TRACE_MGMT_DBG("Reject: req %p, reason %x", req, reason);
627
628         rsp = iscsi_cmnd_create_rsp_cmnd(req);
629         rsp_hdr = (struct iscsi_reject_hdr *)&rsp->pdu.bhs;
630
631         rsp_hdr->opcode = ISCSI_OP_REJECT;
632         rsp_hdr->ffffffff = ISCSI_RESERVED_TAG;
633         rsp_hdr->reason = reason;
634
635         /* ToDo: __GFP_NOFAIL ?? */
636         sg = rsp->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
637                                 &rsp->sg_cnt);
638         if (sg == NULL) {
639                 /* ToDo(); */
640         }
641         rsp->own_sg = 1;
642         addr = page_address(sg[0].page);
643         clear_page(addr);
644         memcpy(addr, &req->pdu.bhs, sizeof(struct iscsi_hdr));
645         rsp->bufflen = rsp->pdu.datasize = sizeof(struct iscsi_hdr);
646         cmnd_prepare_skip_pdu(req);
647
648         req->pdu.bhs.opcode = ISCSI_OP_PDU_REJECT;
649 }
650
651 static inline int iscsi_get_allowed_cmds(struct iscsi_session *sess)
652 {
653         int res = max(-1, (int)sess->max_queued_cmnds - 
654                                 atomic_read(&sess->active_cmds)-1);
655         TRACE_DBG("allowed cmds %d (sess %p, active_cmds %d)", res,
656                 sess, atomic_read(&sess->active_cmds));
657         return res;
658 }
659
660 static u32 cmnd_set_sn(struct iscsi_cmnd *cmnd, int set_stat_sn)
661 {
662         struct iscsi_conn *conn = cmnd->conn;
663         struct iscsi_session *sess = conn->session;
664         u32 res;
665
666         spin_lock(&sess->sn_lock);
667
668         if (set_stat_sn)
669                 cmnd->pdu.bhs.sn = cpu_to_be32(conn->stat_sn++);
670         cmnd->pdu.bhs.exp_sn = cpu_to_be32(sess->exp_cmd_sn);
671         cmnd->pdu.bhs.max_sn = cpu_to_be32(sess->exp_cmd_sn +
672                                  iscsi_get_allowed_cmds(sess));
673
674         res = cpu_to_be32(conn->stat_sn);
675
676         spin_unlock(&sess->sn_lock);
677         return res;
678 }
679
680 /* Called under sn_lock */
681 static void __update_stat_sn(struct iscsi_cmnd *cmnd)
682 {
683         struct iscsi_conn *conn = cmnd->conn;
684         u32 exp_stat_sn;
685
686         cmnd->pdu.bhs.exp_sn = exp_stat_sn = be32_to_cpu(cmnd->pdu.bhs.exp_sn);
687         TRACE_DBG("%x,%x", cmnd_opcode(cmnd), exp_stat_sn);
688         if ((int)(exp_stat_sn - conn->exp_stat_sn) > 0 &&
689             (int)(exp_stat_sn - conn->stat_sn) <= 0) {
690                 // free pdu resources
691                 cmnd->conn->exp_stat_sn = exp_stat_sn;
692         }
693 }
694
695 static inline void update_stat_sn(struct iscsi_cmnd *cmnd)
696 {
697         spin_lock(&cmnd->conn->session->sn_lock);
698         __update_stat_sn(cmnd);
699         spin_unlock(&cmnd->conn->session->sn_lock);
700 }
701
702 /* Called under sn_lock */
703 static int check_cmd_sn(struct iscsi_cmnd *cmnd)
704 {
705         struct iscsi_session *session = cmnd->conn->session;
706         u32 cmd_sn;
707
708         cmnd->pdu.bhs.sn = cmd_sn = be32_to_cpu(cmnd->pdu.bhs.sn);
709         TRACE_DBG("%d(%d)", cmd_sn, session->exp_cmd_sn);
710         if ((s32)(cmd_sn - session->exp_cmd_sn) >= 0)
711                 return 0;
712         PRINT_ERROR("sequence error (%x,%x)", cmd_sn, session->exp_cmd_sn);
713         return -ISCSI_REASON_PROTOCOL_ERROR;
714 }
715
716 static inline struct iscsi_cmnd *__cmnd_find_hash(struct iscsi_session *session,
717         u32 itt, u32 ttt)
718 {
719         struct list_head *head;
720         struct iscsi_cmnd *cmnd;
721
722         head = &session->cmnd_hash[cmnd_hashfn(itt)];
723
724         list_for_each_entry(cmnd, head, hash_list_entry) {
725                 if (cmnd->pdu.bhs.itt == itt) {
726                         if ((ttt != ISCSI_RESERVED_TAG) && (ttt != cmnd->target_task_tag))
727                                 continue;
728                         return cmnd;
729                 }
730         }
731         return NULL;
732 }
733
734 static struct iscsi_cmnd *cmnd_find_hash(struct iscsi_session *session,
735         u32 itt, u32 ttt)
736 {
737         struct iscsi_cmnd *cmnd;
738
739         spin_lock(&session->cmnd_hash_lock);
740         cmnd = __cmnd_find_hash(session, itt, ttt);
741         spin_unlock(&session->cmnd_hash_lock);
742
743         return cmnd;
744 }
745
746 static struct iscsi_cmnd *cmnd_find_hash_get(struct iscsi_session *session,
747         u32 itt, u32 ttt)
748 {
749         struct iscsi_cmnd *cmnd;
750
751         spin_lock(&session->cmnd_hash_lock);
752         cmnd = __cmnd_find_hash(session, itt, ttt);
753         if (cmnd)
754                 cmnd_get(cmnd);
755         spin_unlock(&session->cmnd_hash_lock);
756
757         return cmnd;
758 }
759
760 static int cmnd_insert_hash(struct iscsi_cmnd *cmnd)
761 {
762         struct iscsi_session *session = cmnd->conn->session;
763         struct iscsi_cmnd *tmp;
764         struct list_head *head;
765         int err = 0;
766         u32 itt = cmnd->pdu.bhs.itt;
767
768         TRACE_DBG("%p:%x", cmnd, itt);
769         if (itt == ISCSI_RESERVED_TAG) {
770                 PRINT_ERROR("%s", "ITT is RESERVED_TAG");
771                 err = -ISCSI_REASON_PROTOCOL_ERROR;
772                 goto out;
773         }
774
775         spin_lock(&session->cmnd_hash_lock);
776
777         head = &session->cmnd_hash[cmnd_hashfn(cmnd->pdu.bhs.itt)];
778
779         tmp = __cmnd_find_hash(session, itt, ISCSI_RESERVED_TAG);
780         if (!tmp) {
781                 list_add_tail(&cmnd->hash_list_entry, head);
782                 cmnd->hashed = 1;
783         } else {
784                 PRINT_ERROR("Task %x in progress, cmnd %p", itt, cmnd);
785                 err = -ISCSI_REASON_TASK_IN_PROGRESS;
786         }
787
788         spin_unlock(&session->cmnd_hash_lock);
789
790         if (!err) {
791                 spin_lock(&session->sn_lock);
792                 __update_stat_sn(cmnd);
793                 err = check_cmd_sn(cmnd);
794                 spin_unlock(&session->sn_lock);
795         }
796
797 out:
798         return err;
799 }
800
801 static void cmnd_remove_hash(struct iscsi_cmnd *cmnd)
802 {
803         struct iscsi_session *session = cmnd->conn->session;
804         struct iscsi_cmnd *tmp;
805
806         spin_lock(&session->cmnd_hash_lock);
807
808         tmp = __cmnd_find_hash(session, cmnd->pdu.bhs.itt, ISCSI_RESERVED_TAG);
809
810         if (tmp && tmp == cmnd) {
811                 list_del(&cmnd->hash_list_entry);
812                 cmnd->hashed = 0;
813         } else {
814                 PRINT_ERROR("%p:%x not found", cmnd, cmnd_itt(cmnd));
815         }
816
817         spin_unlock(&session->cmnd_hash_lock);
818 }
819
820 static void cmnd_prepare_skip_pdu(struct iscsi_cmnd *cmnd)
821 {
822         struct iscsi_conn *conn = cmnd->conn;
823         struct scatterlist *sg = cmnd->sg;
824         char *addr;
825         u32 size;
826         int i;
827
828         TRACE_MGMT_DBG("Skipping (%p, %x %x %x %u, %p, scst state %d)", cmnd,
829                 cmnd_itt(cmnd), cmnd_opcode(cmnd), cmnd_hdr(cmnd)->scb[0],
830                 cmnd->pdu.datasize, cmnd->scst_cmd, cmnd->scst_state);
831
832         iscsi_extracheck_is_rd_thread(conn);
833
834         if (!(size = cmnd->pdu.datasize))
835                 return;
836
837         if (sg == NULL) {
838                 /* ToDo: __GFP_NOFAIL ?? */
839                 sg = cmnd->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
840                                         &cmnd->sg_cnt);
841                 if (sg == NULL) {
842                         /* ToDo(); */
843                 }
844                 cmnd->own_sg = 1;
845                 cmnd->bufflen = PAGE_SIZE;
846         }
847
848         addr = page_address(sg[0].page);
849         sBUG_ON(addr == NULL);
850         size = (size + 3) & -4;
851         conn->read_size = size;
852         for (i = 0; size > PAGE_SIZE; i++, size -= cmnd->bufflen) {
853                 sBUG_ON(i >= ISCSI_CONN_IOV_MAX);
854                 conn->read_iov[i].iov_base = addr;
855                 conn->read_iov[i].iov_len = cmnd->bufflen;
856         }
857         conn->read_iov[i].iov_base = addr;
858         conn->read_iov[i].iov_len = size;
859         conn->read_msg.msg_iov = conn->read_iov;
860         conn->read_msg.msg_iovlen = ++i;
861 }
862
863 static void cmnd_prepare_skip_pdu_set_resid(struct iscsi_cmnd *req)
864 {
865         struct iscsi_cmnd *rsp;
866         struct iscsi_scsi_rsp_hdr *rsp_hdr;
867         u32 size;
868
869         TRACE_DBG("%p", req);
870
871         rsp = get_rsp_cmnd(req);
872         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
873         if (cmnd_opcode(rsp) != ISCSI_OP_SCSI_RSP) {
874                 PRINT_ERROR("unexpected response command %u", cmnd_opcode(rsp));
875                 return;
876         }
877
878         size = cmnd_write_size(req);
879         if (size) {
880                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
881                 rsp_hdr->residual_count = cpu_to_be32(size);
882         }
883         size = cmnd_read_size(req);
884         if (size) {
885                 if (cmnd_hdr(req)->flags & ISCSI_CMD_WRITE) {
886                         rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_UNDERFLOW;
887                         rsp_hdr->bi_residual_count = cpu_to_be32(size);
888                 } else {
889                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
890                         rsp_hdr->residual_count = cpu_to_be32(size);
891                 }
892         }
893         req->pdu.bhs.opcode =
894                 (req->pdu.bhs.opcode & ~ISCSI_OPCODE_MASK) | ISCSI_OP_SCSI_REJECT;
895
896         cmnd_prepare_skip_pdu(req);
897 }
898
899 static int cmnd_prepare_recv_pdu(struct iscsi_conn *conn,
900         struct iscsi_cmnd *cmd, u32 offset, u32 size)
901 {
902         struct scatterlist *sg = cmd->sg;
903         int bufflen = cmd->bufflen;
904         int idx, i;
905         char *addr;
906         int res = 0;
907
908         TRACE_DBG("%p %u,%u", cmd->sg, offset, size);
909
910         iscsi_extracheck_is_rd_thread(conn);
911
912         if ((offset >= bufflen) ||
913             (offset + size > bufflen)) {
914                 PRINT_ERROR("Wrong ltn (%u %u %u)", offset, size, bufflen);
915                 mark_conn_closed(conn);
916                 res = -EIO;
917                 goto out;
918         }
919
920         offset += sg[0].offset;
921         idx = offset >> PAGE_SHIFT;
922         offset &= ~PAGE_MASK;
923
924         conn->read_msg.msg_iov = conn->read_iov;
925         conn->read_size = size = (size + 3) & -4;
926
927         i = 0;
928         while (1) {
929                 sBUG_ON(sg[idx].page == NULL);
930                 addr = page_address(sg[idx].page);
931                 sBUG_ON(addr == NULL);
932                 conn->read_iov[i].iov_base =  addr + offset;
933                 if (offset + size <= PAGE_SIZE) {
934                         TRACE_DBG("idx=%d, offset=%u, size=%d, addr=%p",
935                                 idx, offset, size, addr);
936                         conn->read_iov[i].iov_len = size;
937                         conn->read_msg.msg_iovlen = ++i;
938                         break;
939                 }
940                 conn->read_iov[i].iov_len = PAGE_SIZE - offset;
941                 TRACE_DBG("idx=%d, offset=%u, size=%d, iov_len=%d, addr=%p",
942                         idx, offset, size, conn->read_iov[i].iov_len, addr);
943                 size -= conn->read_iov[i].iov_len;
944                 offset = 0;
945                 if (++i >= ISCSI_CONN_IOV_MAX) {
946                         PRINT_ERROR("Initiator %s violated negotiated "
947                                 "parameters by sending too much data (size "
948                                 "left %d)", conn->session->initiator_name, size);
949                         mark_conn_closed(conn);
950                         res = -EINVAL;
951                         break;
952                 }
953                 idx++;
954         }
955         TRACE_DBG("msg_iov=%p, msg_iovlen=%d",
956                 conn->read_msg.msg_iov, conn->read_msg.msg_iovlen);
957
958 out:
959         return res;
960 }
961
962 static void send_r2t(struct iscsi_cmnd *req)
963 {
964         struct iscsi_session *session = req->conn->session;
965         struct iscsi_cmnd *rsp;
966         struct iscsi_r2t_hdr *rsp_hdr;
967         u32 length, offset, burst;
968         LIST_HEAD(send);
969
970         if (unlikely(req->tmfabort)) {
971                 TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted on R2T", req,
972                         req->scst_cmd);
973                 req_cmnd_release_force(req, ISCSI_FORCE_RELEASE_WRITE);
974                 goto out;
975         }
976
977         /*
978          * There is no race with data_out_start() and __cmnd_abort(), since
979          * all functions called from single read thread
980          */
981         iscsi_extracheck_is_rd_thread(req->conn);
982
983         length = req->r2t_length;
984         burst = session->sess_param.max_burst_length;
985         offset = be32_to_cpu(cmnd_hdr(req)->data_length) - length;
986
987         do {
988                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
989                 rsp->pdu.bhs.ttt = req->target_task_tag;
990                 rsp_hdr = (struct iscsi_r2t_hdr *)&rsp->pdu.bhs;
991                 rsp_hdr->opcode = ISCSI_OP_R2T;
992                 rsp_hdr->flags = ISCSI_FLG_FINAL;
993                 rsp_hdr->lun = cmnd_hdr(req)->lun;
994                 rsp_hdr->itt = cmnd_hdr(req)->itt;
995                 rsp_hdr->r2t_sn = cpu_to_be32(req->r2t_sn++);
996                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
997                 if (length > burst) {
998                         rsp_hdr->data_length = cpu_to_be32(burst);
999                         length -= burst;
1000                         offset += burst;
1001                 } else {
1002                         rsp_hdr->data_length = cpu_to_be32(length);
1003                         length = 0;
1004                 }
1005
1006                 TRACE(TRACE_D_WRITE, "%x %u %u %u %u", cmnd_itt(req),
1007                         be32_to_cpu(rsp_hdr->data_length),
1008                         be32_to_cpu(rsp_hdr->buffer_offset),
1009                         be32_to_cpu(rsp_hdr->r2t_sn), req->outstanding_r2t);
1010
1011                 list_add_tail(&rsp->write_list_entry, &send);
1012
1013                 if (++req->outstanding_r2t >= session->sess_param.max_outstanding_r2t)
1014                         break;
1015
1016         } while (length);
1017
1018         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_WAKE);
1019
1020         req->data_waiting = 1;
1021
1022 out:
1023         return;
1024 }
1025
1026 static int iscsi_pre_exec(struct scst_cmd *scst_cmd)
1027 {
1028         int res = SCST_PREPROCESS_STATUS_SUCCESS;
1029         struct iscsi_cmnd *req = (struct iscsi_cmnd*)
1030                 scst_cmd_get_tgt_priv(scst_cmd);
1031         struct iscsi_cmnd *c, *t;
1032
1033         TRACE_ENTRY();
1034
1035         EXTRACHECKS_BUG_ON(scst_cmd_atomic(scst_cmd));
1036
1037         /* If data digest isn't used this list will be empty */
1038         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
1039                                 rx_ddigest_cmd_list_entry) {
1040                 TRACE_DBG("Checking digest of RX ddigest cmd %p", c);
1041                 if (digest_rx_data(c) != 0) {
1042                         scst_set_cmd_error(scst_cmd,
1043                                 SCST_LOAD_SENSE(iscsi_sense_crc_error));
1044                         res = SCST_PREPROCESS_STATUS_ERROR_SENSE_SET;
1045                         /*
1046                          * The rest of rx_ddigest_cmd_list will be freed 
1047                          * in req_cmnd_release()
1048                          */
1049                         goto out;
1050                 }
1051                 TRACE_DBG("Deleting RX digest cmd %p from digest list", c);
1052                 list_del(&c->rx_ddigest_cmd_list_entry);
1053                 cmnd_put(c);
1054         }
1055
1056 out:
1057         TRACE_EXIT_RES(res);
1058         return res;
1059 }
1060
1061
1062 static void scsi_cmnd_exec(struct iscsi_cmnd *cmnd)
1063 {
1064         if (cmnd->r2t_length) {
1065                 if (!cmnd->is_unsolicited_data)
1066                         send_r2t(cmnd);
1067         } else {
1068                 /*
1069                  * There is no race with send_r2t() and __cmnd_abort(),
1070                  * since all functions called from single read thread
1071                  */
1072                 cmnd->data_waiting = 0;
1073                 iscsi_restart_cmnd(cmnd);
1074         }
1075 }
1076
1077 static int noop_out_start(struct iscsi_cmnd *cmnd)
1078 {
1079         struct iscsi_conn *conn = cmnd->conn;
1080         u32 size, tmp;
1081         int i, err = 0;
1082
1083         TRACE_DBG("%p", cmnd);
1084
1085         iscsi_extracheck_is_rd_thread(conn);
1086
1087         if (cmnd_ttt(cmnd) != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1088                 /*
1089                  * We don't request a NOP-Out by sending a NOP-In.
1090                  * See 10.18.2 in the draft 20.
1091                  */
1092                 PRINT_ERROR("initiator bug %x", cmnd_itt(cmnd));
1093                 err = -ISCSI_REASON_PROTOCOL_ERROR;
1094                 goto out;
1095         }
1096
1097         if (cmnd_itt(cmnd) == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1098                 if (!(cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE))
1099                         PRINT_ERROR("%s","initiator bug!");
1100                 spin_lock(&conn->session->sn_lock);
1101                 __update_stat_sn(cmnd);
1102                 err = check_cmd_sn(cmnd);
1103                 spin_unlock(&conn->session->sn_lock);
1104                 if (err)
1105                         goto out;
1106         } else if ((err = cmnd_insert_hash(cmnd)) < 0) {
1107                 PRINT_ERROR("Can't insert in hash: ignore this request %x",
1108                         cmnd_itt(cmnd));
1109                 goto out;
1110         }
1111
1112         if ((size = cmnd->pdu.datasize)) {
1113                 size = (size + 3) & -4;
1114                 conn->read_msg.msg_iov = conn->read_iov;
1115                 if (cmnd->pdu.bhs.itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1116                         struct scatterlist *sg;
1117
1118                         /* ToDo: __GFP_NOFAIL ?? */
1119                         cmnd->sg = sg = scst_alloc(size,
1120                                 GFP_KERNEL|__GFP_NOFAIL, &cmnd->sg_cnt);
1121                         if (sg == NULL) {
1122                                 /* ToDo(); */
1123                         }
1124                         if (cmnd->sg_cnt > ISCSI_CONN_IOV_MAX) {
1125                                 /* ToDo(); */
1126                         }
1127                         cmnd->own_sg = 1;
1128                         cmnd->bufflen = size;
1129
1130                         for (i = 0; i < cmnd->sg_cnt; i++) {
1131                                 conn->read_iov[i].iov_base =
1132                                         page_address(sg[i].page);
1133                                 tmp = min_t(u32, size, PAGE_SIZE);
1134                                 conn->read_iov[i].iov_len = tmp;
1135                                 conn->read_size += tmp;
1136                                 size -= tmp;
1137                         }
1138                 } else {
1139                         /*
1140                          * There are no problems with the safety from concurrent
1141                          * accesses to dummy_data, since for ISCSI_RESERVED_TAG
1142                          * the data only read and then discarded.
1143                          */
1144                         for (i = 0; i < ISCSI_CONN_IOV_MAX; i++) {
1145                                 conn->read_iov[i].iov_base = dummy_data;
1146                                 tmp = min_t(u32, size, sizeof(dummy_data));
1147                                 conn->read_iov[i].iov_len = tmp;
1148                                 conn->read_size += tmp;
1149                                 size -= tmp;
1150                         }
1151                 }
1152                 sBUG_ON(size == 0);
1153                 conn->read_msg.msg_iovlen = i;
1154                 TRACE_DBG("msg_iov=%p, msg_iovlen=%d", conn->read_msg.msg_iov,
1155                         conn->read_msg.msg_iovlen);
1156         }
1157 out:
1158         return err;
1159 }
1160
1161 static inline u32 get_next_ttt(struct iscsi_conn *conn)
1162 {
1163         u32 ttt;
1164         struct iscsi_session *session = conn->session;
1165
1166         iscsi_extracheck_is_rd_thread(conn);
1167
1168         if (session->next_ttt == ISCSI_RESERVED_TAG)
1169                 session->next_ttt++;
1170         ttt = session->next_ttt++;
1171
1172         return cpu_to_be32(ttt);
1173 }
1174
1175 static int scsi_cmnd_start(struct iscsi_cmnd *req)
1176 {
1177         struct iscsi_conn *conn = req->conn;
1178         struct iscsi_session *session = conn->session;
1179         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1180         struct scst_cmd *scst_cmd;
1181         scst_data_direction dir;
1182         int res = 0;
1183
1184         TRACE_ENTRY();
1185
1186         TRACE_DBG("scsi command: %02x", req_hdr->scb[0]);
1187
1188         TRACE_DBG("Incrementing active_cmds (cmd %p, sess %p, "
1189                 "new value %d)", req, session,
1190                 atomic_read(&session->active_cmds)+1);
1191         atomic_inc(&session->active_cmds);
1192         req->dec_active_cmnds = 1;
1193
1194         scst_cmd = scst_rx_cmd(session->scst_sess,
1195                 (uint8_t*)&req_hdr->lun, sizeof(req_hdr->lun),
1196                 req_hdr->scb, sizeof(req_hdr->scb), SCST_NON_ATOMIC);
1197         if (scst_cmd == NULL) {
1198                 create_status_rsp(req, SAM_STAT_BUSY, NULL, 0);
1199                 cmnd_prepare_skip_pdu_set_resid(req);
1200                 goto out;
1201         }
1202
1203         req->scst_cmd = scst_cmd;
1204         scst_cmd_set_tag(scst_cmd, req_hdr->itt);
1205         scst_cmd_set_tgt_priv(scst_cmd, req);
1206 #ifndef NET_PAGE_CALLBACKS_DEFINED
1207         scst_cmd_set_data_buf_tgt_alloc(scst_cmd);
1208 #endif
1209
1210         if (req_hdr->flags & ISCSI_CMD_READ)
1211                 dir = SCST_DATA_READ;
1212         else if (req_hdr->flags & ISCSI_CMD_WRITE)
1213                 dir = SCST_DATA_WRITE;
1214         else
1215                 dir = SCST_DATA_NONE;
1216         scst_cmd_set_expected(scst_cmd, dir, be32_to_cpu(req_hdr->data_length));
1217
1218         switch(req_hdr->flags & ISCSI_CMD_ATTR_MASK) {
1219         case ISCSI_CMD_SIMPLE:
1220                 scst_cmd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1221                 break;
1222         case ISCSI_CMD_HEAD_OF_QUEUE:
1223                 scst_cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1224                 break;
1225         case ISCSI_CMD_ORDERED:
1226                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1227                 break;
1228         case ISCSI_CMD_ACA:
1229                 scst_cmd->queue_type = SCST_CMD_QUEUE_ACA;
1230                 break;
1231         case ISCSI_CMD_UNTAGGED:
1232                 scst_cmd->queue_type = SCST_CMD_QUEUE_UNTAGGED;
1233                 break;
1234         default:
1235                 PRINT_ERROR("Unknown task code %x, use ORDERED instead",
1236                         req_hdr->flags & ISCSI_CMD_ATTR_MASK);
1237                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1238                 break;
1239         }
1240
1241         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
1242         scst_cmd_set_tgt_sn(scst_cmd, req_hdr->cmd_sn);
1243
1244         TRACE_DBG("START Command (tag %d, queue_type %d)",
1245                 req_hdr->itt, scst_cmd->queue_type);
1246         req->scst_state = ISCSI_CMD_STATE_RX_CMD;
1247         scst_cmd_init_stage1_done(scst_cmd, SCST_CONTEXT_DIRECT, 0);
1248
1249         wait_event(req->scst_waitQ, (req->scst_state != ISCSI_CMD_STATE_RX_CMD));
1250
1251         if (unlikely(req->scst_state != ISCSI_CMD_STATE_AFTER_PREPROC)) {
1252                 TRACE_DBG("req %p is in %x state", req, req->scst_state);
1253                 if (req->scst_state == ISCSI_CMD_STATE_PROCESSED) {
1254                         /* Response is already prepared */
1255                         cmnd_prepare_skip_pdu_set_resid(req);
1256                         goto out;
1257                 }
1258                 if (unlikely(req->tmfabort)) {
1259                         TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
1260                                 req->scst_cmd);
1261                         cmnd_prepare_skip_pdu(req);
1262                         goto out;
1263                 }
1264                 sBUG();
1265         }
1266
1267         dir = scst_cmd_get_data_direction(scst_cmd);
1268         if (dir != SCST_DATA_WRITE) {
1269                 if (!(req_hdr->flags & ISCSI_CMD_FINAL) || req->pdu.datasize) {
1270                         PRINT_ERROR("Unexpected unsolicited data (ITT %x "
1271                                 "CDB %x", cmnd_itt(req), req_hdr->scb[0]);
1272                         create_sense_rsp(req, ABORTED_COMMAND, 0xc, 0xc);
1273                         cmnd_prepare_skip_pdu_set_resid(req);
1274                         goto out;
1275                 }
1276         }
1277
1278         if (dir == SCST_DATA_WRITE) {
1279                 req->is_unsolicited_data = !(req_hdr->flags & ISCSI_CMD_FINAL);
1280                 req->r2t_length = be32_to_cpu(req_hdr->data_length) - req->pdu.datasize;
1281         }
1282         req->target_task_tag = get_next_ttt(conn);
1283         req->sg = scst_cmd_get_sg(scst_cmd);
1284         req->bufflen = scst_cmd_get_bufflen(scst_cmd);
1285         if (req->r2t_length > req->bufflen) {
1286                 PRINT_ERROR("req->r2t_length %d > req->bufflen %d",
1287                         req->r2t_length, req->bufflen);
1288                 req->r2t_length = req->bufflen;
1289         }
1290
1291         TRACE_DBG("req=%p, dir=%d, is_unsolicited_data=%d, "
1292                 "r2t_length=%d, bufflen=%d", req, dir,
1293                 req->is_unsolicited_data, req->r2t_length, req->bufflen);
1294
1295         if (!session->sess_param.immediate_data &&
1296             req->pdu.datasize) {
1297                 PRINT_ERROR("Initiator %s violated negotiated paremeters: "
1298                         "forbidden immediate data sent (ITT %x, op  %x)",
1299                         session->initiator_name, cmnd_itt(req), req_hdr->scb[0]);
1300                 res = -EINVAL;
1301                 goto out;
1302         }
1303
1304         if (session->sess_param.initial_r2t &&
1305             !(req_hdr->flags & ISCSI_CMD_FINAL)) {
1306                 PRINT_ERROR("Initiator %s violated negotiated paremeters: "
1307                         "initial R2T is required (ITT %x, op  %x)",
1308                         session->initiator_name, cmnd_itt(req), req_hdr->scb[0]);
1309                 res = -EINVAL;
1310                 goto out;
1311         }
1312
1313         if (req->pdu.datasize) {
1314                 if (dir != SCST_DATA_WRITE) {
1315                         PRINT_ERROR("pdu.datasize(%d) >0, but dir(%x) isn't WRITE",
1316                                 req->pdu.datasize, dir);
1317                         create_sense_rsp(req, ABORTED_COMMAND, 0xc, 0xc);
1318                         cmnd_prepare_skip_pdu_set_resid(req);
1319                 } else
1320                         res = cmnd_prepare_recv_pdu(conn, req, 0, req->pdu.datasize);
1321         }
1322 out:
1323         /* Aborted commands will be freed in cmnd_rx_end() */
1324         TRACE_EXIT_RES(res);
1325         return res;
1326 }
1327
1328 static int data_out_start(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
1329 {
1330         struct iscsi_data_out_hdr *req_hdr = (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1331         struct iscsi_cmnd *req = NULL;
1332         u32 offset = be32_to_cpu(req_hdr->buffer_offset);
1333         int res = 0;
1334
1335         TRACE_ENTRY();
1336
1337         /*
1338          * There is no race with send_r2t() and __cmnd_abort(), since
1339          * all functions called from single read thread
1340          */
1341         iscsi_extracheck_is_rd_thread(cmnd->conn);
1342
1343         update_stat_sn(cmnd);
1344
1345         cmnd->cmd_req = req = cmnd_find_hash(conn->session, req_hdr->itt,
1346                                         req_hdr->ttt);
1347         if (!req) {
1348                 /* It might happen if req was aborted and then freed */
1349                 TRACE(TRACE_MGMT_MINOR, "Unable to find scsi task %x %x",
1350                         cmnd_itt(cmnd), cmnd_ttt(cmnd));
1351                 goto skip_pdu;
1352         }
1353
1354         if (req->r2t_length < cmnd->pdu.datasize) {
1355                 PRINT_ERROR("Invalid data len %x %u %u", cmnd_itt(req),
1356                         cmnd->pdu.datasize, req->r2t_length);
1357                 mark_conn_closed(conn);
1358                 res = -EINVAL;
1359                 goto out;
1360         }
1361
1362         if (req->r2t_length + offset != cmnd_write_size(req)) {
1363                 PRINT_ERROR("Wrong cmd lengths (%x %u %u %u)",
1364                         cmnd_itt(req), req->r2t_length,
1365                         offset, cmnd_write_size(req));
1366                 mark_conn_closed(conn);
1367                 res = -EINVAL;
1368                 goto out;
1369         }
1370
1371         req->r2t_length -= cmnd->pdu.datasize;
1372
1373         /* Check unsolicited burst data */
1374         if ((req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) &&
1375             (req->pdu.bhs.flags & ISCSI_FLG_FINAL)) {
1376                 PRINT_ERROR("unexpected data from %x %x",
1377                         cmnd_itt(cmnd), cmnd_ttt(cmnd));
1378                 mark_conn_closed(conn);
1379                 res = -EINVAL;
1380                 goto out;
1381         }
1382
1383         TRACE(TRACE_D_WRITE, "%u %p %p %u %u", req_hdr->ttt, cmnd, req,
1384                 offset, cmnd->pdu.datasize);
1385
1386         res = cmnd_prepare_recv_pdu(conn, req, offset, cmnd->pdu.datasize);
1387
1388 out:
1389         TRACE_EXIT_RES(res);
1390         return res;
1391
1392 skip_pdu:
1393         cmnd->pdu.bhs.opcode = ISCSI_OP_DATA_REJECT;
1394         cmnd_prepare_skip_pdu(cmnd);
1395         goto out;
1396 }
1397
1398 static void data_out_end(struct iscsi_cmnd *cmnd)
1399 {
1400         struct iscsi_data_out_hdr *req_hdr = (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1401         struct iscsi_cmnd *req;
1402
1403         sBUG_ON(cmnd == NULL);
1404         req = cmnd->cmd_req;
1405         sBUG_ON(req == NULL);
1406
1407         TRACE_DBG("cmnd %p, req %p", cmnd, req);
1408
1409         iscsi_extracheck_is_rd_thread(cmnd->conn);
1410
1411         if (!(cmnd->conn->ddigest_type & DIGEST_NONE)) {
1412                 TRACE_DBG("Adding RX ddigest cmd %p to digest list "
1413                         "of req %p", cmnd, req);
1414                 list_add_tail(&cmnd->rx_ddigest_cmd_list_entry,
1415                         &req->rx_ddigest_cmd_list);
1416                 cmnd_get(cmnd);
1417         }
1418
1419         if (req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1420                 TRACE_DBG("ISCSI_RESERVED_TAG, FINAL %x",
1421                         req_hdr->flags & ISCSI_FLG_FINAL);
1422                 if (req_hdr->flags & ISCSI_FLG_FINAL) {
1423                         req->is_unsolicited_data = 0;
1424                         if (!req->pending)
1425                                 scsi_cmnd_exec(req);
1426                 }
1427         } else {
1428                 TRACE_DBG("FINAL %x, outstanding_r2t %d, "
1429                         "r2t_length %d", req_hdr->flags & ISCSI_FLG_FINAL,
1430                         req->outstanding_r2t, req->r2t_length);
1431                 /* ToDo : proper error handling */
1432                 if (!(req_hdr->flags & ISCSI_FLG_FINAL) && (req->r2t_length == 0))
1433                         PRINT_ERROR("initiator error %x", cmnd_itt(req));
1434
1435                 if (!(req_hdr->flags & ISCSI_FLG_FINAL))
1436                         goto out;
1437
1438                 req->outstanding_r2t--;
1439
1440                 scsi_cmnd_exec(req);
1441         }
1442
1443 out:
1444         cmnd_put(cmnd);
1445         return;
1446 }
1447
1448 /*
1449  * Must be called from the read thread. Also called under cmd_list_lock,
1450  * but may drop it inside.
1451  *
1452  * Returns >0 if cmd_list_lock was dropped inside, 0 otherwise.
1453  */
1454 static inline int __cmnd_abort(struct iscsi_cmnd *cmnd)
1455 {
1456         int res = 0;
1457
1458         /* Check to avoid double release */
1459         if (cmnd->tmfabort)
1460                 goto out;
1461
1462         TRACE_MGMT_DBG("Aborting cmd %p, scst_cmd %p (scst state %x, "
1463                 "ref_cnt %d, itt %x, sn %d, op %x, r2t_len %x, CDB op %x, "
1464                 "size to write %u, is_unsolicited_data %u, "
1465                 "outstanding_r2t %u)", cmnd, cmnd->scst_cmd,
1466                 cmnd->scst_state, atomic_read(&cmnd->ref_cnt),
1467                 cmnd_itt(cmnd), cmnd->pdu.bhs.sn, cmnd_opcode(cmnd),
1468                 cmnd->r2t_length, cmnd_scsicode(cmnd), cmnd_write_size(cmnd),
1469                 cmnd->is_unsolicited_data, cmnd->outstanding_r2t);
1470
1471 #ifdef NET_PAGE_CALLBACKS_DEFINED
1472         TRACE_MGMT_DBG("net_ref_cnt %d", atomic_read(&cmnd->net_ref_cnt));
1473 #endif
1474
1475         iscsi_extracheck_is_rd_thread(cmnd->conn);
1476
1477         cmnd->tmfabort = 1;
1478
1479         if (cmnd->data_waiting) {
1480                 struct iscsi_conn *conn = cmnd->conn;
1481                 res = 1;
1482                 spin_unlock_bh(&conn->cmd_list_lock);
1483                 TRACE_MGMT_DBG("Releasing data waiting cmd %p", cmnd);
1484                 req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
1485                 /*
1486                  * We are in the read thread, so we may not worry that after
1487                  * cmnd release conn gets released as well.
1488                  */
1489                 spin_lock_bh(&conn->cmd_list_lock);
1490         }
1491
1492 out:
1493         return res;
1494 }
1495
1496 /* Must be called from the read thread */
1497 static int cmnd_abort(struct iscsi_cmnd *req)
1498 {
1499         struct iscsi_session *session = req->conn->session;
1500         struct iscsi_task_mgt_hdr *req_hdr =
1501                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1502         struct iscsi_cmnd *cmnd;
1503         int err;
1504
1505         if ((cmnd = cmnd_find_hash_get(session, req_hdr->rtt, ISCSI_RESERVED_TAG))) {
1506                 struct iscsi_conn *conn = cmnd->conn;
1507                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
1508
1509                 if (req_hdr->lun != hdr->lun) {
1510                          PRINT_ERROR("ABORT TASK: LUN mismatch: req LUN "
1511                                 "%Lx, cmd LUN %Lx, rtt %u", req_hdr->lun,
1512                                 hdr->lun, req_hdr->rtt);
1513                         err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1514                         goto out_put;
1515                 }
1516
1517                 if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
1518                     (req_hdr->cmd_sn == hdr->cmd_sn)) {
1519                         PRINT_ERROR("ABORT TASK: SN mismatch: req SN %x, "
1520                                 "cmd SN %x, rtt %u", req_hdr->cmd_sn,
1521                                 hdr->cmd_sn, req_hdr->rtt);
1522                         err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1523                         goto out_put;
1524                 }
1525
1526                 spin_lock_bh(&conn->cmd_list_lock);
1527                 __cmnd_abort(cmnd);
1528                 spin_unlock_bh(&conn->cmd_list_lock);
1529
1530                 cmnd_put(cmnd);
1531                 err = 0;
1532         } else
1533                 err = ISCSI_RESPONSE_UNKNOWN_TASK;
1534
1535 out:
1536         return err;
1537
1538 out_put:
1539         cmnd_put(cmnd);
1540         goto out;
1541 }
1542
1543 /* Must be called from the read thread */
1544 static int target_abort(struct iscsi_cmnd *req, int all)
1545 {
1546         struct iscsi_target *target = req->conn->session->target;
1547         struct iscsi_task_mgt_hdr *req_hdr =
1548                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1549         struct iscsi_session *session;
1550         struct iscsi_conn *conn;
1551         struct iscsi_cmnd *cmnd;
1552
1553         mutex_lock(&target->target_mutex);
1554
1555         list_for_each_entry(session, &target->session_list, session_list_entry) {
1556                 list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
1557                         spin_lock_bh(&conn->cmd_list_lock);
1558 again:
1559                         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1560                                 int again = 0;
1561                                 if (cmnd == req)
1562                                         continue;
1563                                 if (all)
1564                                         again = __cmnd_abort(cmnd);
1565                                 else if (req_hdr->lun == cmnd_hdr(cmnd)->lun)
1566                                         again = __cmnd_abort(cmnd);
1567                                 if (again)
1568                                         goto again;
1569                         }
1570                         spin_unlock_bh(&conn->cmd_list_lock);
1571                 }
1572         }
1573
1574         mutex_unlock(&target->target_mutex);
1575         return 0;
1576 }
1577
1578 /* Must be called from the read thread */
1579 static void task_set_abort(struct iscsi_cmnd *req)
1580 {
1581         struct iscsi_session *session = req->conn->session;
1582         struct iscsi_task_mgt_hdr *req_hdr =
1583                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1584         struct iscsi_target *target = session->target;
1585         struct iscsi_conn *conn;
1586         struct iscsi_cmnd *cmnd;
1587
1588         mutex_lock(&target->target_mutex);
1589
1590         list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
1591                 spin_lock_bh(&conn->cmd_list_lock);
1592 again:
1593                 list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1594                         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
1595                         if (cmnd == req)
1596                                 continue;
1597                         if (req_hdr->lun != hdr->lun)
1598                                 continue;
1599                         if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
1600                             req_hdr->cmd_sn == hdr->cmd_sn)
1601                                 continue;
1602                         if (__cmnd_abort(cmnd))
1603                                 goto again;
1604                 }
1605                 spin_unlock_bh(&conn->cmd_list_lock);
1606         }
1607
1608         mutex_unlock(&target->target_mutex);
1609         return;
1610 }
1611
1612 /* Must be called from the read thread */
1613 void conn_abort(struct iscsi_conn *conn)
1614 {
1615         struct iscsi_cmnd *cmnd;
1616
1617         TRACE_MGMT_DBG("Aborting conn %p", conn);
1618
1619         spin_lock_bh(&conn->cmd_list_lock);
1620 again:
1621         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1622                 if (__cmnd_abort(cmnd))
1623                         goto again;
1624         }
1625         spin_unlock_bh(&conn->cmd_list_lock);
1626 }
1627
1628 static void execute_task_management(struct iscsi_cmnd *req)
1629 {
1630         struct iscsi_conn *conn = req->conn;
1631         struct iscsi_session *sess = conn->session;
1632         struct iscsi_task_mgt_hdr *req_hdr =
1633                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1634         int err = 0, function = req_hdr->function & ISCSI_FUNCTION_MASK;
1635         struct scst_rx_mgmt_params params;
1636
1637         TRACE((function == ISCSI_FUNCTION_ABORT_TASK) ? TRACE_MGMT_MINOR : TRACE_MGMT,
1638                 "TM cmd: req %p, itt %x, fn %d, rtt %x, sn %d", req, cmnd_itt(req),
1639                 function, req_hdr->rtt, req_hdr->cmd_sn);
1640
1641         spin_lock(&sess->sn_lock);
1642         sess->tm_active = 1;
1643         sess->tm_sn = req_hdr->cmd_sn;
1644         if (sess->tm_rsp != NULL) {
1645                 struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
1646                 TRACE(TRACE_MGMT_MINOR, "Dropping delayed TM rsp %p", tm_rsp);
1647                 sess->tm_rsp = NULL;
1648                 spin_unlock(&sess->sn_lock);
1649                 rsp_cmnd_release(tm_rsp);
1650         } else
1651                 spin_unlock(&sess->sn_lock);
1652
1653         memset(&params, 0, sizeof(params));
1654         params.atomic = SCST_NON_ATOMIC;
1655         params.tgt_priv = req;
1656
1657         if ((function != ISCSI_FUNCTION_ABORT_TASK) &&
1658             (req_hdr->rtt != ISCSI_RESERVED_TAG)) {
1659                 PRINT_ERROR("Invalid RTT %x (TM fn %x)", req_hdr->rtt,
1660                         function);
1661                 err = -1;
1662                 goto reject;
1663         }
1664
1665         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
1666
1667         switch (function) {
1668         case ISCSI_FUNCTION_ABORT_TASK:
1669                 err = cmnd_abort(req);
1670                 if (err == 0) {
1671                         params.fn = SCST_ABORT_TASK;
1672                         params.tag = req_hdr->rtt;
1673                         params.tag_set = 1;
1674                         params.lun = (uint8_t *)&req_hdr->lun;
1675                         params.lun_len = sizeof(req_hdr->lun);
1676                         params.lun_set = 1;
1677                         params.cmd_sn = req_hdr->cmd_sn;
1678                         params.cmd_sn_set = 1;
1679                         err = scst_rx_mgmt_fn(conn->session->scst_sess,
1680                                 &params);
1681                 }
1682                 break;
1683         case ISCSI_FUNCTION_ABORT_TASK_SET:
1684                 task_set_abort(req);
1685                 params.fn = SCST_ABORT_TASK_SET;
1686                 params.lun = (uint8_t *)&req_hdr->lun;
1687                 params.lun_len = sizeof(req_hdr->lun);
1688                 params.lun_set = 1;
1689                 params.cmd_sn = req_hdr->cmd_sn;
1690                 params.cmd_sn_set = 1;
1691                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1692                         &params);
1693                 break;
1694         case ISCSI_FUNCTION_CLEAR_TASK_SET:
1695                 task_set_abort(req);
1696                 params.fn = SCST_CLEAR_TASK_SET;
1697                 params.lun = (uint8_t *)&req_hdr->lun;
1698                 params.lun_len = sizeof(req_hdr->lun);
1699                 params.lun_set = 1;
1700                 params.cmd_sn = req_hdr->cmd_sn;
1701                 params.cmd_sn_set = 1;
1702                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1703                         &params);
1704                 break;
1705         case ISCSI_FUNCTION_CLEAR_ACA:
1706                 params.fn = SCST_CLEAR_ACA;
1707                 params.lun = (uint8_t *)&req_hdr->lun;
1708                 params.lun_len = sizeof(req_hdr->lun);
1709                 params.lun_set = 1;
1710                 params.cmd_sn = req_hdr->cmd_sn;
1711                 params.cmd_sn_set = 1;
1712                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1713                         &params);
1714                 break;
1715         case ISCSI_FUNCTION_TARGET_COLD_RESET:
1716         case ISCSI_FUNCTION_TARGET_WARM_RESET:
1717                 target_abort(req, 1);
1718                 params.fn = SCST_TARGET_RESET;
1719                 params.cmd_sn = req_hdr->cmd_sn;
1720                 params.cmd_sn_set = 1;
1721                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1722                         &params);
1723                 break;
1724         case ISCSI_FUNCTION_LOGICAL_UNIT_RESET:
1725                 target_abort(req, 0);
1726                 params.fn = SCST_LUN_RESET;
1727                 params.lun = (uint8_t *)&req_hdr->lun;
1728                 params.lun_len = sizeof(req_hdr->lun);
1729                 params.lun_set = 1;
1730                 params.cmd_sn = req_hdr->cmd_sn;
1731                 params.cmd_sn_set = 1;
1732                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1733                         &params);
1734                 break;
1735         case ISCSI_FUNCTION_TASK_REASSIGN:
1736                 iscsi_send_task_mgmt_resp(req, 
1737                         ISCSI_RESPONSE_FUNCTION_UNSUPPORTED);
1738                 break;
1739         default:
1740                 iscsi_send_task_mgmt_resp(req,
1741                         ISCSI_RESPONSE_FUNCTION_REJECTED);
1742                 break;
1743         }
1744
1745 reject:
1746         if (err != 0) {
1747                 iscsi_send_task_mgmt_resp(req,
1748                         ISCSI_RESPONSE_FUNCTION_REJECTED);
1749         }
1750 }
1751
1752 static void noop_out_exec(struct iscsi_cmnd *req)
1753 {
1754         struct iscsi_cmnd *rsp;
1755         struct iscsi_nop_in_hdr *rsp_hdr;
1756
1757         TRACE_DBG("%p", req);
1758
1759         if (cmnd_itt(req) != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1760                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
1761
1762                 rsp_hdr = (struct iscsi_nop_in_hdr *)&rsp->pdu.bhs;
1763                 rsp_hdr->opcode = ISCSI_OP_NOOP_IN;
1764                 rsp_hdr->flags = ISCSI_FLG_FINAL;
1765                 rsp_hdr->itt = req->pdu.bhs.itt;
1766                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
1767
1768                 if (req->pdu.datasize)
1769                         sBUG_ON(req->sg == NULL);
1770                 else
1771                         sBUG_ON(req->sg != NULL);
1772
1773                 if (req->sg) {
1774                         rsp->sg = req->sg;
1775                         rsp->bufflen = req->bufflen;
1776                 }
1777
1778                 sBUG_ON(get_pgcnt(req->pdu.datasize, 0) > ISCSI_CONN_IOV_MAX);
1779                 rsp->pdu.datasize = req->pdu.datasize;
1780                 iscsi_cmnd_init_write(rsp,
1781                         ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
1782                 req_cmnd_release(req);
1783         } else
1784                 cmnd_put(req);
1785 }
1786
1787 static void logout_exec(struct iscsi_cmnd *req)
1788 {
1789         struct iscsi_logout_req_hdr *req_hdr;
1790         struct iscsi_cmnd *rsp;
1791         struct iscsi_logout_rsp_hdr *rsp_hdr;
1792
1793         PRINT_INFO("Logout received from initiator %s",
1794                 req->conn->session->initiator_name);
1795         TRACE_DBG("%p", req);
1796
1797         req_hdr = (struct iscsi_logout_req_hdr *)&req->pdu.bhs;
1798         rsp = iscsi_cmnd_create_rsp_cmnd(req);
1799         rsp_hdr = (struct iscsi_logout_rsp_hdr *)&rsp->pdu.bhs;
1800         rsp_hdr->opcode = ISCSI_OP_LOGOUT_RSP;
1801         rsp_hdr->flags = ISCSI_FLG_FINAL;
1802         rsp_hdr->itt = req_hdr->itt;
1803         rsp->should_close_conn = 1;
1804         iscsi_cmnd_init_write(rsp,
1805                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
1806         req_cmnd_release(req);
1807 }
1808
1809 static void iscsi_cmnd_exec(struct iscsi_cmnd *cmnd)
1810 {
1811         TRACE_DBG("%p,%x,%u", cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn);
1812
1813         if (unlikely(cmnd->tmfabort)) {
1814                 TRACE_MGMT_DBG("cmnd %p (scst_cmd %p) aborted", cmnd,
1815                         cmnd->scst_cmd);
1816                 req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
1817                 goto out;
1818         }
1819
1820         switch (cmnd_opcode(cmnd)) {
1821         case ISCSI_OP_NOOP_OUT:
1822                 noop_out_exec(cmnd);
1823                 break;
1824         case ISCSI_OP_SCSI_CMD:
1825                 scsi_cmnd_exec(cmnd);
1826                 break;
1827         case ISCSI_OP_SCSI_TASK_MGT_MSG:
1828                 execute_task_management(cmnd);
1829                 break;
1830         case ISCSI_OP_LOGOUT_CMD:
1831                 logout_exec(cmnd);
1832                 break;
1833         case ISCSI_OP_SCSI_REJECT:
1834                 TRACE_MGMT_DBG("REJECT cmnd %p (scst_cmd %p)", cmnd,
1835                         cmnd->scst_cmd);
1836                 iscsi_cmnd_init_write(get_rsp_cmnd(cmnd),
1837                         ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
1838                 req_cmnd_release(cmnd);
1839                 break;
1840         default:
1841                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
1842                 req_cmnd_release(cmnd);
1843                 break;
1844         }
1845 out:
1846         return;
1847 }
1848
1849 static void __cmnd_send_pdu(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd,
1850         u32 offset, u32 size)
1851 {
1852         TRACE_DBG("%p %u,%u,%u", cmnd, offset, size, cmnd->bufflen);
1853
1854         iscsi_extracheck_is_wr_thread(conn);
1855
1856         sBUG_ON(offset > cmnd->bufflen);
1857         sBUG_ON(offset + size > cmnd->bufflen);
1858
1859         conn->write_offset = offset;
1860         conn->write_size += size;
1861 }
1862
1863 static void cmnd_send_pdu(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
1864 {
1865         u32 size;
1866
1867         if (!cmnd->pdu.datasize)
1868                 return;
1869
1870         size = (cmnd->pdu.datasize + 3) & -4;
1871         sBUG_ON(cmnd->sg == NULL);
1872         sBUG_ON(cmnd->bufflen != size);
1873         __cmnd_send_pdu(conn, cmnd, 0, size);
1874 }
1875
1876 static void set_cork(struct socket *sock, int on)
1877 {
1878         int opt = on;
1879         mm_segment_t oldfs;
1880
1881         oldfs = get_fs();
1882         set_fs(get_ds());
1883         sock->ops->setsockopt(sock, SOL_TCP, TCP_CORK, (void *)&opt, sizeof(opt));
1884         set_fs(oldfs);
1885 }
1886
1887 void cmnd_tx_start(struct iscsi_cmnd *cmnd)
1888 {
1889         struct iscsi_conn *conn = cmnd->conn;
1890
1891         TRACE_DBG("%p:%p:%x", conn, cmnd, cmnd_opcode(cmnd));
1892         iscsi_cmnd_set_length(&cmnd->pdu);
1893
1894         iscsi_extracheck_is_wr_thread(conn);
1895
1896         set_cork(conn->sock, 1);
1897
1898         conn->write_iop = conn->write_iov;
1899         conn->write_iop->iov_base = &cmnd->pdu.bhs;
1900         conn->write_iop->iov_len = sizeof(cmnd->pdu.bhs);
1901         conn->write_iop_used = 1;
1902         conn->write_size = sizeof(cmnd->pdu.bhs);
1903
1904         switch (cmnd_opcode(cmnd)) {
1905         case ISCSI_OP_NOOP_IN:
1906                 cmnd_set_sn(cmnd, 1);
1907                 cmnd_send_pdu(conn, cmnd);
1908                 break;
1909         case ISCSI_OP_SCSI_RSP:
1910                 cmnd_set_sn(cmnd, 1);
1911                 cmnd_send_pdu(conn, cmnd);
1912                 break;
1913         case ISCSI_OP_SCSI_TASK_MGT_RSP:
1914                 cmnd_set_sn(cmnd, 1);
1915                 break;
1916         case ISCSI_OP_TEXT_RSP:
1917                 cmnd_set_sn(cmnd, 1);
1918                 break;
1919         case ISCSI_OP_SCSI_DATA_IN:
1920         {
1921                 struct iscsi_data_in_hdr *rsp = (struct iscsi_data_in_hdr *)&cmnd->pdu.bhs;
1922                 u32 offset = cpu_to_be32(rsp->buffer_offset);
1923
1924                 cmnd_set_sn(cmnd, (rsp->flags & ISCSI_FLG_FINAL) ? 1 : 0);
1925                 __cmnd_send_pdu(conn, cmnd, offset, cmnd->pdu.datasize);
1926                 break;
1927         }
1928         case ISCSI_OP_LOGOUT_RSP:
1929                 cmnd_set_sn(cmnd, 1);
1930                 break;
1931         case ISCSI_OP_R2T:
1932                 cmnd->pdu.bhs.sn = cmnd_set_sn(cmnd, 0);
1933                 break;
1934         case ISCSI_OP_ASYNC_MSG:
1935                 cmnd_set_sn(cmnd, 1);
1936                 break;
1937         case ISCSI_OP_REJECT:
1938                 cmnd_set_sn(cmnd, 1);
1939                 cmnd_send_pdu(conn, cmnd);
1940                 break;
1941         default:
1942                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
1943                 break;
1944         }
1945
1946         // move this?
1947         conn->write_size = (conn->write_size + 3) & -4;
1948         iscsi_dump_pdu(&cmnd->pdu);
1949 }
1950
1951 void cmnd_tx_end(struct iscsi_cmnd *cmnd)
1952 {
1953         struct iscsi_conn *conn = cmnd->conn;
1954
1955         TRACE_DBG("%p:%x (should_close_conn %d)", cmnd, cmnd_opcode(cmnd),
1956                 cmnd->should_close_conn);
1957
1958         switch (cmnd_opcode(cmnd)) {
1959         case ISCSI_OP_NOOP_IN:
1960         case ISCSI_OP_SCSI_RSP:
1961         case ISCSI_OP_SCSI_TASK_MGT_RSP:
1962         case ISCSI_OP_TEXT_RSP:
1963         case ISCSI_OP_R2T:
1964         case ISCSI_OP_ASYNC_MSG:
1965         case ISCSI_OP_REJECT:
1966         case ISCSI_OP_SCSI_DATA_IN:
1967         case ISCSI_OP_LOGOUT_RSP:
1968                 break;
1969         default:
1970                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
1971                 sBUG();
1972                 break;
1973         }
1974
1975         if (cmnd->should_close_conn) {
1976                 PRINT_INFO("Closing connection at initiator %s request",
1977                         conn->session->initiator_name);
1978                 mark_conn_closed(conn);
1979         }
1980
1981         set_cork(cmnd->conn->sock, 0);
1982 }
1983
1984 /*
1985  * Push the command for execution. This functions reorders the commands.
1986  * Called from the read thread.
1987  */
1988 static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd)
1989 {
1990         struct iscsi_session *session = cmnd->conn->session;
1991         struct list_head *entry;
1992         u32 cmd_sn;
1993
1994         TRACE_DBG("%p:%x %u,%u",
1995                 cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn, session->exp_cmd_sn);
1996
1997         iscsi_extracheck_is_rd_thread(cmnd->conn);
1998
1999         if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
2000                 iscsi_cmnd_exec(cmnd);
2001                 return;
2002         }
2003
2004         spin_lock(&session->sn_lock);
2005
2006         cmd_sn = cmnd->pdu.bhs.sn;
2007         if (cmd_sn == session->exp_cmd_sn) {
2008                 while (1) {
2009                         session->exp_cmd_sn = ++cmd_sn;
2010
2011                         if (unlikely(session->tm_active)) {
2012                                 if (before(cmd_sn, session->tm_sn)) {
2013                                         struct iscsi_conn *conn = cmnd->conn;
2014
2015                                         spin_unlock(&session->sn_lock);
2016
2017                                         spin_lock_bh(&conn->cmd_list_lock);
2018                                         __cmnd_abort(cmnd);
2019                                         spin_unlock_bh(&conn->cmd_list_lock);
2020
2021                                         spin_lock(&session->sn_lock);
2022                                 }
2023                                 iscsi_check_send_delayed_tm_resp(session);
2024                         }
2025
2026                         spin_unlock(&session->sn_lock);
2027
2028                         iscsi_cmnd_exec(cmnd);
2029
2030                         if (list_empty(&session->pending_list))
2031                                 break;
2032                         cmnd = list_entry(session->pending_list.next, struct iscsi_cmnd,
2033                                                 pending_list_entry);
2034                         if (cmnd->pdu.bhs.sn != cmd_sn)
2035                                 break;
2036                         list_del(&cmnd->pending_list_entry);
2037                         cmnd->pending = 0;
2038
2039                         spin_lock(&session->sn_lock);
2040                 }
2041         } else {
2042                 cmnd->pending = 1;
2043                 if (before(cmd_sn, session->exp_cmd_sn)) {
2044                         PRINT_ERROR("unexpected cmd_sn (%u,%u)", cmd_sn,
2045                                 session->exp_cmd_sn);
2046                 }
2047
2048                 if (after(cmd_sn, session->exp_cmd_sn + iscsi_get_allowed_cmds(session))) {
2049                         PRINT_INFO("Suspicious: too large cmd_sn %u (exp_cmd_sn %u, "
2050                                 "max_sn %u)", cmd_sn, session->exp_cmd_sn,
2051                                 iscsi_get_allowed_cmds(session));
2052                 }
2053
2054                 spin_unlock(&session->sn_lock);
2055
2056                 list_for_each(entry, &session->pending_list) {
2057                         struct iscsi_cmnd *tmp = list_entry(entry, struct iscsi_cmnd,
2058                                                         pending_list_entry);
2059                         if (before(cmd_sn, tmp->pdu.bhs.sn))
2060                                 break;
2061                 }
2062
2063                 list_add_tail(&cmnd->pending_list_entry, entry);
2064         }
2065
2066         return;
2067 }
2068
2069 static int check_segment_length(struct iscsi_cmnd *cmnd)
2070 {
2071         struct iscsi_conn *conn = cmnd->conn;
2072         struct iscsi_session *session = conn->session;
2073
2074         if (cmnd->pdu.datasize > session->sess_param.max_recv_data_length) {
2075                 PRINT_ERROR("Initiator %s violated negotiated parameters: "
2076                         "data too long (ITT %x, datasize %u, "
2077                         "max_recv_data_length %u", session->initiator_name,
2078                         cmnd_itt(cmnd), cmnd->pdu.datasize,
2079                         session->sess_param.max_recv_data_length);
2080                 mark_conn_closed(conn);
2081                 return -EINVAL;
2082         }
2083         return 0;
2084 }
2085
2086 int cmnd_rx_start(struct iscsi_cmnd *cmnd)
2087 {
2088         struct iscsi_conn *conn = cmnd->conn;
2089         int res, rc;
2090
2091         iscsi_dump_pdu(&cmnd->pdu);
2092
2093         res = check_segment_length(cmnd);
2094         if (res != 0)
2095                 goto out;
2096
2097         switch (cmnd_opcode(cmnd)) {
2098         case ISCSI_OP_NOOP_OUT:
2099                 rc = noop_out_start(cmnd);
2100                 break;
2101         case ISCSI_OP_SCSI_CMD:
2102                 rc = cmnd_insert_hash(cmnd);
2103                 if (likely(rc == 0)) {
2104                         res = scsi_cmnd_start(cmnd);
2105                         if (unlikely(res != 0))
2106                                 goto out;
2107                 }
2108                 break;
2109         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2110                 rc = cmnd_insert_hash(cmnd);
2111                 break;
2112         case ISCSI_OP_SCSI_DATA_OUT:
2113                 res = data_out_start(conn, cmnd);
2114                 rc = 0; /* to avoid compiler warning */
2115                 if (unlikely(res != 0))
2116                         goto out;
2117                 break;
2118         case ISCSI_OP_LOGOUT_CMD:
2119                 rc = cmnd_insert_hash(cmnd);
2120                 break;
2121         case ISCSI_OP_TEXT_CMD:
2122         case ISCSI_OP_SNACK_CMD:
2123                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2124                 break;
2125         default:
2126                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2127                 break;
2128         }
2129
2130         if (unlikely(rc < 0)) {
2131                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
2132                 PRINT_ERROR("Error %d (iSCSI opcode %x, ITT %x, op %x)", rc,
2133                         cmnd_opcode(cmnd), cmnd_itt(cmnd),
2134                         (cmnd_opcode(cmnd) == ISCSI_OP_SCSI_CMD ?
2135                                 hdr->scb[0] : -1));
2136                 iscsi_cmnd_reject(cmnd, -rc);
2137         }
2138
2139 out:
2140         TRACE_EXIT_RES(res);
2141         return res;
2142 }
2143
2144 void cmnd_rx_end(struct iscsi_cmnd *cmnd)
2145 {
2146         TRACE_DBG("%p:%x", cmnd, cmnd_opcode(cmnd));
2147
2148         switch (cmnd_opcode(cmnd)) {
2149         case ISCSI_OP_SCSI_REJECT:
2150         case ISCSI_OP_NOOP_OUT:
2151         case ISCSI_OP_SCSI_CMD:
2152         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2153         case ISCSI_OP_LOGOUT_CMD:
2154                 iscsi_session_push_cmnd(cmnd);
2155                 break;
2156         case ISCSI_OP_SCSI_DATA_OUT:
2157                 data_out_end(cmnd);
2158                 break;
2159         case ISCSI_OP_PDU_REJECT:
2160                 iscsi_cmnd_init_write(get_rsp_cmnd(cmnd),
2161                         ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2162                 req_cmnd_release(cmnd);
2163                 break;
2164         case ISCSI_OP_DATA_REJECT:
2165                 req_cmnd_release(cmnd);
2166                 break;
2167         default:
2168                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2169                 req_cmnd_release(cmnd);
2170                 break;
2171         }
2172         return;
2173 }
2174
2175 #ifndef NET_PAGE_CALLBACKS_DEFINED
2176 static int iscsi_alloc_data_buf(struct scst_cmd *cmd)
2177 {
2178         if (scst_cmd_get_data_direction(cmd) == SCST_DATA_READ) {
2179                 /*
2180                  * sock->ops->sendpage() is async zero copy operation,
2181                  * so we must be sure not to free and reuse
2182                  * the command's buffer before the sending was completed
2183                  * by the network layers. It is possible only if we
2184                  * don't use SGV cache.
2185                  */
2186                 scst_cmd_set_no_sgv(cmd);
2187         }
2188         return 1;
2189 }
2190 #endif
2191
2192 static inline void iscsi_set_state_wake_up(struct iscsi_cmnd *req,
2193         int new_state)
2194 {
2195         /*
2196          * We use wait_event() to wait for the state change, but it checks its
2197          * condition without any protection, so without cmnd_get() it is
2198          * possible that req will die "immediately" after the state assignment
2199          * and wake_up() will operate on dead data.
2200          */
2201         cmnd_get_ordered(req);
2202         req->scst_state = new_state;
2203         wake_up(&req->scst_waitQ);
2204         cmnd_put(req);
2205         return;
2206 }
2207
2208 static void iscsi_preprocessing_done(struct scst_cmd *scst_cmd)
2209 {
2210         struct iscsi_cmnd *req = (struct iscsi_cmnd*)
2211                                 scst_cmd_get_tgt_priv(scst_cmd);
2212
2213         TRACE_DBG("req %p", req);
2214
2215         EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_RX_CMD);
2216
2217         iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_AFTER_PREPROC);
2218         return;
2219 }
2220
2221 /* 
2222  * No locks.
2223  *
2224  * IMPORTANT! Connection conn must be protected by additional conn_get()
2225  * upon entrance in this function, because otherwise it could be destroyed
2226  * inside as a result of iscsi_send(), which releases sent commands.
2227  */
2228 static void iscsi_try_local_processing(struct iscsi_conn *conn)
2229 {
2230         int local;
2231
2232         TRACE_ENTRY();
2233
2234         spin_lock_bh(&iscsi_wr_lock);
2235         switch(conn->wr_state) {
2236         case ISCSI_CONN_WR_STATE_IN_LIST:
2237                 list_del(&conn->wr_list_entry);
2238                 /* go through */
2239         case ISCSI_CONN_WR_STATE_IDLE:
2240 #ifdef EXTRACHECKS
2241                 conn->wr_task = current;
2242 #endif
2243                 conn->wr_state = ISCSI_CONN_WR_STATE_PROCESSING;
2244                 conn->wr_space_ready = 0;
2245                 local = 1;
2246                 break;
2247         default:
2248                 local = 0;
2249                 break;
2250         }
2251         spin_unlock_bh(&iscsi_wr_lock);
2252
2253         if (local) {
2254                 int rc = 1;
2255                 while(test_write_ready(conn)) {
2256                         rc = iscsi_send(conn);
2257                         if (rc <= 0) {
2258                                 break;
2259                         }
2260                 }
2261
2262                 spin_lock_bh(&iscsi_wr_lock);
2263 #ifdef EXTRACHECKS
2264                 conn->wr_task = NULL;
2265 #endif
2266                 if ((rc <= 0) || test_write_ready(conn)) {
2267                         list_add_tail(&conn->wr_list_entry, &iscsi_wr_list);
2268                         conn->wr_state = ISCSI_CONN_WR_STATE_IN_LIST;
2269                         wake_up(&iscsi_wr_waitQ);
2270                 } else
2271                         conn->wr_state = ISCSI_CONN_WR_STATE_IDLE;
2272                 spin_unlock_bh(&iscsi_wr_lock);
2273         }
2274
2275         TRACE_EXIT();
2276         return;
2277 }
2278
2279 static int iscsi_xmit_response(struct scst_cmd *scst_cmd)
2280 {
2281         int resp_flags = scst_cmd_get_tgt_resp_flags(scst_cmd);
2282         struct iscsi_cmnd *req = (struct iscsi_cmnd*)
2283                                         scst_cmd_get_tgt_priv(scst_cmd);
2284         struct iscsi_conn *conn = req->conn;
2285         int status = scst_cmd_get_status(scst_cmd);
2286         u8 *sense = scst_cmd_get_sense_buffer(scst_cmd);
2287         int sense_len = scst_cmd_get_sense_buffer_len(scst_cmd);
2288         int old_state = req->scst_state;
2289
2290         scst_cmd_set_tgt_priv(scst_cmd, NULL);
2291
2292         req->tmfabort |= scst_cmd_aborted(scst_cmd) ? 1 : 0;
2293         if (unlikely(req->tmfabort)) {
2294                 TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
2295                         req->scst_cmd);
2296                 if (old_state == ISCSI_CMD_STATE_RESTARTED) {
2297                         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
2298                         req_cmnd_release_force(req, ISCSI_FORCE_RELEASE_WRITE);
2299                 } else
2300                         iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_PROCESSED);
2301                 goto out;
2302         }
2303
2304         if (unlikely(old_state != ISCSI_CMD_STATE_RESTARTED)) {
2305                 TRACE_DBG("req %p on %d state", req, old_state);
2306                 create_status_rsp(req, status, sense, sense_len);
2307                 switch(old_state) {
2308                 case ISCSI_CMD_STATE_RX_CMD:
2309                 case ISCSI_CMD_STATE_AFTER_PREPROC:
2310                         break;
2311                 default:
2312                         sBUG();
2313                 }
2314                 iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_PROCESSED);
2315                 goto out;
2316         }
2317
2318         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
2319
2320         req->bufflen = scst_cmd_get_resp_data_len(scst_cmd);
2321         req->sg = scst_cmd_get_sg(scst_cmd);
2322
2323         TRACE_DBG("req %p, resp_flags=%x, req->bufflen=%d, req->sg=%p", req,
2324                 resp_flags, req->bufflen, req->sg);
2325
2326         if ((req->bufflen != 0) && !(resp_flags & SCST_TSC_FLAG_STATUS)) {
2327                 PRINT_ERROR("%s", "Sending DATA without STATUS is unsupported");
2328                 scst_set_cmd_error(scst_cmd,
2329                         SCST_LOAD_SENSE(scst_sense_hardw_error));
2330                 resp_flags = scst_cmd_get_tgt_resp_flags(scst_cmd);
2331                 sBUG();
2332         }
2333
2334         if (req->bufflen != 0) {
2335                 /* 
2336                  * Check above makes sure that SCST_TSC_FLAG_STATUS is set,
2337                  * so status is valid here, but in future that could change.
2338                  * ToDo
2339                  */
2340                 if (status != SAM_STAT_CHECK_CONDITION) {
2341                         send_data_rsp(req, status,
2342                                 resp_flags & SCST_TSC_FLAG_STATUS);
2343                 } else {
2344                         struct iscsi_cmnd *rsp;
2345                         struct iscsi_scsi_rsp_hdr *rsp_hdr;
2346                         int resid;
2347                         send_data_rsp(req, 0, 0);
2348                         if (resp_flags & SCST_TSC_FLAG_STATUS) {
2349                                 rsp = create_status_rsp(req, status, sense,
2350                                         sense_len);
2351                                 rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
2352                                 resid = cmnd_read_size(req) - req->bufflen;
2353                                 if (resid > 0) {
2354                                         rsp_hdr->flags |=
2355                                                 ISCSI_FLG_RESIDUAL_UNDERFLOW;
2356                                         rsp_hdr->residual_count = cpu_to_be32(resid);
2357                                 } else if (resid < 0) {
2358                                         rsp_hdr->flags |=
2359                                                 ISCSI_FLG_RESIDUAL_OVERFLOW;
2360                                         rsp_hdr->residual_count = cpu_to_be32(-resid);
2361                                 }
2362                                 iscsi_cmnd_init_write(rsp,
2363                                         ISCSI_INIT_WRITE_REMOVE_HASH);
2364                         }
2365                 }
2366         } else if (resp_flags & SCST_TSC_FLAG_STATUS) {
2367                 struct iscsi_cmnd *rsp;
2368                 struct iscsi_scsi_rsp_hdr *rsp_hdr;
2369                 u32 resid;
2370                 rsp = create_status_rsp(req, status, sense, sense_len);
2371                 rsp_hdr = (struct iscsi_scsi_rsp_hdr *) &rsp->pdu.bhs;
2372                 resid = cmnd_read_size(req);
2373                 if (resid != 0) {
2374                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
2375                         rsp_hdr->residual_count = cpu_to_be32(resid);
2376                 }
2377                 iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH);
2378         }
2379 #ifdef EXTRACHECKS
2380         else
2381                 sBUG();
2382 #endif
2383
2384         conn_get_ordered(conn);
2385         req_cmnd_release(req);
2386         iscsi_try_local_processing(conn);
2387         conn_put(conn);
2388
2389 out:
2390         return SCST_TGT_RES_SUCCESS;
2391 }
2392
2393 /* Called under sn_lock */
2394 static bool iscsi_is_delay_tm_resp(struct iscsi_cmnd *rsp)
2395 {
2396         bool res = 0;
2397         struct iscsi_task_mgt_hdr *req_hdr =
2398                 (struct iscsi_task_mgt_hdr *)&rsp->parent_req->pdu.bhs;
2399         int function = req_hdr->function & ISCSI_FUNCTION_MASK;
2400         struct iscsi_session *sess = rsp->conn->session;
2401
2402         TRACE_ENTRY();
2403
2404         switch(function) {
2405         default:
2406                 if (before(sess->exp_cmd_sn, req_hdr->cmd_sn))
2407                         res = 1;
2408                 break;
2409         }
2410
2411         TRACE_EXIT_RES(res);
2412         return res;
2413 }
2414
2415 /* Called under sn_lock, but might drop it inside, then reaquire */
2416 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess)
2417 {
2418         struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
2419
2420         TRACE_ENTRY();
2421
2422         if (tm_rsp == NULL)
2423                 goto out;
2424
2425         if (iscsi_is_delay_tm_resp(tm_rsp))
2426                 goto out;
2427
2428         TRACE(TRACE_MGMT_MINOR, "Sending delayed rsp %p", tm_rsp);
2429
2430         sess->tm_rsp = NULL;
2431         sess->tm_active = 0;
2432
2433         spin_unlock(&sess->sn_lock);
2434
2435         iscsi_cmnd_init_write(tm_rsp,
2436                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2437
2438         spin_lock(&sess->sn_lock);
2439
2440 out:
2441         TRACE_EXIT();
2442         return;
2443 }
2444
2445 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status)
2446 {
2447         struct iscsi_cmnd *rsp;
2448         struct iscsi_task_mgt_hdr *req_hdr =
2449                                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2450         struct iscsi_task_rsp_hdr *rsp_hdr;
2451         struct iscsi_session *sess = req->conn->session;
2452
2453         TRACE_ENTRY();
2454
2455         TRACE((req_hdr->function == ISCSI_FUNCTION_ABORT_TASK) ?
2456                          TRACE_MGMT_MINOR : TRACE_MGMT,
2457                 "TM req %p finished, status %d", req, status);
2458
2459         rsp = iscsi_cmnd_create_rsp_cmnd(req);
2460         rsp_hdr = (struct iscsi_task_rsp_hdr *)&rsp->pdu.bhs;
2461
2462         rsp_hdr->opcode = ISCSI_OP_SCSI_TASK_MGT_RSP;
2463         rsp_hdr->flags = ISCSI_FLG_FINAL;
2464         rsp_hdr->itt = req_hdr->itt;
2465         rsp_hdr->response = status;
2466
2467         if ((req_hdr->function & ISCSI_FUNCTION_MASK) ==
2468                         ISCSI_FUNCTION_TARGET_COLD_RESET)
2469                 rsp->should_close_conn = 1;
2470
2471         sBUG_ON(sess->tm_rsp != NULL);
2472
2473         spin_lock(&sess->sn_lock);
2474         if (iscsi_is_delay_tm_resp(rsp)) {
2475                 TRACE(TRACE_MGMT_MINOR, "Delaying TM fn %x response %p "
2476                         "(req %p), because not all affected commands received "
2477                         "(cmd sn %d, exp sn %d)",
2478                         req_hdr->function & ISCSI_FUNCTION_MASK, rsp, req,
2479                         req_hdr->cmd_sn, sess->exp_cmd_sn);
2480                 sess->tm_rsp = rsp;
2481                 spin_unlock(&sess->sn_lock);
2482                 goto out_release;
2483         }
2484         sess->tm_active = 0;
2485         spin_unlock(&sess->sn_lock);
2486
2487         iscsi_cmnd_init_write(rsp,
2488                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2489
2490 out_release:
2491         req_cmnd_release(req);
2492
2493         TRACE_EXIT();
2494         return;
2495 }
2496
2497 static inline int iscsi_get_mgmt_response(int status)
2498 {
2499         switch(status) {
2500         case SCST_MGMT_STATUS_SUCCESS:
2501                 return ISCSI_RESPONSE_FUNCTION_COMPLETE;
2502
2503         case SCST_MGMT_STATUS_TASK_NOT_EXIST:
2504                 return ISCSI_RESPONSE_UNKNOWN_TASK;
2505
2506         case SCST_MGMT_STATUS_LUN_NOT_EXIST:
2507                 return ISCSI_RESPONSE_UNKNOWN_LUN;
2508
2509         case SCST_MGMT_STATUS_FN_NOT_SUPPORTED:
2510                 return ISCSI_RESPONSE_FUNCTION_UNSUPPORTED;
2511
2512         case SCST_MGMT_STATUS_REJECTED:
2513         case SCST_MGMT_STATUS_FAILED:
2514         default:
2515                 return ISCSI_RESPONSE_FUNCTION_REJECTED;
2516         }
2517 }
2518
2519 static void iscsi_task_mgmt_fn_done(struct scst_mgmt_cmd *scst_mcmd)
2520 {
2521         struct iscsi_cmnd *req = (struct iscsi_cmnd*)
2522                                 scst_mgmt_cmd_get_tgt_priv(scst_mcmd);
2523         int status = iscsi_get_mgmt_response(scst_mgmt_cmd_get_status(scst_mcmd));
2524
2525         TRACE_MGMT_DBG("req %p, scst_mcmd %p, scst status %d", req, scst_mcmd, 
2526                 scst_mgmt_cmd_get_status(scst_mcmd));
2527
2528         iscsi_send_task_mgmt_resp(req, status);
2529
2530         scst_mgmt_cmd_set_tgt_priv(scst_mcmd, NULL);
2531 }
2532
2533 static int iscsi_target_detect(struct scst_tgt_template *templ)
2534 {
2535         /* Nothing to do */
2536         return 0;
2537 }
2538
2539 static int iscsi_target_release(struct scst_tgt *scst_tgt)
2540 {
2541         /* Nothing to do */
2542         return 0;
2543 }
2544
2545 struct scst_tgt_template iscsi_template = {
2546         .name = "iscsi",
2547         .sg_tablesize = ISCSI_CONN_IOV_MAX,
2548         .threads_num = 0,
2549         .no_clustering = 1,
2550         .xmit_response_atomic = 0,
2551         .preprocessing_done_atomic = 1,
2552         .detect = iscsi_target_detect,
2553         .release = iscsi_target_release,
2554         .xmit_response = iscsi_xmit_response,
2555 #ifndef NET_PAGE_CALLBACKS_DEFINED
2556         .alloc_data_buf = iscsi_alloc_data_buf,
2557 #endif
2558         .preprocessing_done = iscsi_preprocessing_done,
2559         .pre_exec = iscsi_pre_exec,
2560         .task_mgmt_fn_done = iscsi_task_mgmt_fn_done,
2561 };
2562
2563 static __init int iscsi_run_threads(int count, char *name, int (*fn)(void *))
2564 {
2565         int res = 0;
2566         int i;
2567         struct iscsi_thread_t *thr;
2568
2569         for (i = 0; i < count; i++) {
2570                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
2571                 if (!thr) {
2572                         res = -ENOMEM;
2573                         PRINT_ERROR("Failed to allocate thr %d", res);
2574                         goto out;
2575                 }
2576                 thr->thr = kthread_run(fn, NULL, "%s%d", name, i);
2577                 if (IS_ERR(thr->thr)) {
2578                         res = PTR_ERR(thr->thr);
2579                         PRINT_ERROR("kthread_create() failed: %d", res);
2580                         kfree(thr);
2581                         goto out;
2582                 }
2583                 list_add(&thr->threads_list_entry, &iscsi_threads_list);
2584         }
2585
2586 out:
2587         return res;
2588 }
2589
2590 static void iscsi_stop_threads(void)
2591 {
2592         struct iscsi_thread_t *t, *tmp;
2593
2594         list_for_each_entry_safe(t, tmp, &iscsi_threads_list,
2595                                 threads_list_entry) {
2596                 int rc = kthread_stop(t->thr);
2597                 if (rc < 0) {
2598                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
2599                 }
2600                 list_del(&t->threads_list_entry);
2601                 kfree(t);
2602         }
2603 }
2604
2605 static int __init iscsi_init(void)
2606 {
2607         int err;
2608         int num;
2609
2610         PRINT_INFO("iSCSI SCST Target - version %s", ISCSI_VERSION_STRING);
2611
2612 #ifdef NET_PAGE_CALLBACKS_DEFINED
2613         err = net_set_get_put_page_callbacks(iscsi_get_page_callback,
2614                         iscsi_put_page_callback);
2615         if (err != 0) {
2616                 PRINT_INFO("Unable to set page callbackes: %d", err);
2617                 goto out;
2618         }
2619 #else
2620         PRINT_INFO("%s", "Patch put_page_callback-<kernel-version>.patch "
2621                 "not applied on your kernel. Running in the performance "
2622                 "degraded mode. Refer README file for details");
2623 #endif
2624
2625         BUILD_BUG_ON(MAX_DATA_SEG_LEN != (ISCSI_CONN_IOV_MAX<<PAGE_SHIFT));
2626
2627         if ((ctr_major = register_chrdev(0, ctr_name, &ctr_fops)) < 0) {
2628                 PRINT_ERROR("failed to register the control device %d", ctr_major);
2629                 err = ctr_major;
2630                 goto out_callb;
2631         }
2632
2633         if ((err = event_init()) < 0)
2634                 goto out_reg;
2635
2636         iscsi_cmnd_cache = KMEM_CACHE(iscsi_cmnd, SCST_SLAB_FLAGS);
2637         if (!iscsi_cmnd_cache) {
2638                 err = -ENOMEM;
2639                 goto out_event;
2640         }
2641
2642         if (scst_register_target_template(&iscsi_template) < 0) {
2643                 err = -ENODEV;
2644                 goto out_kmem;
2645         }
2646         iscsi_template_registered = 1;
2647
2648         if ((err = iscsi_procfs_init()) < 0)
2649                 goto out_reg_tmpl;
2650
2651         num = max(num_online_cpus(), 2);
2652
2653         err = iscsi_run_threads(num, "iscsird", istrd);
2654         if (err != 0)
2655                 goto out_thr;
2656
2657         err = iscsi_run_threads(num, "iscsiwr", istwr);
2658         if (err != 0)
2659                 goto out_thr;
2660
2661 out:
2662         return err;
2663
2664 out_thr:
2665         iscsi_procfs_exit();
2666         iscsi_stop_threads();
2667
2668 out_reg_tmpl:
2669         scst_unregister_target_template(&iscsi_template);
2670
2671 out_kmem:
2672         kmem_cache_destroy(iscsi_cmnd_cache);
2673
2674 out_event:
2675         event_exit();
2676
2677 out_reg:
2678         unregister_chrdev(ctr_major, ctr_name);
2679
2680 out_callb:
2681 #ifdef NET_PAGE_CALLBACKS_DEFINED
2682         net_set_get_put_page_callbacks(NULL, NULL);
2683 #endif  
2684         goto out;
2685 }
2686
2687 static void __exit iscsi_exit(void)
2688 {
2689         iscsi_stop_threads();
2690
2691         unregister_chrdev(ctr_major, ctr_name);
2692
2693         iscsi_procfs_exit();
2694         event_exit();
2695
2696         kmem_cache_destroy(iscsi_cmnd_cache);
2697
2698         scst_unregister_target_template(&iscsi_template);
2699
2700 #ifdef NET_PAGE_CALLBACKS_DEFINED
2701         net_set_get_put_page_callbacks(NULL, NULL);
2702 #endif
2703 }
2704
2705 module_init(iscsi_init);
2706 module_exit(iscsi_exit);
2707
2708 MODULE_LICENSE("GPL");