- Now iSCSI target service load considered successful if at least one listen socket...
[mirror/scst/.git] / iscsi-scst / kernel / iscsi.c
1 /*
2  *  Copyright (C) 2002-2003 Ardis Technolgies <roman@ardistech.com>
3  *  Copyright (C) 2007 Vladislav Bolkhovitin
4  *  Copyright (C) 2007 CMS Distribution Limited
5  * 
6  *  This program is free software; you can redistribute it and/or
7  *  modify it under the terms of the GNU General Public License
8  *  as published by the Free Software Foundation, version 2
9  *  of the License.
10  * 
11  *  This program is distributed in the hope that it will be useful,
12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  *  GNU General Public License for more details.
15  */
16
17 #include <linux/module.h>
18 #include <linux/hash.h>
19 #include <linux/kthread.h>
20 #include <net/tcp.h>
21 #include <scsi/scsi.h>
22
23 #include "iscsi.h"
24 #include "digest.h"
25
26 #ifndef NET_PAGE_CALLBACKS_DEFINED
27 #warning Patch put_page_callback-<kernel-version>.patch not applied on your \
28         kernel. ISCSI-SCST will run in the performance degraded mode. Refer \
29         README file for details.
30 #endif
31
32 #define ISCSI_INIT_WRITE_WAKE           0x1
33 #define ISCSI_INIT_WRITE_REMOVE_HASH    0x2
34
35 static int ctr_major;
36 static char ctr_name[] = "iscsi-scst-ctl";
37 static int iscsi_template_registered;
38
39 #if defined(DEBUG) || defined(TRACING)
40 unsigned long iscsi_trace_flag = ISCSI_DEFAULT_LOG_FLAGS;
41 #endif
42
43 static struct kmem_cache *iscsi_cmnd_cache;
44
45 spinlock_t iscsi_rd_lock = SPIN_LOCK_UNLOCKED;
46 LIST_HEAD(iscsi_rd_list);
47 DECLARE_WAIT_QUEUE_HEAD(iscsi_rd_waitQ);
48
49 spinlock_t iscsi_wr_lock = SPIN_LOCK_UNLOCKED;
50 LIST_HEAD(iscsi_wr_list);
51 DECLARE_WAIT_QUEUE_HEAD(iscsi_wr_waitQ);
52
53 static char dummy_data[1024];
54
55 struct iscsi_thread_t {
56         struct task_struct *thr;
57         struct list_head threads_list_entry;
58 };
59
60 static LIST_HEAD(iscsi_threads_list);
61
62 static void cmnd_remove_hash(struct iscsi_cmnd *cmnd);
63 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status);
64 static void cmnd_prepare_skip_pdu(struct iscsi_cmnd *cmnd);
65
66 static inline u32 cmnd_write_size(struct iscsi_cmnd *cmnd)
67 {
68         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
69
70         if (hdr->flags & ISCSI_CMD_WRITE)
71                 return be32_to_cpu(hdr->data_length);
72         return 0;
73 }
74
75 static inline u32 cmnd_read_size(struct iscsi_cmnd *cmnd)
76 {
77         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
78
79         if (hdr->flags & ISCSI_CMD_READ) {
80                 struct iscsi_rlength_ahdr *ahdr =
81                         (struct iscsi_rlength_ahdr *)cmnd->pdu.ahs;
82
83                 if (!(hdr->flags & ISCSI_CMD_WRITE))
84                         return be32_to_cpu(hdr->data_length);
85                 if (ahdr && ahdr->ahstype == ISCSI_AHSTYPE_RLENGTH)
86                         return be32_to_cpu(ahdr->read_length);
87         }
88         return 0;
89 }
90
91 static inline void iscsi_restart_cmnd(struct iscsi_cmnd *cmnd)
92 {
93         cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
94         scst_restart_cmd(cmnd->scst_cmd, SCST_PREPROCESS_STATUS_SUCCESS,
95                 SCST_CONTEXT_THREAD);
96 }
97
98 struct iscsi_cmnd *cmnd_alloc(struct iscsi_conn *conn, struct iscsi_cmnd *parent)
99 {
100         struct iscsi_cmnd *cmnd;
101
102         /* ToDo: __GFP_NOFAIL?? */
103 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,17)
104         cmnd = kmem_cache_alloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
105         memset(cmnd, 0, sizeof(*cmnd));
106 #else
107         cmnd = kmem_cache_zalloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
108 #endif
109
110         atomic_set(&cmnd->ref_cnt, 1);
111         cmnd->scst_state = ISCSI_CMD_STATE_NEW;
112         cmnd->conn = conn;
113         cmnd->parent_req = parent;
114         init_waitqueue_head(&cmnd->scst_waitQ);
115
116         if (parent == NULL) {
117                 conn_get(conn);
118 #ifdef NET_PAGE_CALLBACKS_DEFINED
119                 atomic_set(&cmnd->net_ref_cnt, 0);
120 #endif          
121                 cmnd->target = conn->target;
122                 spin_lock_init(&cmnd->rsp_cmd_lock);
123                 INIT_LIST_HEAD(&cmnd->rsp_cmd_list);
124                 INIT_LIST_HEAD(&cmnd->rx_ddigest_cmd_list);
125
126                 spin_lock_bh(&conn->cmd_list_lock);
127                 list_add_tail(&cmnd->cmd_list_entry, &conn->cmd_list);
128                 spin_unlock_bh(&conn->cmd_list_lock);
129         }
130
131         TRACE_DBG("conn %p, parent %p, cmnd %p", conn, parent, cmnd);
132         return cmnd;
133 }
134
135 /* Frees a command. Also frees the additional header. */
136 void cmnd_free(struct iscsi_cmnd *cmnd)
137 {
138         TRACE_DBG("%p", cmnd);
139
140         /* Catch users from cmd_list or rsp_cmd_list */
141         EXTRACHECKS_BUG_ON(atomic_read(&cmnd->ref_cnt) != 0);
142
143         kfree(cmnd->pdu.ahs);
144
145         if (unlikely(cmnd->on_write_list)) {
146                 struct iscsi_scsi_cmd_hdr *req = cmnd_hdr(cmnd);
147
148                 PRINT_ERROR("cmnd %p still on some list?, %x, %x, %x, %x, %x, %x, %x",
149                         cmnd, req->opcode, req->scb[0], req->flags, req->itt,
150                         be32_to_cpu(req->data_length),
151                         req->cmd_sn, be32_to_cpu(cmnd->pdu.datasize));
152
153                 if (cmnd->parent_req) {
154                         struct iscsi_scsi_cmd_hdr *req =
155                                         cmnd_hdr(cmnd->parent_req);
156                         PRINT_ERROR("%p %x %u", req, req->opcode, req->scb[0]);
157                 }
158                 sBUG();
159         }
160
161         kmem_cache_free(iscsi_cmnd_cache, cmnd);
162         return;
163 }
164
165 void cmnd_done(struct iscsi_cmnd *cmnd)
166 {
167         TRACE_DBG("%p", cmnd);
168
169         if (unlikely(cmnd->tmfabort)) {
170                 TRACE_MGMT_DBG("Done aborted cmd %p (scst cmd %p, state %d)",
171                         cmnd, cmnd->scst_cmd, cmnd->scst_state);
172         }
173
174         if (cmnd->parent_req == NULL) {
175                 struct iscsi_conn *conn = cmnd->conn;
176                 TRACE_DBG("Deleting req %p from conn %p", cmnd, conn);
177                 spin_lock_bh(&conn->cmd_list_lock);
178                 list_del(&cmnd->cmd_list_entry);
179                 spin_unlock_bh(&conn->cmd_list_lock);
180
181                 conn_put(conn);
182
183                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rsp_cmd_list));
184                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rx_ddigest_cmd_list));
185
186                 /* Order between above and below code is important! */
187
188                 if (cmnd->scst_cmd) {
189                         switch(cmnd->scst_state) {
190                         case ISCSI_CMD_STATE_AFTER_PREPROC:
191                                 TRACE_DBG("%s", "AFTER_PREPROC");
192                                 cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
193                                 scst_restart_cmd(cmnd->scst_cmd,
194                                         SCST_PREPROCESS_STATUS_ERROR_FATAL,
195                                         SCST_CONTEXT_THREAD);
196                                 break;
197                         case ISCSI_CMD_STATE_PROCESSED:
198                                 TRACE_DBG("%s", "PROCESSED");
199                                 scst_tgt_cmd_done(cmnd->scst_cmd);
200                                 break;
201                         default:
202                                 PRINT_ERROR("Unexpected cmnd scst state %d",
203                                         cmnd->scst_state);
204                                 sBUG();
205                                 break;
206                         }
207                 }
208         } else {
209                 EXTRACHECKS_BUG_ON(cmnd->scst_cmd != NULL);
210
211                 spin_lock_bh(&cmnd->parent_req->rsp_cmd_lock);
212                 TRACE_DBG("Deleting rsp %p from parent %p", cmnd,
213                         cmnd->parent_req);
214                 list_del(&cmnd->rsp_cmd_list_entry);
215                 spin_unlock_bh(&cmnd->parent_req->rsp_cmd_lock);
216
217                 cmnd_put(cmnd->parent_req);
218         }
219
220         /* Order between above and below code is important! */
221
222         if (cmnd->own_sg) {
223                 TRACE_DBG("%s", "own_sg");
224                 scst_free(cmnd->sg, cmnd->sg_cnt);
225 #ifdef DEBUG
226                 cmnd->own_sg = 0;
227                 cmnd->sg = NULL;
228                 cmnd->sg_cnt = -1;
229 #endif
230         }
231
232         cmnd_free(cmnd);
233         return;
234 }
235
236 /*
237  * Corresponding conn may also gets destroyed atfer this function, except only
238  * if it's called from the read thread! 
239  */
240 void req_cmnd_release_force(struct iscsi_cmnd *req, int flags)
241 {
242         struct iscsi_cmnd *rsp;
243         struct iscsi_conn *conn = req->conn;
244
245         TRACE_ENTRY();
246
247         TRACE_DBG("%p", req);
248
249         if (flags & ISCSI_FORCE_RELEASE_WRITE) {
250 again_write:
251                 spin_lock(&conn->write_list_lock);
252                 list_for_each_entry(rsp, &conn->write_list, write_list_entry) {
253                         if (rsp->parent_req != req)
254                                 continue;
255
256                         cmd_del_from_write_list(rsp);
257
258                         spin_unlock(&conn->write_list_lock);
259
260                         cmnd_put(rsp);
261                         goto again_write;
262                 }
263                 spin_unlock(&conn->write_list_lock);
264         }
265
266 again_rsp:
267         spin_lock_bh(&req->rsp_cmd_lock);
268         list_for_each_entry(rsp, &req->rsp_cmd_list, rsp_cmd_list_entry) {
269                 int f;
270
271                 if (rsp->on_write_list || rsp->write_processing_started ||
272                     rsp->force_cleanup_done)
273                         continue;
274
275                 spin_unlock_bh(&req->rsp_cmd_lock);
276
277                 /* 
278                  * Recheck is necessary to not take write_list_lock under
279                  * rsp_cmd_lock.
280                  */
281                 spin_lock(&conn->write_list_lock);
282                 f = rsp->on_write_list || rsp->write_processing_started ||
283                         rsp->force_cleanup_done;
284                 spin_unlock(&conn->write_list_lock);
285                 if (f)
286                         goto again_rsp;
287
288                 rsp->force_cleanup_done = 1;
289                 cmnd_put(rsp);
290
291                 goto again_rsp;
292         }
293         spin_unlock_bh(&req->rsp_cmd_lock);
294
295         req_cmnd_release(req);
296
297         TRACE_EXIT();
298         return;
299 }
300
301 /*
302  * Corresponding conn may also gets destroyed atfer this function, except only
303  * if it's called from the read thread! 
304  */
305 void req_cmnd_release(struct iscsi_cmnd *req)
306 {
307         struct iscsi_cmnd *c, *t;
308
309         TRACE_ENTRY();
310
311         TRACE_DBG("%p", req);
312
313 #ifdef EXTRACHECKS
314         sBUG_ON(req->release_called);
315         req->release_called = 1;
316 #endif
317
318         if (unlikely(req->tmfabort)) {
319                 TRACE_MGMT_DBG("Release aborted req cmd %p (scst cmd %p, "
320                         "state %d)", req, req->scst_cmd, req->scst_state);
321         }
322
323         sBUG_ON(req->parent_req != NULL);
324
325         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
326                                 rx_ddigest_cmd_list_entry) {
327                 TRACE_DBG("Deleting RX ddigest cmd %p from digest "
328                         "list of req %p", c, req);
329                 list_del(&c->rx_ddigest_cmd_list_entry);
330                 cmnd_put(c);
331         }
332
333         if (req->hashed)
334                 cmnd_remove_hash(req);
335
336         cmnd_put(req);
337
338         TRACE_EXIT();
339         return;
340 }
341
342 /*
343  * Corresponding conn may also gets destroyed atfer this function, except only
344  * if it's called from the read thread! 
345  */
346 void rsp_cmnd_release(struct iscsi_cmnd *cmnd)
347 {
348         TRACE_DBG("%p", cmnd);
349
350 #ifdef EXTRACHECKS
351         sBUG_ON(cmnd->release_called);
352         cmnd->release_called = 1;
353 #endif
354
355         sBUG_ON(cmnd->hashed);
356         sBUG_ON(cmnd->parent_req == NULL);
357
358         if (unlikely(cmnd->tmfabort)) {
359                 TRACE_MGMT_DBG("Release aborted rsp cmd %p (parent req %p, "
360                         "scst cmd %p, state %d)", cmnd, cmnd->parent_req,
361                         cmnd->parent_req->scst_cmd,
362                         cmnd->parent_req->scst_state);
363         }
364
365         cmnd_put(cmnd);
366         return;
367 }
368
369 /**
370  * create a new command used as response.
371  *
372  * iscsi_cmnd_create_rsp_cmnd - 
373  * @cmnd: ptr to request command
374  *
375  * @return    ptr to response command or NULL
376  */
377 static struct iscsi_cmnd *iscsi_cmnd_create_rsp_cmnd(struct iscsi_cmnd *parent)
378 {
379         struct iscsi_cmnd *rsp = cmnd_alloc(parent->conn, parent);
380
381         spin_lock_bh(&parent->rsp_cmd_lock);
382         TRACE_DBG("Adding rsp %p to parent %p", rsp, parent);
383         list_add_tail(&rsp->rsp_cmd_list_entry, &parent->rsp_cmd_list);
384         spin_unlock_bh(&parent->rsp_cmd_lock);
385         cmnd_get(parent);
386         return rsp;
387 }
388
389 static inline struct iscsi_cmnd *get_rsp_cmnd(struct iscsi_cmnd *req)
390 {
391         struct iscsi_cmnd *res;
392
393         /* Currently this lock isn't needed, but just in case.. */
394         spin_lock_bh(&req->rsp_cmd_lock);
395         res = list_entry(req->rsp_cmd_list.prev, struct iscsi_cmnd,
396                 rsp_cmd_list_entry);
397         spin_unlock_bh(&req->rsp_cmd_lock);
398
399         return res;
400 }
401
402 static void iscsi_cmnds_init_write(struct list_head *send, int flags)
403 {
404         struct iscsi_cmnd *rsp = list_entry(send->next, struct iscsi_cmnd, 
405                                                 write_list_entry);
406         struct iscsi_conn *conn = rsp->conn;
407         struct list_head *pos, *next;
408
409         /*
410          * If we don't remove hashed req cmd from the hash list here, before
411          * submitting it for transmittion, we will have a race, when for
412          * some reason cmd's release is delayed after transmittion and
413          * initiator sends cmd with the same ITT => this command will be
414          * erroneously rejected as a duplicate.
415          */
416         if ((flags & ISCSI_INIT_WRITE_REMOVE_HASH) && rsp->parent_req->hashed &&
417             (rsp->parent_req->outstanding_r2t == 0))
418                 cmnd_remove_hash(rsp->parent_req);
419
420         list_for_each_safe(pos, next, send) {
421                 rsp = list_entry(pos, struct iscsi_cmnd, write_list_entry);
422
423                 TRACE_DBG("%p:%x", rsp, cmnd_opcode(rsp));
424
425                 sBUG_ON(conn != rsp->conn);
426
427                 if (!(conn->ddigest_type & DIGEST_NONE) &&
428                     (rsp->pdu.datasize != 0))
429                         digest_tx_data(rsp);
430
431                 list_del(&rsp->write_list_entry);
432
433                 spin_lock(&conn->write_list_lock);
434                 cmd_add_on_write_list(conn, rsp);
435                 spin_unlock(&conn->write_list_lock);
436         }
437         
438
439         if (flags & ISCSI_INIT_WRITE_WAKE)
440                 iscsi_make_conn_wr_active(conn);
441 }
442
443 static void iscsi_cmnd_init_write(struct iscsi_cmnd *rsp, int flags)
444 {
445         LIST_HEAD(head);
446
447         if (unlikely(rsp->on_write_list)) {
448                 PRINT_ERROR("cmd already on write list (%x %x %x %x %u %u "
449                         "%u %u %u %u %u %d %d",
450                         cmnd_itt(rsp), cmnd_ttt(rsp), cmnd_opcode(rsp),
451                         cmnd_scsicode(rsp), rsp->r2t_sn,
452                         rsp->r2t_length, rsp->is_unsolicited_data,
453                         rsp->target_task_tag, rsp->outstanding_r2t,
454                         rsp->hdigest, rsp->ddigest,
455                         list_empty(&rsp->rsp_cmd_list), rsp->hashed);
456                 sBUG();
457         }
458         list_add(&rsp->write_list_entry, &head);
459         iscsi_cmnds_init_write(&head, flags);
460 }
461
462 static void iscsi_set_datasize(struct iscsi_cmnd *cmnd, u32 offset, u32 size)
463 {
464         cmnd->pdu.datasize = size;
465
466         if (cmnd->pdu.datasize & 3) {
467                 int idx = (offset + cmnd->pdu.datasize) >> PAGE_SHIFT;
468                 u8 *p = (u8 *)page_address(cmnd->sg[idx].page) + 
469                         ((offset + cmnd->pdu.datasize) & ~PAGE_MASK);
470                 int i = 4 - (cmnd->pdu.datasize & 3);
471                 while (i--) 
472                     *p++ = 0;
473         }
474 }
475
476 static void send_data_rsp(struct iscsi_cmnd *req, u8 status, int send_status)
477 {
478         struct iscsi_cmnd *rsp;
479         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
480         struct iscsi_data_in_hdr *rsp_hdr;
481         u32 pdusize, expsize, scsisize, size, offset, sn;
482         LIST_HEAD(send);
483
484         TRACE_DBG("req %p", req);
485         pdusize = req->conn->session->sess_param.max_xmit_data_length;
486         expsize = cmnd_read_size(req);
487         size = min(expsize, (u32)req->bufflen);
488         offset = 0;
489         sn = 0;
490
491         while (1) {
492                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
493                 TRACE_DBG("rsp %p", rsp);
494                 rsp->sg = req->sg;
495                 rsp->bufflen = req->bufflen;
496                 rsp_hdr = (struct iscsi_data_in_hdr *)&rsp->pdu.bhs;
497
498                 rsp_hdr->opcode = ISCSI_OP_SCSI_DATA_IN;
499                 rsp_hdr->itt = req_hdr->itt;
500                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
501                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
502                 rsp_hdr->data_sn = cpu_to_be32(sn);
503
504                 if (size <= pdusize) {
505                         iscsi_set_datasize(rsp, offset, size);
506                         if (send_status) {
507                                 TRACE_DBG("status %x", status);
508                                 rsp_hdr->flags =
509                                         ISCSI_FLG_FINAL | ISCSI_FLG_STATUS;
510                                 rsp_hdr->cmd_status = status;
511                         }
512                         scsisize = req->bufflen;
513                         if (scsisize < expsize) {
514                                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
515                                 size = expsize - scsisize;
516                         } else if (scsisize > expsize) {
517                                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
518                                 size = scsisize - expsize;
519                         } else
520                                 size = 0;
521                         rsp_hdr->residual_count = cpu_to_be32(size);
522                         list_add_tail(&rsp->write_list_entry, &send);
523                         break;
524                 }
525
526                 iscsi_set_datasize(rsp, offset, pdusize);
527
528                 size -= pdusize;
529                 offset += pdusize;
530                 sn++;
531
532                 list_add_tail(&rsp->write_list_entry, &send);
533         }
534         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_REMOVE_HASH);
535 }
536
537 static struct iscsi_cmnd *create_status_rsp(struct iscsi_cmnd *req, int status,
538         const u8 *sense_buf, int sense_len)
539 {
540         struct iscsi_cmnd *rsp;
541         struct iscsi_scsi_rsp_hdr *rsp_hdr;
542         struct iscsi_sense_data *sense;
543         struct scatterlist *sg;
544
545         rsp = iscsi_cmnd_create_rsp_cmnd(req);
546         TRACE_DBG("%p", rsp);
547
548         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
549         rsp_hdr->opcode = ISCSI_OP_SCSI_RSP;
550         rsp_hdr->flags = ISCSI_FLG_FINAL;
551         rsp_hdr->response = ISCSI_RESPONSE_COMMAND_COMPLETED;
552         rsp_hdr->cmd_status = status;
553         rsp_hdr->itt = cmnd_hdr(req)->itt;
554
555         if (status == SAM_STAT_CHECK_CONDITION) {
556                 TRACE_DBG("%s", "CHECK_CONDITION");
557                 /* ToDo: __GFP_NOFAIL ?? */
558                 sg = rsp->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
559                                         &rsp->sg_cnt);
560                 if (sg == NULL) {
561                         /* ToDo(); */
562                 }
563                 rsp->own_sg = 1;
564                 sense = (struct iscsi_sense_data *)page_address(sg[0].page);
565                 sense->length = cpu_to_be16(sense_len);
566                 memcpy(sense->data, sense_buf, sense_len);
567                 rsp->pdu.datasize = sizeof(struct iscsi_sense_data) + sense_len;
568                 rsp->bufflen = (rsp->pdu.datasize + 3) & -4;
569                 if (rsp->bufflen - rsp->pdu.datasize) {
570                     int i = rsp->pdu.datasize;
571                     u8 *p = (u8 *)sense + i;
572                     
573                     while (i < rsp->bufflen) {
574                         *p ++ = 0;
575                         i++;
576                     }
577                 }
578         } else {
579                 rsp->pdu.datasize = 0;
580                 rsp->bufflen = 0;
581         }
582
583         return rsp;
584 }
585
586 static struct iscsi_cmnd *create_sense_rsp(struct iscsi_cmnd *req,
587         u8 sense_key, u8 asc, u8 ascq)
588 {
589         u8 sense[14];
590         memset(sense, 0, sizeof(sense));
591         sense[0] = 0xf0;
592         sense[2] = sense_key;
593         sense[7] = 6;   // Additional sense length
594         sense[12] = asc;
595         sense[13] = ascq;
596         return create_status_rsp(req, SAM_STAT_CHECK_CONDITION, sense,
597                 sizeof(sense));
598 }
599
600 static void iscsi_cmnd_reject(struct iscsi_cmnd *req, int reason)
601 {
602         struct iscsi_cmnd *rsp;
603         struct iscsi_reject_hdr *rsp_hdr;
604         struct scatterlist *sg;
605         char *addr;
606
607         TRACE_MGMT_DBG("Reject: req %p, reason %x", req, reason);
608
609         rsp = iscsi_cmnd_create_rsp_cmnd(req);
610         rsp_hdr = (struct iscsi_reject_hdr *)&rsp->pdu.bhs;
611
612         rsp_hdr->opcode = ISCSI_OP_REJECT;
613         rsp_hdr->ffffffff = ISCSI_RESERVED_TAG;
614         rsp_hdr->reason = reason;
615
616         /* ToDo: __GFP_NOFAIL ?? */
617         sg = rsp->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
618                                 &rsp->sg_cnt);
619         if (sg == NULL) {
620                 /* ToDo(); */
621         }
622         rsp->own_sg = 1;
623         addr = page_address(sg[0].page);
624         clear_page(addr);
625         memcpy(addr, &req->pdu.bhs, sizeof(struct iscsi_hdr));
626         rsp->bufflen = rsp->pdu.datasize = sizeof(struct iscsi_hdr);
627         cmnd_prepare_skip_pdu(req);
628
629         req->pdu.bhs.opcode = ISCSI_OP_PDU_REJECT;
630 }
631
632 static u32 cmnd_set_sn(struct iscsi_cmnd *cmnd, int set_stat_sn)
633 {
634         struct iscsi_conn *conn = cmnd->conn;
635         struct iscsi_session *sess = conn->session;
636         u32 res;
637
638         spin_lock(&sess->sn_lock);
639
640         if (set_stat_sn)
641                 cmnd->pdu.bhs.sn = cpu_to_be32(conn->stat_sn++);
642         cmnd->pdu.bhs.exp_sn = cpu_to_be32(sess->exp_cmd_sn);
643         cmnd->pdu.bhs.max_sn = cpu_to_be32(sess->exp_cmd_sn + sess->max_queued_cmnds);
644
645         res = cpu_to_be32(conn->stat_sn);
646
647         spin_unlock(&sess->sn_lock);
648         return res;
649 }
650
651 /* Called under sn_lock */
652 static void __update_stat_sn(struct iscsi_cmnd *cmnd)
653 {
654         struct iscsi_conn *conn = cmnd->conn;
655         u32 exp_stat_sn;
656
657         cmnd->pdu.bhs.exp_sn = exp_stat_sn = be32_to_cpu(cmnd->pdu.bhs.exp_sn);
658         TRACE_DBG("%x,%x", cmnd_opcode(cmnd), exp_stat_sn);
659         if ((int)(exp_stat_sn - conn->exp_stat_sn) > 0 &&
660             (int)(exp_stat_sn - conn->stat_sn) <= 0) {
661                 // free pdu resources
662                 cmnd->conn->exp_stat_sn = exp_stat_sn;
663         }
664 }
665
666 static inline void update_stat_sn(struct iscsi_cmnd *cmnd)
667 {
668         spin_lock(&cmnd->conn->session->sn_lock);
669         __update_stat_sn(cmnd);
670         spin_unlock(&cmnd->conn->session->sn_lock);
671 }
672
673 /* Called under sn_lock */
674 static int check_cmd_sn(struct iscsi_cmnd *cmnd)
675 {
676         struct iscsi_session *session = cmnd->conn->session;
677         u32 cmd_sn;
678
679         cmnd->pdu.bhs.sn = cmd_sn = be32_to_cpu(cmnd->pdu.bhs.sn);
680         TRACE_DBG("%d(%d)", cmd_sn, session->exp_cmd_sn);
681         if ((s32)(cmd_sn - session->exp_cmd_sn) >= 0)
682                 return 0;
683         PRINT_ERROR("sequence error (%x,%x)", cmd_sn, session->exp_cmd_sn);
684         return -ISCSI_REASON_PROTOCOL_ERROR;
685 }
686
687 static inline struct iscsi_cmnd *__cmnd_find_hash(struct iscsi_session *session,
688         u32 itt, u32 ttt)
689 {
690         struct list_head *head;
691         struct iscsi_cmnd *cmnd;
692
693         head = &session->cmnd_hash[cmnd_hashfn(itt)];
694
695         list_for_each_entry(cmnd, head, hash_list_entry) {
696                 if (cmnd->pdu.bhs.itt == itt) {
697                         if ((ttt != ISCSI_RESERVED_TAG) && (ttt != cmnd->target_task_tag))
698                                 continue;
699                         return cmnd;
700                 }
701         }
702         return NULL;
703 }
704
705 static struct iscsi_cmnd *cmnd_find_hash(struct iscsi_session *session,
706         u32 itt, u32 ttt)
707 {
708         struct iscsi_cmnd *cmnd;
709
710         spin_lock(&session->cmnd_hash_lock);
711         cmnd = __cmnd_find_hash(session, itt, ttt);
712         spin_unlock(&session->cmnd_hash_lock);
713
714         return cmnd;
715 }
716
717 static struct iscsi_cmnd *cmnd_find_hash_get(struct iscsi_session *session,
718         u32 itt, u32 ttt)
719 {
720         struct iscsi_cmnd *cmnd;
721
722         spin_lock(&session->cmnd_hash_lock);
723         cmnd = __cmnd_find_hash(session, itt, ttt);
724         cmnd_get(cmnd);
725         spin_unlock(&session->cmnd_hash_lock);
726
727         return cmnd;
728 }
729
730 static int cmnd_insert_hash(struct iscsi_cmnd *cmnd)
731 {
732         struct iscsi_session *session = cmnd->conn->session;
733         struct iscsi_cmnd *tmp;
734         struct list_head *head;
735         int err = 0;
736         u32 itt = cmnd->pdu.bhs.itt;
737
738         TRACE_DBG("%p:%x", cmnd, itt);
739         if (itt == ISCSI_RESERVED_TAG) {
740                 err = -ISCSI_REASON_PROTOCOL_ERROR;
741                 goto out;
742         }
743
744         spin_lock(&session->cmnd_hash_lock);
745
746         head = &session->cmnd_hash[cmnd_hashfn(cmnd->pdu.bhs.itt)];
747
748         tmp = __cmnd_find_hash(session, itt, ISCSI_RESERVED_TAG);
749         if (!tmp) {
750                 list_add_tail(&cmnd->hash_list_entry, head);
751                 cmnd->hashed = 1;
752         } else
753                 err = -ISCSI_REASON_TASK_IN_PROGRESS;
754
755         spin_unlock(&session->cmnd_hash_lock);
756
757         if (!err) {
758                 spin_lock(&session->sn_lock);
759                 __update_stat_sn(cmnd);
760                 err = check_cmd_sn(cmnd);
761                 spin_unlock(&session->sn_lock);
762         }
763
764 out:
765         return err;
766 }
767
768 static void cmnd_remove_hash(struct iscsi_cmnd *cmnd)
769 {
770         struct iscsi_session *session = cmnd->conn->session;
771         struct iscsi_cmnd *tmp;
772
773         spin_lock(&session->cmnd_hash_lock);
774
775         tmp = __cmnd_find_hash(session, cmnd->pdu.bhs.itt, ISCSI_RESERVED_TAG);
776
777         if (tmp && tmp == cmnd) {
778                 list_del(&cmnd->hash_list_entry);
779                 cmnd->hashed = 0;
780         } else {
781                 PRINT_ERROR("%p:%x not found", cmnd, cmnd_itt(cmnd));
782         }
783
784         spin_unlock(&session->cmnd_hash_lock);
785 }
786
787 static void cmnd_prepare_skip_pdu(struct iscsi_cmnd *cmnd)
788 {
789         struct iscsi_conn *conn = cmnd->conn;
790         struct scatterlist *sg = cmnd->sg;
791         char *addr;
792         u32 size;
793         int i;
794
795         TRACE_MGMT_DBG("Skipping (%p, %x %x %x %u, %p, scst state %d)", cmnd,
796                 cmnd_itt(cmnd), cmnd_opcode(cmnd), cmnd_hdr(cmnd)->scb[0],
797                 cmnd->pdu.datasize, cmnd->scst_cmd, cmnd->scst_state);
798
799         iscsi_extracheck_is_rd_thread(conn);
800
801         if (!(size = cmnd->pdu.datasize))
802                 return;
803
804         if (sg == NULL) {
805                 /* ToDo: __GFP_NOFAIL ?? */
806                 sg = cmnd->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
807                                         &cmnd->sg_cnt);
808                 if (sg == NULL) {
809                         /* ToDo(); */
810                 }
811                 cmnd->own_sg = 1;
812                 cmnd->bufflen = PAGE_SIZE;
813         }
814
815         addr = page_address(sg[0].page);
816         sBUG_ON(addr == NULL);
817         size = (size + 3) & -4;
818         conn->read_size = size;
819         for (i = 0; size > PAGE_SIZE; i++, size -= cmnd->bufflen) {
820                 sBUG_ON(i >= ISCSI_CONN_IOV_MAX);
821                 conn->read_iov[i].iov_base = addr;
822                 conn->read_iov[i].iov_len = cmnd->bufflen;
823         }
824         conn->read_iov[i].iov_base = addr;
825         conn->read_iov[i].iov_len = size;
826         conn->read_msg.msg_iov = conn->read_iov;
827         conn->read_msg.msg_iovlen = ++i;
828 }
829
830 static void cmnd_prepare_skip_pdu_set_resid(struct iscsi_cmnd *req)
831 {
832         struct iscsi_cmnd *rsp;
833         struct iscsi_scsi_rsp_hdr *rsp_hdr;
834         u32 size;
835
836         TRACE_DBG("%p", req);
837
838         rsp = get_rsp_cmnd(req);
839         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
840         if (cmnd_opcode(rsp) != ISCSI_OP_SCSI_RSP) {
841                 PRINT_ERROR("unexpected response command %u", cmnd_opcode(rsp));
842                 return;
843         }
844
845         size = cmnd_write_size(req);
846         if (size) {
847                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
848                 rsp_hdr->residual_count = cpu_to_be32(size);
849         }
850         size = cmnd_read_size(req);
851         if (size) {
852                 if (cmnd_hdr(req)->flags & ISCSI_CMD_WRITE) {
853                         rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_UNDERFLOW;
854                         rsp_hdr->bi_residual_count = cpu_to_be32(size);
855                 } else {
856                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
857                         rsp_hdr->residual_count = cpu_to_be32(size);
858                 }
859         }
860         req->pdu.bhs.opcode =
861                 (req->pdu.bhs.opcode & ~ISCSI_OPCODE_MASK) | ISCSI_OP_SCSI_REJECT;
862
863         cmnd_prepare_skip_pdu(req);
864 }
865
866 static int cmnd_prepare_recv_pdu(struct iscsi_conn *conn,
867         struct iscsi_cmnd *cmd, u32 offset, u32 size)
868 {
869         struct scatterlist *sg = cmd->sg;
870         int bufflen = cmd->bufflen;
871         int idx, i;
872         char *addr;
873         int res = 0;
874
875         TRACE_DBG("%p %u,%u", cmd->sg, offset, size);
876
877         iscsi_extracheck_is_rd_thread(conn);
878
879         if ((offset >= bufflen) ||
880             (offset + size > bufflen)) {
881                 PRINT_ERROR("Wrong ltn (%u %u %u)", offset, size, bufflen);
882                 mark_conn_closed(conn);
883                 res = -EIO;
884                 goto out;
885         }
886
887         offset += sg[0].offset;
888         idx = offset >> PAGE_SHIFT;
889         offset &= ~PAGE_MASK;
890
891         conn->read_msg.msg_iov = conn->read_iov;
892         conn->read_size = size = (size + 3) & -4;
893
894         i = 0;
895         while (1) {
896                 sBUG_ON(sg[idx].page == NULL);
897                 addr = page_address(sg[idx].page);
898                 sBUG_ON(addr == NULL);
899                 conn->read_iov[i].iov_base =  addr + offset;
900                 if (offset + size <= PAGE_SIZE) {
901                         TRACE_DBG("idx=%d, offset=%u, size=%d, addr=%p",
902                                 idx, offset, size, addr);
903                         conn->read_iov[i].iov_len = size;
904                         conn->read_msg.msg_iovlen = ++i;
905                         break;
906                 }
907                 conn->read_iov[i].iov_len = PAGE_SIZE - offset;
908                 TRACE_DBG("idx=%d, offset=%u, size=%d, iov_len=%d, addr=%p",
909                         idx, offset, size, conn->read_iov[i].iov_len, addr);
910                 size -= conn->read_iov[i].iov_len;
911                 offset = 0;
912                 if (++i >= ISCSI_CONN_IOV_MAX) {
913                         PRINT_ERROR("Initiator %s violated negotiated "
914                                 "parameters by sending too much data (size "
915                                 "left %d)", conn->session->initiator_name, size);
916                         mark_conn_closed(conn);
917                         res = -EINVAL;
918                         break;
919                 }
920                 idx++;
921         }
922         TRACE_DBG("msg_iov=%p, msg_iovlen=%d",
923                 conn->read_msg.msg_iov, conn->read_msg.msg_iovlen);
924
925 out:
926         return res;
927 }
928
929 static void send_r2t(struct iscsi_cmnd *req)
930 {
931         struct iscsi_session *session = req->conn->session;
932         struct iscsi_cmnd *rsp;
933         struct iscsi_r2t_hdr *rsp_hdr;
934         u32 length, offset, burst;
935         LIST_HEAD(send);
936
937         /*
938          * There is no race with data_out_start() and __cmnd_abort(), since
939          * all functions called from single read thread
940          */
941         iscsi_extracheck_is_rd_thread(req->conn);
942
943         length = req->r2t_length;
944         burst = session->sess_param.max_burst_length;
945         offset = be32_to_cpu(cmnd_hdr(req)->data_length) - length;
946
947         do {
948                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
949                 rsp->pdu.bhs.ttt = req->target_task_tag;
950                 rsp_hdr = (struct iscsi_r2t_hdr *)&rsp->pdu.bhs;
951                 rsp_hdr->opcode = ISCSI_OP_R2T;
952                 rsp_hdr->flags = ISCSI_FLG_FINAL;
953                 rsp_hdr->lun = cmnd_hdr(req)->lun;
954                 rsp_hdr->itt = cmnd_hdr(req)->itt;
955                 rsp_hdr->r2t_sn = cpu_to_be32(req->r2t_sn++);
956                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
957                 if (length > burst) {
958                         rsp_hdr->data_length = cpu_to_be32(burst);
959                         length -= burst;
960                         offset += burst;
961                 } else {
962                         rsp_hdr->data_length = cpu_to_be32(length);
963                         length = 0;
964                 }
965
966                 TRACE(TRACE_D_WRITE, "%x %u %u %u %u", cmnd_itt(req),
967                         be32_to_cpu(rsp_hdr->data_length),
968                         be32_to_cpu(rsp_hdr->buffer_offset),
969                         be32_to_cpu(rsp_hdr->r2t_sn), req->outstanding_r2t);
970
971                 list_add_tail(&rsp->write_list_entry, &send);
972
973                 if (++req->outstanding_r2t >= session->sess_param.max_outstanding_r2t)
974                         break;
975
976         } while (length);
977
978         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_WAKE);
979
980         req->data_waiting = 1;
981 }
982
983 static int iscsi_pre_exec(struct scst_cmd *scst_cmd)
984 {
985         int res = SCST_PREPROCESS_STATUS_SUCCESS;
986         struct iscsi_cmnd *req = (struct iscsi_cmnd*)
987                 scst_cmd_get_tgt_priv(scst_cmd);
988         struct iscsi_cmnd *c, *t;
989
990         TRACE_ENTRY();
991
992         EXTRACHECKS_BUG_ON(scst_cmd_atomic(scst_cmd));
993
994         /* If data digest isn't used this list will be empty */
995         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
996                                 rx_ddigest_cmd_list_entry) {
997                 TRACE_DBG("Checking digest of RX ddigest cmd %p", c);
998                 if (digest_rx_data(c) != 0) {
999                         scst_set_cmd_error(scst_cmd,
1000                                 SCST_LOAD_SENSE(iscsi_sense_crc_error));
1001                         res = SCST_PREPROCESS_STATUS_ERROR_SENSE_SET;
1002                         /*
1003                          * The rest of rx_ddigest_cmd_list will be freed 
1004                          * in req_cmnd_release()
1005                          */
1006                         goto out;
1007                 }
1008                 TRACE_DBG("Deleting RX digest cmd %p from digest list", c);
1009                 list_del(&c->rx_ddigest_cmd_list_entry);
1010                 cmnd_put(c);
1011         }
1012
1013 out:
1014         TRACE_EXIT_RES(res);
1015         return res;
1016 }
1017
1018
1019 static void scsi_cmnd_exec(struct iscsi_cmnd *cmnd)
1020 {
1021         if (cmnd->r2t_length) {
1022                 if (!cmnd->is_unsolicited_data)
1023                         send_r2t(cmnd);
1024         } else {
1025                 /*
1026                  * There is no race with send_r2t() and __cmnd_abort(),
1027                  * since all functions called from single read thread
1028                  */
1029                 cmnd->data_waiting = 0;
1030                 iscsi_restart_cmnd(cmnd);
1031         }
1032 }
1033
1034 static int noop_out_start(struct iscsi_cmnd *cmnd)
1035 {
1036         struct iscsi_conn *conn = cmnd->conn;
1037         u32 size, tmp;
1038         int i, err = 0;
1039
1040         TRACE_DBG("%p", cmnd);
1041
1042         iscsi_extracheck_is_rd_thread(conn);
1043
1044         if (cmnd_ttt(cmnd) != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1045                 /*
1046                  * We don't request a NOP-Out by sending a NOP-In.
1047                  * See 10.18.2 in the draft 20.
1048                  */
1049                 PRINT_ERROR("initiator bug %x", cmnd_itt(cmnd));
1050                 err = -ISCSI_REASON_PROTOCOL_ERROR;
1051                 goto out;
1052         }
1053
1054         if (cmnd_itt(cmnd) == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1055                 if (!(cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE))
1056                         PRINT_ERROR("%s","initiator bug!");
1057                 spin_lock(&conn->session->sn_lock);
1058                 __update_stat_sn(cmnd);
1059                 err = check_cmd_sn(cmnd);
1060                 spin_unlock(&conn->session->sn_lock);
1061                 if (err)
1062                         goto out;
1063         } else if ((err = cmnd_insert_hash(cmnd)) < 0) {
1064                 PRINT_ERROR("Can't insert in hash: ignore this request %x",
1065                         cmnd_itt(cmnd));
1066                 goto out;
1067         }
1068
1069         if ((size = cmnd->pdu.datasize)) {
1070                 size = (size + 3) & -4;
1071                 conn->read_msg.msg_iov = conn->read_iov;
1072                 if (cmnd->pdu.bhs.itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1073                         struct scatterlist *sg;
1074
1075                         /* ToDo: __GFP_NOFAIL ?? */
1076                         cmnd->sg = sg = scst_alloc(size,
1077                                 GFP_KERNEL|__GFP_NOFAIL, &cmnd->sg_cnt);
1078                         if (sg == NULL) {
1079                                 /* ToDo(); */
1080                         }
1081                         if (cmnd->sg_cnt > ISCSI_CONN_IOV_MAX) {
1082                                 /* ToDo(); */
1083                         }
1084                         cmnd->own_sg = 1;
1085                         cmnd->bufflen = size;
1086
1087                         for (i = 0; i < cmnd->sg_cnt; i++) {
1088                                 conn->read_iov[i].iov_base =
1089                                         page_address(sg[i].page);
1090                                 tmp = min_t(u32, size, PAGE_SIZE);
1091                                 conn->read_iov[i].iov_len = tmp;
1092                                 conn->read_size += tmp;
1093                                 size -= tmp;
1094                         }
1095                 } else {
1096                         /*
1097                          * There are no problems with the safety from concurrent
1098                          * accesses to dummy_data, since for ISCSI_RESERVED_TAG
1099                          * the data only read and then discarded.
1100                          */
1101                         for (i = 0; i < ISCSI_CONN_IOV_MAX; i++) {
1102                                 conn->read_iov[i].iov_base = dummy_data;
1103                                 tmp = min_t(u32, size, sizeof(dummy_data));
1104                                 conn->read_iov[i].iov_len = tmp;
1105                                 conn->read_size += tmp;
1106                                 size -= tmp;
1107                         }
1108                 }
1109                 sBUG_ON(size == 0);
1110                 conn->read_msg.msg_iovlen = i;
1111                 TRACE_DBG("msg_iov=%p, msg_iovlen=%d", conn->read_msg.msg_iov,
1112                         conn->read_msg.msg_iovlen);
1113         }
1114 out:
1115         return err;
1116 }
1117
1118 static inline u32 get_next_ttt(struct iscsi_conn *conn)
1119 {
1120         u32 ttt;
1121         struct iscsi_session *session = conn->session;
1122
1123         iscsi_extracheck_is_rd_thread(conn);
1124
1125         if (session->next_ttt == ISCSI_RESERVED_TAG)
1126                 session->next_ttt++;
1127         ttt = session->next_ttt++;
1128
1129         return cpu_to_be32(ttt);
1130 }
1131
1132 static int scsi_cmnd_start(struct iscsi_cmnd *req)
1133 {
1134         struct iscsi_conn *conn = req->conn;
1135         struct iscsi_session *session = conn->session;
1136         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1137         struct scst_cmd *scst_cmd;
1138         scst_data_direction dir;
1139         int res = 0;
1140
1141         TRACE_ENTRY();
1142
1143         TRACE_DBG("scsi command: %02x", req_hdr->scb[0]);
1144
1145         scst_cmd = scst_rx_cmd(session->scst_sess,
1146                 (uint8_t*)&req_hdr->lun, sizeof(req_hdr->lun),
1147                 req_hdr->scb, sizeof(req_hdr->scb), SCST_NON_ATOMIC);
1148         if (scst_cmd == NULL) {
1149                 create_status_rsp(req, SAM_STAT_BUSY, NULL, 0);
1150                 cmnd_prepare_skip_pdu_set_resid(req);
1151                 goto out;
1152         }
1153
1154         req->scst_cmd = scst_cmd;
1155         scst_cmd_set_tag(scst_cmd, req_hdr->itt);
1156         scst_cmd_set_tgt_priv(scst_cmd, req);
1157 #ifndef NET_PAGE_CALLBACKS_DEFINED
1158         scst_cmd_set_data_buf_tgt_alloc(scst_cmd);
1159 #endif
1160
1161         if (req_hdr->flags & ISCSI_CMD_READ)
1162                 dir = SCST_DATA_READ;
1163         else if (req_hdr->flags & ISCSI_CMD_WRITE)
1164                 dir = SCST_DATA_WRITE;
1165         else
1166                 dir = SCST_DATA_NONE;
1167         scst_cmd_set_expected(scst_cmd, dir, be32_to_cpu(req_hdr->data_length));
1168
1169         switch(req_hdr->flags & ISCSI_CMD_ATTR_MASK) {
1170         case ISCSI_CMD_SIMPLE:
1171                 scst_cmd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1172                 break;
1173         case ISCSI_CMD_HEAD_OF_QUEUE:
1174                 scst_cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1175                 break;
1176         case ISCSI_CMD_ORDERED:
1177                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1178                 break;
1179         case ISCSI_CMD_ACA:
1180                 scst_cmd->queue_type = SCST_CMD_QUEUE_ACA;
1181                 break;
1182         case ISCSI_CMD_UNTAGGED:
1183                 scst_cmd->queue_type = SCST_CMD_QUEUE_UNTAGGED;
1184                 break;
1185         default:
1186                 PRINT_ERROR("Unknown task code %x, use ORDERED instead",
1187                         req_hdr->flags & ISCSI_CMD_ATTR_MASK);
1188                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1189                 break;
1190         }
1191
1192         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
1193         scst_cmd_set_tgt_sn(scst_cmd, req_hdr->cmd_sn);
1194
1195         TRACE_DBG("START Command (tag %d, queue_type %d)",
1196                 req_hdr->itt, scst_cmd->queue_type);
1197         req->scst_state = ISCSI_CMD_STATE_RX_CMD;
1198         scst_cmd_init_stage1_done(scst_cmd, SCST_CONTEXT_DIRECT, 0);
1199
1200         wait_event(req->scst_waitQ, (req->scst_state != ISCSI_CMD_STATE_RX_CMD));
1201
1202         if (unlikely(req->scst_state != ISCSI_CMD_STATE_AFTER_PREPROC)) {
1203                 TRACE_DBG("req %p is in %x state", req, req->scst_state);
1204                 if (req->scst_state == ISCSI_CMD_STATE_PROCESSED) {
1205                         /* Response is already prepared */
1206                         cmnd_prepare_skip_pdu_set_resid(req);
1207                         goto out;
1208                 }
1209                 if (unlikely(req->tmfabort)) {
1210                         TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
1211                                 req->scst_cmd);
1212                         cmnd_prepare_skip_pdu(req);
1213                         goto out;
1214                 }
1215                 sBUG();
1216         }
1217
1218         dir = scst_cmd_get_data_direction(scst_cmd);
1219         if (dir != SCST_DATA_WRITE) {
1220                 if (!(req_hdr->flags & ISCSI_CMD_FINAL) || req->pdu.datasize) {
1221                         PRINT_ERROR("Unexpected unsolicited data (ITT %x "
1222                                 "CDB %x", cmnd_itt(req), req_hdr->scb[0]);
1223                         create_sense_rsp(req, ABORTED_COMMAND, 0xc, 0xc);
1224                         cmnd_prepare_skip_pdu_set_resid(req);
1225                         goto out;
1226                 }
1227         }
1228
1229         if (dir == SCST_DATA_WRITE) {
1230                 req->is_unsolicited_data = !(req_hdr->flags & ISCSI_CMD_FINAL);
1231                 req->r2t_length = be32_to_cpu(req_hdr->data_length) - req->pdu.datasize;
1232         }
1233         req->target_task_tag = get_next_ttt(conn);
1234         req->sg = scst_cmd_get_sg(scst_cmd);
1235         req->bufflen = scst_cmd_get_bufflen(scst_cmd);
1236         if (req->r2t_length > req->bufflen) {
1237                 PRINT_ERROR("req->r2t_length %d > req->bufflen %d",
1238                         req->r2t_length, req->bufflen);
1239                 req->r2t_length = req->bufflen;
1240         }
1241
1242         TRACE_DBG("req=%p, dir=%d, is_unsolicited_data=%d, "
1243                 "r2t_length=%d, bufflen=%d", req, dir,
1244                 req->is_unsolicited_data, req->r2t_length, req->bufflen);
1245
1246         if (!session->sess_param.immediate_data &&
1247             req->pdu.datasize) {
1248                 PRINT_ERROR("Initiator %s violated negotiated paremeters: "
1249                         "forbidden immediate data sent (ITT %x, op  %x)",
1250                         session->initiator_name, cmnd_itt(req), req_hdr->scb[0]);
1251                 res = -EINVAL;
1252                 goto out;
1253         }
1254
1255         if (session->sess_param.initial_r2t &&
1256             !(req_hdr->flags & ISCSI_CMD_FINAL)) {
1257                 PRINT_ERROR("Initiator %s violated negotiated paremeters: "
1258                         "initial R2T is required (ITT %x, op  %x)",
1259                         session->initiator_name, cmnd_itt(req), req_hdr->scb[0]);
1260                 res = -EINVAL;
1261                 goto out;
1262         }
1263
1264         if (req->pdu.datasize) {
1265                 if (dir != SCST_DATA_WRITE) {
1266                         PRINT_ERROR("pdu.datasize(%d) >0, but dir(%x) isn't WRITE",
1267                                 req->pdu.datasize, dir);
1268                         create_sense_rsp(req, ABORTED_COMMAND, 0xc, 0xc);
1269                         cmnd_prepare_skip_pdu_set_resid(req);
1270                 } else
1271                         res = cmnd_prepare_recv_pdu(conn, req, 0, req->pdu.datasize);
1272         }
1273 out:
1274         /* Aborted commands will be freed in cmnd_rx_end() */
1275         TRACE_EXIT_RES(res);
1276         return res;
1277 }
1278
1279 static int data_out_start(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
1280 {
1281         struct iscsi_data_out_hdr *req_hdr = (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1282         struct iscsi_cmnd *req = NULL;
1283         u32 offset = be32_to_cpu(req_hdr->buffer_offset);
1284         int res = 0;
1285
1286         TRACE_ENTRY();
1287
1288         /*
1289          * There is no race with send_r2t() and __cmnd_abort(), since
1290          * all functions called from single read thread
1291          */
1292         iscsi_extracheck_is_rd_thread(cmnd->conn);
1293
1294         update_stat_sn(cmnd);
1295
1296         cmnd->cmd_req = req = cmnd_find_hash(conn->session, req_hdr->itt,
1297                                         req_hdr->ttt);
1298         if (!req) {
1299                 PRINT_ERROR("unable to find scsi task %x %x",
1300                         cmnd_itt(cmnd), cmnd_ttt(cmnd));
1301                 goto skip_pdu;
1302         }
1303
1304         if (req->r2t_length < cmnd->pdu.datasize) {
1305                 PRINT_ERROR("Invalid data len %x %u %u", cmnd_itt(req),
1306                         cmnd->pdu.datasize, req->r2t_length);
1307                 mark_conn_closed(conn);
1308                 res = -EINVAL;
1309                 goto out;
1310         }
1311
1312         if (req->r2t_length + offset != cmnd_write_size(req)) {
1313                 PRINT_ERROR("Wrong cmd lengths (%x %u %u %u)",
1314                         cmnd_itt(req), req->r2t_length,
1315                         offset, cmnd_write_size(req));
1316                 mark_conn_closed(conn);
1317                 res = -EINVAL;
1318                 goto out;
1319         }
1320
1321         req->r2t_length -= cmnd->pdu.datasize;
1322
1323         /* Check unsolicited burst data */
1324         if ((req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) &&
1325             (req->pdu.bhs.flags & ISCSI_FLG_FINAL)) {
1326                 PRINT_ERROR("unexpected data from %x %x",
1327                         cmnd_itt(cmnd), cmnd_ttt(cmnd));
1328                 mark_conn_closed(conn);
1329                 res = -EINVAL;
1330                 goto out;
1331         }
1332
1333         TRACE(TRACE_D_WRITE, "%u %p %p %u %u", req_hdr->ttt, cmnd, req,
1334                 offset, cmnd->pdu.datasize);
1335
1336         res = cmnd_prepare_recv_pdu(conn, req, offset, cmnd->pdu.datasize);
1337
1338 out:
1339         TRACE_EXIT_RES(res);
1340         return res;
1341
1342 skip_pdu:
1343         cmnd->pdu.bhs.opcode = ISCSI_OP_DATA_REJECT;
1344         cmnd_prepare_skip_pdu(cmnd);
1345         goto out;
1346 }
1347
1348 static void data_out_end(struct iscsi_cmnd *cmnd)
1349 {
1350         struct iscsi_data_out_hdr *req_hdr = (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1351         struct iscsi_cmnd *req;
1352
1353         sBUG_ON(cmnd == NULL);
1354         req = cmnd->cmd_req;
1355         sBUG_ON(req == NULL);
1356
1357         TRACE_DBG("cmnd %p, req %p", cmnd, req);
1358
1359         iscsi_extracheck_is_rd_thread(cmnd->conn);
1360
1361         if (!(cmnd->conn->ddigest_type & DIGEST_NONE)) {
1362                 TRACE_DBG("Adding RX ddigest cmd %p to digest list "
1363                         "of req %p", cmnd, req);
1364                 list_add_tail(&cmnd->rx_ddigest_cmd_list_entry,
1365                         &req->rx_ddigest_cmd_list);
1366                 cmnd_get(cmnd);
1367         }
1368
1369         if (req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1370                 TRACE_DBG("ISCSI_RESERVED_TAG, FINAL %x",
1371                         req_hdr->flags & ISCSI_FLG_FINAL);
1372                 if (req_hdr->flags & ISCSI_FLG_FINAL) {
1373                         req->is_unsolicited_data = 0;
1374                         if (!req->pending)
1375                                 scsi_cmnd_exec(req);
1376                 }
1377         } else {
1378                 TRACE_DBG("FINAL %x, outstanding_r2t %d, "
1379                         "r2t_length %d", req_hdr->flags & ISCSI_FLG_FINAL,
1380                         req->outstanding_r2t, req->r2t_length);
1381                 /* ToDo : proper error handling */
1382                 if (!(req_hdr->flags & ISCSI_FLG_FINAL) && (req->r2t_length == 0))
1383                         PRINT_ERROR("initiator error %x", cmnd_itt(req));
1384
1385                 if (!(req_hdr->flags & ISCSI_FLG_FINAL))
1386                         goto out;
1387
1388                 req->outstanding_r2t--;
1389
1390                 scsi_cmnd_exec(req);
1391         }
1392
1393 out:
1394         cmnd_put(cmnd);
1395         return;
1396 }
1397
1398 /*
1399  * Must be called from the read thread. Also called under cmd_list_lock,
1400  * but may drop it inside.
1401  *
1402  * Returns >0 if cmd_list_lock was dropped inside, 0 otherwise.
1403  */
1404 static inline int __cmnd_abort(struct iscsi_cmnd *cmnd)
1405 {
1406         int res = 0;
1407
1408         /* Check to avoid double release */
1409         if (cmnd->tmfabort)
1410                 goto out;
1411
1412         TRACE_MGMT_DBG("Aborting cmd %p, scst_cmd %p (scst state %x, "
1413                 "ref_cnt %d, itt %x, op %x, r2t_len %x, CDB op %x, "
1414                 "size to write %u, is_unsolicited_data %u, "
1415                 "outstanding_r2t %u)", cmnd, cmnd->scst_cmd,
1416                 cmnd->scst_state, atomic_read(&cmnd->ref_cnt),
1417                 cmnd_itt(cmnd), cmnd_opcode(cmnd), cmnd->r2t_length,
1418                 cmnd_scsicode(cmnd), cmnd_write_size(cmnd),
1419                 cmnd->is_unsolicited_data, cmnd->outstanding_r2t);
1420
1421 #ifdef NET_PAGE_CALLBACKS_DEFINED
1422         TRACE_MGMT_DBG("net_ref_cnt %d", atomic_read(&cmnd->net_ref_cnt));
1423 #endif
1424
1425         iscsi_extracheck_is_rd_thread(cmnd->conn);
1426
1427         cmnd->tmfabort = 1;
1428
1429         if (cmnd->data_waiting) {
1430                 struct iscsi_conn *conn = cmnd->conn;
1431                 res = 1;
1432                 spin_unlock_bh(&conn->cmd_list_lock);
1433                 TRACE_MGMT_DBG("Releasing data waiting cmd %p", cmnd);
1434                 req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
1435                 /*
1436                  * We are in the read thread, so we may not worry that after
1437                  * cmnd release conn gets released as well.
1438                  */
1439                 spin_lock_bh(&conn->cmd_list_lock);
1440         }
1441
1442 out:
1443         return res;
1444 }
1445
1446 /* Must be called from the read thread */
1447 static int cmnd_abort(struct iscsi_cmnd *req)
1448 {
1449         struct iscsi_session *session = req->conn->session;
1450         struct iscsi_task_mgt_hdr *req_hdr =
1451                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1452         struct iscsi_cmnd *cmnd;
1453         int err;
1454
1455         if ((cmnd = cmnd_find_hash_get(session, req_hdr->rtt, ISCSI_RESERVED_TAG))) {
1456                 struct iscsi_conn *conn = cmnd->conn;
1457                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
1458
1459                 if (req_hdr->lun != hdr->lun) {
1460                          PRINT_ERROR("ABORT TASK: LUN mismatch: req LUN "
1461                                 "%Lx, cmd LUN %Lx, rtt %u", req_hdr->lun,
1462                                 hdr->lun, req_hdr->rtt);
1463                         err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1464                         goto out_put;
1465                 }
1466
1467                 if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
1468                     (req_hdr->cmd_sn == hdr->cmd_sn)) {
1469                         PRINT_ERROR("ABORT TASK: SN mismatch: req SN %x, "
1470                                 "cmd SN %x, rtt %u", req_hdr->cmd_sn,
1471                                 hdr->cmd_sn, req_hdr->rtt);
1472                         err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1473                         goto out_put;
1474                 }
1475
1476                 spin_lock_bh(&conn->cmd_list_lock);
1477                 __cmnd_abort(cmnd);
1478                 spin_unlock_bh(&conn->cmd_list_lock);
1479
1480                 cmnd_put(cmnd);
1481                 err = 0;
1482         } else
1483                 err = ISCSI_RESPONSE_UNKNOWN_TASK;
1484
1485 out:
1486         return err;
1487
1488 out_put:
1489         cmnd_put(cmnd);
1490         goto out;
1491 }
1492
1493 /* Must be called from the read thread */
1494 static int target_abort(struct iscsi_cmnd *req, int all)
1495 {
1496         struct iscsi_target *target = req->conn->session->target;
1497         struct iscsi_task_mgt_hdr *req_hdr =
1498                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1499         struct iscsi_session *session;
1500         struct iscsi_conn *conn;
1501         struct iscsi_cmnd *cmnd;
1502
1503         mutex_lock(&target->target_mutex);
1504
1505         list_for_each_entry(session, &target->session_list, session_list_entry) {
1506                 list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
1507                         spin_lock_bh(&conn->cmd_list_lock);
1508 again:
1509                         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1510                                 int again = 0;
1511                                 if (cmnd == req)
1512                                         continue;
1513                                 if (all)
1514                                         again = __cmnd_abort(cmnd);
1515                                 else if (req_hdr->lun == cmnd_hdr(cmnd)->lun)
1516                                         again = __cmnd_abort(cmnd);
1517                                 if (again)
1518                                         goto again;
1519                         }
1520                         spin_unlock_bh(&conn->cmd_list_lock);
1521                 }
1522         }
1523
1524         mutex_unlock(&target->target_mutex);
1525         return 0;
1526 }
1527
1528 /* Must be called from the read thread */
1529 static void task_set_abort(struct iscsi_cmnd *req)
1530 {
1531         struct iscsi_session *session = req->conn->session;
1532         struct iscsi_task_mgt_hdr *req_hdr =
1533                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1534         struct iscsi_target *target = session->target;
1535         struct iscsi_conn *conn;
1536         struct iscsi_cmnd *cmnd;
1537
1538         mutex_lock(&target->target_mutex);
1539
1540         list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
1541                 spin_lock_bh(&conn->cmd_list_lock);
1542 again:
1543                 list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1544                         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
1545                         if (cmnd == req)
1546                                 continue;
1547                         if (req_hdr->lun != hdr->lun)
1548                                 continue;
1549                         if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
1550                             req_hdr->cmd_sn == hdr->cmd_sn)
1551                                 continue;
1552                         if (__cmnd_abort(cmnd))
1553                                 goto again;
1554                 }
1555                 spin_unlock_bh(&conn->cmd_list_lock);
1556         }
1557
1558         mutex_unlock(&target->target_mutex);
1559         return;
1560 }
1561
1562 /* Must be called from the read thread */
1563 void conn_abort(struct iscsi_conn *conn)
1564 {
1565         struct iscsi_cmnd *cmnd;
1566
1567         TRACE_MGMT_DBG("Aborting conn %p", conn);
1568
1569         spin_lock_bh(&conn->cmd_list_lock);
1570 again:
1571         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1572                 if (__cmnd_abort(cmnd))
1573                         goto again;
1574         }
1575         spin_unlock_bh(&conn->cmd_list_lock);
1576 }
1577
1578 static void execute_task_management(struct iscsi_cmnd *req)
1579 {
1580         struct iscsi_conn *conn = req->conn;
1581         struct iscsi_task_mgt_hdr *req_hdr =
1582                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1583         int err = 0, function = req_hdr->function & ISCSI_FUNCTION_MASK;
1584         struct scst_rx_mgmt_params params;
1585
1586         TRACE(TRACE_MGMT, "TM cmd: req %p, itt %x, fn %d, rtt %x", req, cmnd_itt(req),
1587                 function, req_hdr->rtt);
1588
1589         memset(&params, 0, sizeof(params));
1590         params.atomic = SCST_NON_ATOMIC;
1591         params.tgt_priv = req;
1592         
1593         if ((function != ISCSI_FUNCTION_ABORT_TASK) &&
1594             (req_hdr->rtt != ISCSI_RESERVED_TAG)) {
1595                 PRINT_ERROR("Invalid RTT %x (TM fn %x)", req_hdr->rtt,
1596                         function);
1597                 err = -1;
1598                 goto reject;
1599         }
1600
1601         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
1602
1603         switch (function) {
1604         case ISCSI_FUNCTION_ABORT_TASK:
1605                 err = cmnd_abort(req);
1606                 if (err == 0) {
1607                         params.fn = SCST_ABORT_TASK;
1608                         params.tag = req_hdr->rtt;
1609                         params.tag_set = 1;
1610                         params.lun = (uint8_t *)&req_hdr->lun;
1611                         params.lun_len = sizeof(req_hdr->lun);
1612                         params.lun_set = 1;
1613                         params.cmd_sn = req_hdr->cmd_sn;
1614                         params.cmd_sn_set = 1;
1615                         err = scst_rx_mgmt_fn(conn->session->scst_sess,
1616                                 &params);
1617                 }
1618                 break;
1619         case ISCSI_FUNCTION_ABORT_TASK_SET:
1620                 task_set_abort(req);
1621                 params.fn = SCST_ABORT_TASK_SET;
1622                 params.lun = (uint8_t *)&req_hdr->lun;
1623                 params.lun_len = sizeof(req_hdr->lun);
1624                 params.lun_set = 1;
1625                 params.cmd_sn = req_hdr->cmd_sn;
1626                 params.cmd_sn_set = 1;
1627                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1628                         &params);
1629                 break;
1630         case ISCSI_FUNCTION_CLEAR_TASK_SET:
1631                 task_set_abort(req);
1632                 params.fn = SCST_CLEAR_TASK_SET;
1633                 params.lun = (uint8_t *)&req_hdr->lun;
1634                 params.lun_len = sizeof(req_hdr->lun);
1635                 params.lun_set = 1;
1636                 params.cmd_sn = req_hdr->cmd_sn;
1637                 params.cmd_sn_set = 1;
1638                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1639                         &params);
1640                 break;
1641         case ISCSI_FUNCTION_CLEAR_ACA:
1642                 params.fn = SCST_CLEAR_ACA;
1643                 params.lun = (uint8_t *)&req_hdr->lun;
1644                 params.lun_len = sizeof(req_hdr->lun);
1645                 params.lun_set = 1;
1646                 params.cmd_sn = req_hdr->cmd_sn;
1647                 params.cmd_sn_set = 1;
1648                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1649                         &params);
1650                 break;
1651         case ISCSI_FUNCTION_TARGET_COLD_RESET:
1652         case ISCSI_FUNCTION_TARGET_WARM_RESET:
1653                 target_abort(req, 1);
1654                 params.fn = SCST_TARGET_RESET;
1655                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1656                         &params);
1657                 break;
1658         case ISCSI_FUNCTION_LOGICAL_UNIT_RESET:
1659                 target_abort(req, 0);
1660                 params.fn = SCST_LUN_RESET;
1661                 params.lun = (uint8_t *)&req_hdr->lun;
1662                 params.lun_len = sizeof(req_hdr->lun);
1663                 params.lun_set = 1;
1664                 err = scst_rx_mgmt_fn(conn->session->scst_sess,
1665                         &params);
1666                 break;
1667         case ISCSI_FUNCTION_TASK_REASSIGN:
1668                 iscsi_send_task_mgmt_resp(req, 
1669                         ISCSI_RESPONSE_FUNCTION_UNSUPPORTED);
1670                 break;
1671         default:
1672                 iscsi_send_task_mgmt_resp(req,
1673                         ISCSI_RESPONSE_FUNCTION_REJECTED);
1674                 break;
1675         }
1676
1677 reject:
1678         if (err != 0) {
1679                 iscsi_send_task_mgmt_resp(req,
1680                         ISCSI_RESPONSE_FUNCTION_REJECTED);
1681         }
1682 }
1683
1684 static void noop_out_exec(struct iscsi_cmnd *req)
1685 {
1686         struct iscsi_cmnd *rsp;
1687         struct iscsi_nop_in_hdr *rsp_hdr;
1688
1689         TRACE_DBG("%p", req);
1690
1691         if (cmnd_itt(req) != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1692                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
1693
1694                 rsp_hdr = (struct iscsi_nop_in_hdr *)&rsp->pdu.bhs;
1695                 rsp_hdr->opcode = ISCSI_OP_NOOP_IN;
1696                 rsp_hdr->flags = ISCSI_FLG_FINAL;
1697                 rsp_hdr->itt = req->pdu.bhs.itt;
1698                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
1699
1700                 if (req->pdu.datasize)
1701                         sBUG_ON(req->sg == NULL);
1702                 else
1703                         sBUG_ON(req->sg != NULL);
1704
1705                 if (req->sg) {
1706                         rsp->sg = req->sg;
1707                         rsp->bufflen = req->bufflen;
1708                 }
1709
1710                 sBUG_ON(get_pgcnt(req->pdu.datasize, 0) > ISCSI_CONN_IOV_MAX);
1711                 rsp->pdu.datasize = req->pdu.datasize;
1712                 iscsi_cmnd_init_write(rsp,
1713                         ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
1714                 req_cmnd_release(req);
1715         } else
1716                 cmnd_put(req);
1717 }
1718
1719 static void logout_exec(struct iscsi_cmnd *req)
1720 {
1721         struct iscsi_logout_req_hdr *req_hdr;
1722         struct iscsi_cmnd *rsp;
1723         struct iscsi_logout_rsp_hdr *rsp_hdr;
1724
1725         PRINT_INFO("Logout received from initiator %s",
1726                 req->conn->session->initiator_name);
1727         TRACE_DBG("%p", req);
1728
1729         req_hdr = (struct iscsi_logout_req_hdr *)&req->pdu.bhs;
1730         rsp = iscsi_cmnd_create_rsp_cmnd(req);
1731         rsp_hdr = (struct iscsi_logout_rsp_hdr *)&rsp->pdu.bhs;
1732         rsp_hdr->opcode = ISCSI_OP_LOGOUT_RSP;
1733         rsp_hdr->flags = ISCSI_FLG_FINAL;
1734         rsp_hdr->itt = req_hdr->itt;
1735         rsp->should_close_conn = 1;
1736         iscsi_cmnd_init_write(rsp,
1737                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
1738         req_cmnd_release(req);
1739 }
1740
1741 static void iscsi_cmnd_exec(struct iscsi_cmnd *cmnd)
1742 {
1743         TRACE_DBG("%p,%x,%u", cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn);
1744
1745         if (unlikely(cmnd->tmfabort)) {
1746                 TRACE_MGMT_DBG("cmnd %p (scst_cmd %p) aborted", cmnd,
1747                         cmnd->scst_cmd);
1748                 req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
1749                 goto out;
1750         }
1751
1752         switch (cmnd_opcode(cmnd)) {
1753         case ISCSI_OP_NOOP_OUT:
1754                 noop_out_exec(cmnd);
1755                 break;
1756         case ISCSI_OP_SCSI_CMD:
1757                 scsi_cmnd_exec(cmnd);
1758                 break;
1759         case ISCSI_OP_SCSI_TASK_MGT_MSG:
1760                 execute_task_management(cmnd);
1761                 break;
1762         case ISCSI_OP_LOGOUT_CMD:
1763                 logout_exec(cmnd);
1764                 break;
1765         case ISCSI_OP_SCSI_REJECT:
1766                 TRACE_MGMT_DBG("REJECT cmnd %p (scst_cmd %p)", cmnd,
1767                         cmnd->scst_cmd);
1768                 iscsi_cmnd_init_write(get_rsp_cmnd(cmnd),
1769                         ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
1770                 req_cmnd_release(cmnd);
1771                 break;
1772         default:
1773                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
1774                 req_cmnd_release(cmnd);
1775                 break;
1776         }
1777 out:
1778         return;
1779 }
1780
1781 static void __cmnd_send_pdu(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd,
1782         u32 offset, u32 size)
1783 {
1784         TRACE_DBG("%p %u,%u,%u", cmnd, offset, size, cmnd->bufflen);
1785
1786         iscsi_extracheck_is_wr_thread(conn);
1787
1788         sBUG_ON(offset > cmnd->bufflen);
1789         sBUG_ON(offset + size > cmnd->bufflen);
1790
1791         conn->write_offset = offset;
1792         conn->write_size += size;
1793 }
1794
1795 static void cmnd_send_pdu(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
1796 {
1797         u32 size;
1798
1799         if (!cmnd->pdu.datasize)
1800                 return;
1801
1802         size = (cmnd->pdu.datasize + 3) & -4;
1803         sBUG_ON(cmnd->sg == NULL);
1804         sBUG_ON(cmnd->bufflen != size);
1805         __cmnd_send_pdu(conn, cmnd, 0, size);
1806 }
1807
1808 static void set_cork(struct socket *sock, int on)
1809 {
1810         int opt = on;
1811         mm_segment_t oldfs;
1812
1813         oldfs = get_fs();
1814         set_fs(get_ds());
1815         sock->ops->setsockopt(sock, SOL_TCP, TCP_CORK, (void *)&opt, sizeof(opt));
1816         set_fs(oldfs);
1817 }
1818
1819 void cmnd_tx_start(struct iscsi_cmnd *cmnd)
1820 {
1821         struct iscsi_conn *conn = cmnd->conn;
1822
1823         TRACE_DBG("%p:%p:%x", conn, cmnd, cmnd_opcode(cmnd));
1824         iscsi_cmnd_set_length(&cmnd->pdu);
1825
1826         iscsi_extracheck_is_wr_thread(conn);
1827
1828         set_cork(conn->sock, 1);
1829
1830         conn->write_iop = conn->write_iov;
1831         conn->write_iop->iov_base = &cmnd->pdu.bhs;
1832         conn->write_iop->iov_len = sizeof(cmnd->pdu.bhs);
1833         conn->write_iop_used = 1;
1834         conn->write_size = sizeof(cmnd->pdu.bhs);
1835
1836         switch (cmnd_opcode(cmnd)) {
1837         case ISCSI_OP_NOOP_IN:
1838                 cmnd_set_sn(cmnd, 1);
1839                 cmnd_send_pdu(conn, cmnd);
1840                 break;
1841         case ISCSI_OP_SCSI_RSP:
1842                 cmnd_set_sn(cmnd, 1);
1843                 cmnd_send_pdu(conn, cmnd);
1844                 break;
1845         case ISCSI_OP_SCSI_TASK_MGT_RSP:
1846                 cmnd_set_sn(cmnd, 1);
1847                 break;
1848         case ISCSI_OP_TEXT_RSP:
1849                 cmnd_set_sn(cmnd, 1);
1850                 break;
1851         case ISCSI_OP_SCSI_DATA_IN:
1852         {
1853                 struct iscsi_data_in_hdr *rsp = (struct iscsi_data_in_hdr *)&cmnd->pdu.bhs;
1854                 u32 offset = cpu_to_be32(rsp->buffer_offset);
1855
1856                 cmnd_set_sn(cmnd, (rsp->flags & ISCSI_FLG_FINAL) ? 1 : 0);
1857                 __cmnd_send_pdu(conn, cmnd, offset, cmnd->pdu.datasize);
1858                 break;
1859         }
1860         case ISCSI_OP_LOGOUT_RSP:
1861                 cmnd_set_sn(cmnd, 1);
1862                 break;
1863         case ISCSI_OP_R2T:
1864                 cmnd->pdu.bhs.sn = cmnd_set_sn(cmnd, 0);
1865                 break;
1866         case ISCSI_OP_ASYNC_MSG:
1867                 cmnd_set_sn(cmnd, 1);
1868                 break;
1869         case ISCSI_OP_REJECT:
1870                 cmnd_set_sn(cmnd, 1);
1871                 cmnd_send_pdu(conn, cmnd);
1872                 break;
1873         default:
1874                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
1875                 break;
1876         }
1877
1878         // move this?
1879         conn->write_size = (conn->write_size + 3) & -4;
1880         iscsi_dump_pdu(&cmnd->pdu);
1881 }
1882
1883 void cmnd_tx_end(struct iscsi_cmnd *cmnd)
1884 {
1885         struct iscsi_conn *conn = cmnd->conn;
1886
1887         TRACE_DBG("%p:%x (should_close_conn %d)", cmnd, cmnd_opcode(cmnd),
1888                 cmnd->should_close_conn);
1889
1890         switch (cmnd_opcode(cmnd)) {
1891         case ISCSI_OP_NOOP_IN:
1892         case ISCSI_OP_SCSI_RSP:
1893         case ISCSI_OP_SCSI_TASK_MGT_RSP:
1894         case ISCSI_OP_TEXT_RSP:
1895         case ISCSI_OP_R2T:
1896         case ISCSI_OP_ASYNC_MSG:
1897         case ISCSI_OP_REJECT:
1898         case ISCSI_OP_SCSI_DATA_IN:
1899         case ISCSI_OP_LOGOUT_RSP:
1900                 break;
1901         default:
1902                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
1903                 sBUG();
1904                 break;
1905         }
1906
1907         if (cmnd->should_close_conn) {
1908                 PRINT_INFO("Closing connection at initiator %s request",
1909                         conn->session->initiator_name);
1910                 mark_conn_closed(conn);
1911         }
1912
1913         set_cork(cmnd->conn->sock, 0);
1914 }
1915
1916 /*
1917  * Push the command for execution. This functions reorders the commands.
1918  * Called from the read thread.
1919  */
1920 static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd)
1921 {
1922         struct iscsi_session *session = cmnd->conn->session;
1923         struct list_head *entry;
1924         u32 cmd_sn;
1925
1926         TRACE_DBG("%p:%x %u,%u",
1927                 cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn, session->exp_cmd_sn);
1928
1929         iscsi_extracheck_is_rd_thread(cmnd->conn);
1930
1931         if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
1932                 iscsi_cmnd_exec(cmnd);
1933                 return;
1934         }
1935
1936         spin_lock(&session->sn_lock);
1937
1938         cmd_sn = cmnd->pdu.bhs.sn;
1939         if (cmd_sn == session->exp_cmd_sn) {
1940                 while (1) {
1941                         session->exp_cmd_sn = ++cmd_sn;
1942
1943                         spin_unlock(&session->sn_lock);
1944
1945                         iscsi_cmnd_exec(cmnd);
1946
1947                         if (list_empty(&session->pending_list))
1948                                 break;
1949                         cmnd = list_entry(session->pending_list.next, struct iscsi_cmnd,
1950                                                 pending_list_entry);
1951                         if (cmnd->pdu.bhs.sn != cmd_sn)
1952                                 break;
1953                         list_del(&cmnd->pending_list_entry);
1954                         cmnd->pending = 0;
1955
1956                         spin_lock(&session->sn_lock);
1957                 }
1958         } else {
1959                 cmnd->pending = 1;
1960                 if (before(cmd_sn, session->exp_cmd_sn)) { /* close the conn */
1961                         PRINT_ERROR("unexpected cmd_sn (%u,%u)", cmd_sn,
1962                                 session->exp_cmd_sn);
1963                 }
1964
1965                 if (after(cmd_sn, session->exp_cmd_sn + session->max_queued_cmnds)) {
1966                         PRINT_ERROR("too large cmd_sn (%u,%u)", cmd_sn,
1967                                 session->exp_cmd_sn);
1968                 }
1969
1970                 spin_unlock(&session->sn_lock);
1971
1972                 list_for_each(entry, &session->pending_list) {
1973                         struct iscsi_cmnd *tmp = list_entry(entry, struct iscsi_cmnd,
1974                                                         pending_list_entry);
1975                         if (before(cmd_sn, tmp->pdu.bhs.sn))
1976                                 break;
1977                 }
1978
1979                 list_add_tail(&cmnd->pending_list_entry, entry);
1980         }
1981
1982         return;
1983 }
1984
1985 static int check_segment_length(struct iscsi_cmnd *cmnd)
1986 {
1987         struct iscsi_conn *conn = cmnd->conn;
1988         struct iscsi_session *session = conn->session;
1989
1990         if (cmnd->pdu.datasize > session->sess_param.max_recv_data_length) {
1991                 PRINT_ERROR("Initiator %s violated negotiated parameters: "
1992                         "data too long (ITT %x, datasize %u, "
1993                         "max_recv_data_length %u", session->initiator_name,
1994                         cmnd_itt(cmnd), cmnd->pdu.datasize,
1995                         session->sess_param.max_recv_data_length);
1996                 mark_conn_closed(conn);
1997                 return -EINVAL;
1998         }
1999         return 0;
2000 }
2001
2002 int cmnd_rx_start(struct iscsi_cmnd *cmnd)
2003 {
2004         struct iscsi_conn *conn = cmnd->conn;
2005         int res, rc;
2006
2007         iscsi_dump_pdu(&cmnd->pdu);
2008
2009         res = check_segment_length(cmnd);
2010         if (res != 0)
2011                 goto out;
2012
2013         switch (cmnd_opcode(cmnd)) {
2014         case ISCSI_OP_NOOP_OUT:
2015                 rc = noop_out_start(cmnd);
2016                 break;
2017         case ISCSI_OP_SCSI_CMD:
2018                 rc = cmnd_insert_hash(cmnd);
2019                 if (likely(rc == 0)) {
2020                         res = scsi_cmnd_start(cmnd);
2021                         if (unlikely(res != 0))
2022                                 goto out;
2023                 }
2024                 break;
2025         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2026                 rc = cmnd_insert_hash(cmnd);
2027                 break;
2028         case ISCSI_OP_SCSI_DATA_OUT:
2029                 res = data_out_start(conn, cmnd);
2030                 rc = 0; /* to avoid compiler warning */
2031                 if (unlikely(res != 0))
2032                         goto out;
2033                 break;
2034         case ISCSI_OP_LOGOUT_CMD:
2035                 rc = cmnd_insert_hash(cmnd);
2036                 break;
2037         case ISCSI_OP_TEXT_CMD:
2038         case ISCSI_OP_SNACK_CMD:
2039                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2040                 break;
2041         default:
2042                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2043                 break;
2044         }
2045
2046         if (unlikely(rc < 0)) {
2047                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
2048                 PRINT_ERROR("Error %d (iSCSI opcode %x, ITT %x, op %x)", rc,
2049                         cmnd_opcode(cmnd), cmnd_itt(cmnd),
2050                         (cmnd_opcode(cmnd) == ISCSI_OP_SCSI_CMD ?
2051                                 hdr->scb[0] : -1));
2052                 iscsi_cmnd_reject(cmnd, -rc);
2053         }
2054
2055 out:
2056         TRACE_EXIT_RES(res);
2057         return res;
2058 }
2059
2060 void cmnd_rx_end(struct iscsi_cmnd *cmnd)
2061 {
2062         if (unlikely(cmnd->tmfabort)) {
2063                 TRACE_MGMT_DBG("cmnd %p (scst_cmd %p) aborted", cmnd,
2064                         cmnd->scst_cmd);
2065                 req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
2066                 return;
2067         }
2068
2069         TRACE_DBG("%p:%x", cmnd, cmnd_opcode(cmnd));
2070         switch (cmnd_opcode(cmnd)) {
2071         case ISCSI_OP_SCSI_REJECT:
2072         case ISCSI_OP_NOOP_OUT:
2073         case ISCSI_OP_SCSI_CMD:
2074         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2075         case ISCSI_OP_LOGOUT_CMD:
2076                 iscsi_session_push_cmnd(cmnd);
2077                 break;
2078         case ISCSI_OP_SCSI_DATA_OUT:
2079                 data_out_end(cmnd);
2080                 break;
2081         case ISCSI_OP_PDU_REJECT:
2082                 iscsi_cmnd_init_write(get_rsp_cmnd(cmnd),
2083                         ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2084                 req_cmnd_release(cmnd);
2085                 break;
2086         case ISCSI_OP_DATA_REJECT:
2087                 req_cmnd_release(cmnd);
2088                 break;
2089         default:
2090                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2091                 req_cmnd_release(cmnd);
2092                 break;
2093         }
2094 }
2095
2096 #ifndef NET_PAGE_CALLBACKS_DEFINED
2097 static int iscsi_alloc_data_buf(struct scst_cmd *cmd)
2098 {
2099         if (scst_cmd_get_data_direction(cmd) == SCST_DATA_READ) {
2100                 /*
2101                  * sock->ops->sendpage() is async zero copy operation,
2102                  * so we must be sure not to free and reuse
2103                  * the command's buffer before the sending was completed
2104                  * by the network layers. It is possible only if we
2105                  * don't use SGV cache.
2106                  */
2107                 scst_cmd_set_no_sgv(cmd);
2108         }
2109         return 1;
2110 }
2111 #endif
2112
2113 static inline void iscsi_set_state_wake_up(struct iscsi_cmnd *req,
2114         int new_state)
2115 {
2116         /*
2117          * We use wait_event() to wait for the state change, but it checks its
2118          * condition without any protection, so without cmnd_get() it is
2119          * possible that req will die "immediately" after the state assignment
2120          * and wake_up() will operate on dead data.
2121          */
2122         cmnd_get_ordered(req);
2123         req->scst_state = new_state;
2124         wake_up(&req->scst_waitQ);
2125         cmnd_put(req);
2126         return;
2127 }
2128
2129 static void iscsi_preprocessing_done(struct scst_cmd *scst_cmd)
2130 {
2131         struct iscsi_cmnd *req = (struct iscsi_cmnd*)
2132                                 scst_cmd_get_tgt_priv(scst_cmd);
2133
2134         TRACE_DBG("req %p", req);
2135
2136         EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_RX_CMD);
2137
2138         iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_AFTER_PREPROC);
2139         return;
2140 }
2141
2142 /* 
2143  * No locks.
2144  *
2145  * IMPORTANT! Connection conn must be protected by additional conn_get()
2146  * upon entrance in this function, because otherwise it could be destroyed
2147  * inside as a result of iscsi_send(), which releases sent commands.
2148  */
2149 static void iscsi_try_local_processing(struct iscsi_conn *conn)
2150 {
2151         int local;
2152
2153         TRACE_ENTRY();
2154
2155         spin_lock_bh(&iscsi_wr_lock);
2156         switch(conn->wr_state) {
2157         case ISCSI_CONN_WR_STATE_IN_LIST:
2158                 list_del(&conn->wr_list_entry);
2159                 /* go through */
2160         case ISCSI_CONN_WR_STATE_IDLE:
2161 #ifdef EXTRACHECKS
2162                 conn->wr_task = current;
2163 #endif
2164                 conn->wr_state = ISCSI_CONN_WR_STATE_PROCESSING;
2165                 conn->wr_space_ready = 0;
2166                 local = 1;
2167                 break;
2168         default:
2169                 local = 0;
2170                 break;
2171         }
2172         spin_unlock_bh(&iscsi_wr_lock);
2173
2174         if (local) {
2175                 int rc = 1;
2176                 while(test_write_ready(conn)) {
2177                         rc = iscsi_send(conn);
2178                         if (rc <= 0) {
2179                                 break;
2180                         }
2181                 }
2182
2183                 spin_lock_bh(&iscsi_wr_lock);
2184 #ifdef EXTRACHECKS
2185                 conn->wr_task = NULL;
2186 #endif
2187                 if ((rc <= 0) || test_write_ready(conn)) {
2188                         list_add_tail(&conn->wr_list_entry, &iscsi_wr_list);
2189                         conn->wr_state = ISCSI_CONN_WR_STATE_IN_LIST;
2190                         wake_up(&iscsi_wr_waitQ);
2191                 } else
2192                         conn->wr_state = ISCSI_CONN_WR_STATE_IDLE;
2193                 spin_unlock_bh(&iscsi_wr_lock);
2194         }
2195
2196         TRACE_EXIT();
2197         return;
2198 }
2199
2200 static int iscsi_xmit_response(struct scst_cmd *scst_cmd)
2201 {
2202         int resp_flags = scst_cmd_get_tgt_resp_flags(scst_cmd);
2203         struct iscsi_cmnd *req = (struct iscsi_cmnd*)
2204                                         scst_cmd_get_tgt_priv(scst_cmd);
2205         struct iscsi_conn *conn = req->conn;
2206         int status = scst_cmd_get_status(scst_cmd);
2207         u8 *sense = scst_cmd_get_sense_buffer(scst_cmd);
2208         int sense_len = scst_cmd_get_sense_buffer_len(scst_cmd);
2209         int old_state = req->scst_state;
2210
2211         scst_cmd_set_tgt_priv(scst_cmd, NULL);
2212
2213         req->tmfabort |= scst_cmd_aborted(scst_cmd) ? 1 : 0;
2214         if (unlikely(req->tmfabort)) {
2215                 TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
2216                         req->scst_cmd);
2217                 if (old_state == ISCSI_CMD_STATE_RESTARTED) {
2218                         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
2219                         req_cmnd_release_force(req, ISCSI_FORCE_RELEASE_WRITE);
2220                 } else
2221                         iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_PROCESSED);
2222                 goto out;
2223         }
2224
2225         if (unlikely(old_state != ISCSI_CMD_STATE_RESTARTED)) {
2226                 TRACE_DBG("req %p on %d state", req, old_state);
2227                 create_status_rsp(req, status, sense, sense_len);
2228                 switch(old_state) {
2229                 case ISCSI_CMD_STATE_RX_CMD:
2230                 case ISCSI_CMD_STATE_AFTER_PREPROC:
2231                         break;
2232                 default:
2233                         sBUG();
2234                 }
2235                 iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_PROCESSED);
2236                 goto out;
2237         }
2238
2239         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
2240
2241         req->bufflen = scst_cmd_get_resp_data_len(scst_cmd);
2242         req->sg = scst_cmd_get_sg(scst_cmd);
2243
2244         TRACE_DBG("req %p, resp_flags=%x, req->bufflen=%d, req->sg=%p", req,
2245                 resp_flags, req->bufflen, req->sg);
2246
2247         if ((req->bufflen != 0) && !(resp_flags & SCST_TSC_FLAG_STATUS)) {
2248                 PRINT_ERROR("%s", "Sending DATA without STATUS is unsupported");
2249                 scst_set_cmd_error(scst_cmd,
2250                         SCST_LOAD_SENSE(scst_sense_hardw_error));
2251                 resp_flags = scst_cmd_get_tgt_resp_flags(scst_cmd);
2252                 sBUG();
2253         }
2254
2255         if (req->bufflen != 0) {
2256                 /* 
2257                  * Check above makes sure that SCST_TSC_FLAG_STATUS is set,
2258                  * so status is valid here, but in future that could change.
2259                  * ToDo
2260                  */
2261                 if (status != SAM_STAT_CHECK_CONDITION) {
2262                         send_data_rsp(req, status,
2263                                 resp_flags & SCST_TSC_FLAG_STATUS);
2264                 } else {
2265                         struct iscsi_cmnd *rsp;
2266                         struct iscsi_scsi_rsp_hdr *rsp_hdr;
2267                         int resid;
2268                         send_data_rsp(req, 0, 0);
2269                         if (resp_flags & SCST_TSC_FLAG_STATUS) {
2270                                 rsp = create_status_rsp(req, status, sense,
2271                                         sense_len);
2272                                 rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
2273                                 resid = cmnd_read_size(req) - req->bufflen;
2274                                 if (resid > 0) {
2275                                         rsp_hdr->flags |=
2276                                                 ISCSI_FLG_RESIDUAL_UNDERFLOW;
2277                                         rsp_hdr->residual_count = cpu_to_be32(resid);
2278                                 } else if (resid < 0) {
2279                                         rsp_hdr->flags |=
2280                                                 ISCSI_FLG_RESIDUAL_OVERFLOW;
2281                                         rsp_hdr->residual_count = cpu_to_be32(-resid);
2282                                 }
2283                                 iscsi_cmnd_init_write(rsp,
2284                                         ISCSI_INIT_WRITE_REMOVE_HASH);
2285                         }
2286                 }
2287         } else if (resp_flags & SCST_TSC_FLAG_STATUS) {
2288                 struct iscsi_cmnd *rsp;
2289                 struct iscsi_scsi_rsp_hdr *rsp_hdr;
2290                 u32 resid;
2291                 rsp = create_status_rsp(req, status, sense, sense_len);
2292                 rsp_hdr = (struct iscsi_scsi_rsp_hdr *) &rsp->pdu.bhs;
2293                 resid = cmnd_read_size(req);
2294                 if (resid != 0) {
2295                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
2296                         rsp_hdr->residual_count = cpu_to_be32(resid);
2297                 }
2298                 iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH);
2299         }
2300 #ifdef EXTRACHECKS
2301         else
2302                 sBUG();
2303 #endif
2304
2305         conn_get_ordered(conn);
2306         req_cmnd_release(req);
2307         iscsi_try_local_processing(conn);
2308         conn_put(conn);
2309
2310 out:
2311         return SCST_TGT_RES_SUCCESS;
2312 }
2313
2314 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status)
2315 {
2316         struct iscsi_cmnd *rsp;
2317         struct iscsi_task_mgt_hdr *req_hdr =
2318                                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2319         struct iscsi_task_rsp_hdr *rsp_hdr;
2320
2321         TRACE(TRACE_MGMT, "TM req %p finished, status %d", req, status);
2322
2323         rsp = iscsi_cmnd_create_rsp_cmnd(req);
2324         rsp_hdr = (struct iscsi_task_rsp_hdr *)&rsp->pdu.bhs;
2325
2326         rsp_hdr->opcode = ISCSI_OP_SCSI_TASK_MGT_RSP;
2327         rsp_hdr->flags = ISCSI_FLG_FINAL;
2328         rsp_hdr->itt = req_hdr->itt;
2329         rsp_hdr->response = status;
2330
2331         if ((req_hdr->function & ISCSI_FUNCTION_MASK) ==
2332                         ISCSI_FUNCTION_TARGET_COLD_RESET)
2333                 rsp->should_close_conn = 1;
2334
2335         iscsi_cmnd_init_write(rsp,
2336                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2337         req_cmnd_release(req);
2338 }
2339
2340 static inline int iscsi_get_mgmt_response(int status)
2341 {
2342         switch(status) {
2343         case SCST_MGMT_STATUS_SUCCESS:
2344                 return ISCSI_RESPONSE_FUNCTION_COMPLETE;
2345
2346         case SCST_MGMT_STATUS_TASK_NOT_EXIST:
2347                 return ISCSI_RESPONSE_UNKNOWN_TASK;
2348
2349         case SCST_MGMT_STATUS_LUN_NOT_EXIST:
2350                 return ISCSI_RESPONSE_UNKNOWN_LUN;
2351
2352         case SCST_MGMT_STATUS_FN_NOT_SUPPORTED:
2353                 return ISCSI_RESPONSE_FUNCTION_UNSUPPORTED;
2354
2355         case SCST_MGMT_STATUS_REJECTED:
2356         case SCST_MGMT_STATUS_FAILED:
2357         default:
2358                 return ISCSI_RESPONSE_FUNCTION_REJECTED;
2359         }
2360 }
2361
2362 static void iscsi_task_mgmt_fn_done(struct scst_mgmt_cmd *scst_mcmd)
2363 {
2364         struct iscsi_cmnd *req = (struct iscsi_cmnd*)
2365                                 scst_mgmt_cmd_get_tgt_priv(scst_mcmd);
2366         int status = iscsi_get_mgmt_response(scst_mgmt_cmd_get_status(scst_mcmd));
2367
2368         TRACE_MGMT_DBG("req %p, scst_mcmd %p, scst status %d", req, scst_mcmd, 
2369                 scst_mgmt_cmd_get_status(scst_mcmd));
2370
2371         iscsi_send_task_mgmt_resp(req, status);
2372
2373         scst_mgmt_cmd_set_tgt_priv(scst_mcmd, NULL);
2374 }
2375
2376 static int iscsi_target_detect(struct scst_tgt_template *templ)
2377 {
2378         /* Nothing to do */
2379         return 0;
2380 }
2381
2382 static int iscsi_target_release(struct scst_tgt *scst_tgt)
2383 {
2384         /* Nothing to do */
2385         return 0;
2386 }
2387
2388 struct scst_tgt_template iscsi_template = {
2389         .name = "iscsi",
2390         .sg_tablesize = ISCSI_CONN_IOV_MAX,
2391         .threads_num = 0,
2392         .no_clustering = 1,
2393         .xmit_response_atomic = 0,
2394         .preprocessing_done_atomic = 1,
2395         .detect = iscsi_target_detect,
2396         .release = iscsi_target_release,
2397         .xmit_response = iscsi_xmit_response,
2398 #ifndef NET_PAGE_CALLBACKS_DEFINED
2399         .alloc_data_buf = iscsi_alloc_data_buf,
2400 #endif
2401         .preprocessing_done = iscsi_preprocessing_done,
2402         .pre_exec = iscsi_pre_exec,
2403         .task_mgmt_fn_done = iscsi_task_mgmt_fn_done,
2404 };
2405
2406 static __init int iscsi_run_threads(int count, char *name, int (*fn)(void *))
2407 {
2408         int res = 0;
2409         int i;
2410         struct iscsi_thread_t *thr;
2411
2412         for (i = 0; i < count; i++) {
2413                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
2414                 if (!thr) {
2415                         res = -ENOMEM;
2416                         PRINT_ERROR("Failed to allocate thr %d", res);
2417                         goto out;
2418                 }
2419                 thr->thr = kthread_run(fn, NULL, "%s%d", name, i);
2420                 if (IS_ERR(thr->thr)) {
2421                         res = PTR_ERR(thr->thr);
2422                         PRINT_ERROR("kthread_create() failed: %d", res);
2423                         kfree(thr);
2424                         goto out;
2425                 }
2426                 list_add(&thr->threads_list_entry, &iscsi_threads_list);
2427         }
2428
2429 out:
2430         return res;
2431 }
2432
2433 static void iscsi_stop_threads(void)
2434 {
2435         struct iscsi_thread_t *t, *tmp;
2436
2437         list_for_each_entry_safe(t, tmp, &iscsi_threads_list,
2438                                 threads_list_entry) {
2439                 int rc = kthread_stop(t->thr);
2440                 if (rc < 0) {
2441                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
2442                 }
2443                 list_del(&t->threads_list_entry);
2444                 kfree(t);
2445         }
2446 }
2447
2448 static int __init iscsi_init(void)
2449 {
2450         int err;
2451         int num;
2452
2453         PRINT_INFO("iSCSI SCST Target - version %s", ISCSI_VERSION_STRING);
2454
2455 #ifdef NET_PAGE_CALLBACKS_DEFINED
2456         err = net_set_get_put_page_callbacks(iscsi_get_page_callback,
2457                         iscsi_put_page_callback);
2458         if (err != 0) {
2459                 PRINT_INFO("Unable to set page callbackes: %d", err);
2460                 goto out;
2461         }
2462 #else
2463         PRINT_INFO("%s", "Patch put_page_callback-<kernel-version>.patch "
2464                 "not applied on your kernel. Running in the performance "
2465                 "degraded mode. Refer README file for details");
2466 #endif
2467
2468         BUILD_BUG_ON(MAX_DATA_SEG_LEN != (ISCSI_CONN_IOV_MAX<<PAGE_SHIFT));
2469
2470         if ((ctr_major = register_chrdev(0, ctr_name, &ctr_fops)) < 0) {
2471                 PRINT_ERROR("failed to register the control device %d", ctr_major);
2472                 err = ctr_major;
2473                 goto out_callb;
2474         }
2475
2476         if ((err = event_init()) < 0)
2477                 goto out_reg;
2478
2479         iscsi_cmnd_cache = KMEM_CACHE(iscsi_cmnd, SCST_SLAB_FLAGS);
2480         if (!iscsi_cmnd_cache) {
2481                 err = -ENOMEM;
2482                 goto out_event;
2483         }
2484
2485         if (scst_register_target_template(&iscsi_template) < 0) {
2486                 err = -ENODEV;
2487                 goto out_kmem;
2488         }
2489         iscsi_template_registered = 1;
2490
2491         if ((err = iscsi_procfs_init()) < 0)
2492                 goto out_reg_tmpl;
2493
2494         num = max(num_online_cpus(), 2);
2495
2496         err = iscsi_run_threads(num, "iscsird", istrd);
2497         if (err != 0)
2498                 goto out_thr;
2499
2500         err = iscsi_run_threads(num, "iscsiwr", istwr);
2501         if (err != 0)
2502                 goto out_thr;
2503
2504 out:
2505         return err;
2506
2507 out_thr:
2508         iscsi_procfs_exit();
2509         iscsi_stop_threads();
2510
2511 out_reg_tmpl:
2512         scst_unregister_target_template(&iscsi_template);
2513
2514 out_kmem:
2515         kmem_cache_destroy(iscsi_cmnd_cache);
2516
2517 out_event:
2518         event_exit();
2519
2520 out_reg:
2521         unregister_chrdev(ctr_major, ctr_name);
2522
2523 out_callb:
2524 #ifdef NET_PAGE_CALLBACKS_DEFINED
2525         net_set_get_put_page_callbacks(NULL, NULL);
2526 #endif  
2527         goto out;
2528 }
2529
2530 static void __exit iscsi_exit(void)
2531 {
2532         iscsi_stop_threads();
2533
2534         unregister_chrdev(ctr_major, ctr_name);
2535
2536         iscsi_procfs_exit();
2537         event_exit();
2538
2539         kmem_cache_destroy(iscsi_cmnd_cache);
2540
2541         scst_unregister_target_template(&iscsi_template);
2542
2543 #ifdef NET_PAGE_CALLBACKS_DEFINED
2544         net_set_get_put_page_callbacks(NULL, NULL);
2545 #endif
2546 }
2547
2548 module_init(iscsi_init);
2549 module_exit(iscsi_exit);
2550
2551 MODULE_LICENSE("GPL");