scst/src/scst_lib.c

   1 /*
   2  *  scst_lib.c
   3  *
   4  *  Copyright (C) 2004-2006 Vladislav Bolkhovitin <vst@vlnb.net>
   5  *                 and Leonid Stoljar
   6  *
   7  *  This program is free software; you can redistribute it and/or
   8  *  modify it under the terms of the GNU General Public License
   9  *  as published by the Free Software Foundation, version 2
  10  *  of the License.
  11  *
  12  *  This program is distributed in the hope that it will be useful,
  13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15  *  GNU General Public License for more details.
  16  */
  17
  18 #include <linux/init.h>
  19 #include <linux/kernel.h>
  20 #include <linux/errno.h>
  21 #include <linux/list.h>
  22 #include <linux/spinlock.h>
  23 #include <linux/slab.h>
  24 #include <linux/sched.h>
  25 #include <asm/unistd.h>
  26 #include <asm/string.h>
  27
  28 #ifdef SCST_HIGHMEM
  29 #include <linux/highmem.h>
  30 #endif
  31
  32 #include "scst_debug.h"
  33 #include "scsi_tgt.h"
  34 #include "scst_priv.h"
  35 #include "scst_mem.h"
  36
  37 #include "scst_cdbprobe.h"
  38
  39 static void scst_free_tgt_dev(struct scst_tgt_dev *tgt_dev);
  40
  41 void scst_set_cmd_error_status(struct scst_cmd *cmd, int status)
  42 {
  43         TRACE_ENTRY();
  44
  45         cmd->status = status;
  46         cmd->masked_status = status >> 1;
  47         cmd->host_status = DID_OK;
  48
  49         cmd->data_direction = SCST_DATA_NONE;
  50         cmd->tgt_resp_flags = SCST_TSC_FLAG_STATUS;
  51         cmd->resp_data_len = 0;
  52
  53         TRACE_EXIT();
  54         return;
  55 }
  56
  57 void scst_set_cmd_error(struct scst_cmd *cmd, int key, int asc, int ascq)
  58 {
  59         TRACE_ENTRY();
  60
  61         scst_set_cmd_error_status(cmd, SAM_STAT_CHECK_CONDITION);
  62         scst_set_sense(cmd->sense_buffer, sizeof(cmd->sense_buffer),
  63                 key, asc, ascq);
  64         TRACE_BUFFER("Sense set", cmd->sense_buffer, sizeof(cmd->sense_buffer));
  65
  66         TRACE_EXIT();
  67         return;
  68 }
  69
  70 void scst_set_cmd_error_sense(struct scst_cmd *cmd, uint8_t *sense,
  71         unsigned int len)
  72 {
  73         TRACE_ENTRY();
  74
  75         scst_set_cmd_error_status(cmd, SAM_STAT_CHECK_CONDITION);
  76
  77         memset(cmd->sense_buffer, 0, sizeof(cmd->sense_buffer));
  78         memcpy(cmd->sense_buffer, sense, min((unsigned long)len,
  79                 (unsigned long)sizeof(cmd->sense_buffer)));
  80         TRACE_BUFFER("Sense set", cmd->sense_buffer, sizeof(cmd->sense_buffer));
  81
  82         TRACE_EXIT();
  83         return;
  84 }
  85
  86 void scst_set_busy(struct scst_cmd *cmd)
  87 {
  88         TRACE_ENTRY();
  89
  90         if ((cmd->sess->sess_cmd_count <= 1) ||
  91             (cmd->sess->init_phase != SCST_SESS_IPH_READY))
  92         {
  93                 scst_set_cmd_error_status(cmd, SAM_STAT_BUSY);
  94                 TRACE_MGMT_DBG("Sending BUSY status to initiator %s "
  95                         "(cmds count %d, queue_type %x, sess->init_phase %d)",
  96                         cmd->sess->initiator_name, cmd->sess->sess_cmd_count,
  97                         cmd->queue_type, cmd->sess->init_phase);
  98         } else {
  99                 scst_set_cmd_error_status(cmd, SAM_STAT_TASK_SET_FULL);
 100                 TRACE_MGMT_DBG("Sending QUEUE_FULL status to initiator %s "
 101                         "(cmds count %d, queue_type %x, sess->init_phase %d)",
 102                         cmd->sess->initiator_name, cmd->sess->sess_cmd_count,
 103                         cmd->queue_type, cmd->sess->init_phase);
 104         }
 105
 106         TRACE_EXIT();
 107         return;
 108 }
 109
 110 void scst_set_resp_data_len(struct scst_cmd *cmd, int resp_data_len)
 111 {
 112         int i, l;
 113
 114         TRACE_ENTRY();
 115
 116         scst_check_restore_sg_buff(cmd);
 117         cmd->resp_data_len = resp_data_len;
 118
 119         if (resp_data_len == cmd->bufflen)
 120                 goto out;
 121
 122         l = 0;
 123         for(i = 0; i < cmd->sg_cnt; i++) {
 124                 l += cmd->sg[i].length;
 125                 if (l >= resp_data_len) {
 126                         int left = resp_data_len - (l - cmd->sg[i].length);
 127                         TRACE(TRACE_SG, "cmd %p (tag %d), "
 128                                 "resp_data_len %d, i %d, cmd->sg[i].length %d, "
 129                                 "left %d", cmd, cmd->tag, resp_data_len, i,
 130                                 cmd->sg[i].length, left);
 131                         cmd->orig_sg_cnt = cmd->sg_cnt;
 132                         cmd->orig_sg_entry = i;
 133                         cmd->orig_entry_len = cmd->sg[i].length;
 134                         cmd->sg_cnt = i+1;
 135                         cmd->sg[i].length = left;
 136                         cmd->sg_buff_modified = 1;
 137                         break;
 138                 }
 139         }
 140
 141 out:
 142         TRACE_EXIT();
 143         return;
 144 }
 145
 146 struct scst_device *scst_alloc_device(int gfp_mask)
 147 {
 148         struct scst_device *dev;
 149
 150         TRACE_ENTRY();
 151
 152         dev = kzalloc(sizeof(*dev), gfp_mask);
 153         if (dev == NULL) {
 154                 TRACE(TRACE_OUT_OF_MEM, "%s",
 155                       "Allocation of scst_device failed");
 156                 goto out;
 157         }
 158
 159         spin_lock_init(&dev->dev_lock);
 160         atomic_set(&dev->on_dev_count, 0);
 161         INIT_LIST_HEAD(&dev->blocked_cmd_list);
 162         INIT_LIST_HEAD(&dev->dev_tgt_dev_list);
 163         INIT_LIST_HEAD(&dev->dev_acg_dev_list);
 164         init_waitqueue_head(&dev->on_dev_waitQ);
 165         dev->dev_double_ua_possible = 1;
 166         dev->dev_serialized = 1;
 167
 168 out:
 169         TRACE_EXIT_HRES(dev);
 170         return dev;
 171 }
 172
 173 void scst_free_device(struct scst_device *dev)
 174 {
 175         TRACE_ENTRY();
 176
 177 #ifdef EXTRACHECKS
 178         if (!list_empty(&dev->dev_tgt_dev_list) ||
 179             !list_empty(&dev->dev_acg_dev_list))
 180         {
 181                 PRINT_ERROR_PR("%s: dev_tgt_dev_list or dev_acg_dev_list "
 182                         "is not empty!", __FUNCTION__);
 183                 BUG();
 184         }
 185 #endif
 186
 187         kfree(dev);
 188
 189         TRACE_EXIT();
 190         return;
 191 }
 192
 193 struct scst_acg_dev *scst_alloc_acg_dev(struct scst_acg *acg,
 194         struct scst_device *dev, lun_t lun)
 195 {
 196         struct scst_acg_dev *res;
 197
 198         TRACE_ENTRY();
 199
 200         res = kmem_cache_alloc(scst_acgd_cachep, GFP_KERNEL);
 201         if (res == NULL) {
 202                 TRACE(TRACE_OUT_OF_MEM, "%s", "Allocation of scst_acg_dev failed");
 203                 goto out;
 204         }
 205         memset(res, 0, sizeof(*res));
 206
 207         res->dev = dev;
 208         res->acg = acg;
 209         res->lun = lun;
 210
 211 out:
 212         TRACE_EXIT_HRES(res);
 213         return res;
 214 }
 215
 216 /* scst_mutex supposed to be held */
 217 void scst_free_acg_dev(struct scst_acg_dev *acg_dev)
 218 {
 219         TRACE_ENTRY();
 220
 221         TRACE_DBG("Removing acg_dev %p from acg_dev_list and dev_acg_dev_list",
 222                 acg_dev);
 223         list_del(&acg_dev->acg_dev_list_entry);
 224         list_del(&acg_dev->dev_acg_dev_list_entry);
 225
 226         kmem_cache_free(scst_acgd_cachep, acg_dev);
 227
 228         TRACE_EXIT();
 229         return;
 230 }
 231
 232 /* scst_mutex supposed to be held */
 233 struct scst_acg *scst_alloc_add_acg(const char *acg_name)
 234 {
 235         struct scst_acg *acg;
 236
 237         TRACE_ENTRY();
 238
 239         acg = kzalloc(sizeof(*acg), GFP_KERNEL);
 240         if (acg == NULL) {
 241                 TRACE(TRACE_OUT_OF_MEM, "%s", "Allocation of acg failed");
 242                 goto out;
 243         }
 244
 245         INIT_LIST_HEAD(&acg->acg_dev_list);
 246         INIT_LIST_HEAD(&acg->acg_sess_list);
 247         INIT_LIST_HEAD(&acg->acn_list);
 248         acg->acg_name = acg_name;
 249
 250         TRACE_DBG("Adding acg %s to scst_acg_list", acg_name);
 251         list_add_tail(&acg->scst_acg_list_entry, &scst_acg_list);
 252
 253 out:
 254         TRACE_EXIT_HRES(acg);
 255         return acg;
 256 }
 257
 258 /* scst_mutex supposed to be held */
 259 int scst_destroy_acg(struct scst_acg *acg)
 260 {
 261         struct scst_acn *n, *nn;
 262         struct scst_acg_dev *acg_dev, *acg_dev_tmp;
 263         int res = 0;
 264
 265         TRACE_ENTRY();
 266
 267         if (!list_empty(&acg->acg_sess_list)) {
 268                 PRINT_ERROR_PR("%s: acg_sess_list is not empty!", __FUNCTION__);
 269                 res = -EBUSY;
 270                 goto out;
 271         }
 272
 273         __scst_suspend_activity();
 274
 275         TRACE_DBG("Removing acg %s from scst_acg_list", acg->acg_name);
 276         list_del(&acg->scst_acg_list_entry);
 277
 278         /* Freeing acg_devs */
 279         list_for_each_entry_safe(acg_dev, acg_dev_tmp, &acg->acg_dev_list,
 280                 acg_dev_list_entry)
 281         {
 282                 struct scst_tgt_dev *tgt_dev, *tt;
 283                 list_for_each_entry_safe(tgt_dev, tt,
 284                          &acg_dev->dev->dev_tgt_dev_list,
 285                          dev_tgt_dev_list_entry)
 286                 {
 287                         if (tgt_dev->acg_dev == acg_dev)
 288                                 scst_free_tgt_dev(tgt_dev);
 289                 }
 290                 scst_free_acg_dev(acg_dev);
 291         }
 292
 293         __scst_resume_activity();
 294
 295         /* Freeing names */
 296         list_for_each_entry_safe(n, nn, &acg->acn_list,
 297                 acn_list_entry)
 298         {
 299                 list_del(&n->acn_list_entry);
 300                 kfree(n->name);
 301                 kfree(n);
 302         }
 303         INIT_LIST_HEAD(&acg->acn_list);
 304
 305         kfree(acg);
 306 out:
 307         TRACE_EXIT_RES(res);
 308         return res;
 309 }
 310
 311 /*
 312  * No spin locks supposed to be held, scst_mutex - held.
 313  * The activity is suspended.
 314  */
 315 static struct scst_tgt_dev *scst_alloc_add_tgt_dev(struct scst_session *sess,
 316         struct scst_acg_dev *acg_dev)
 317 {
 318         struct scst_tgt_dev *tgt_dev;
 319         struct scst_device *dev = acg_dev->dev;
 320         int res;
 321
 322         TRACE_ENTRY();
 323
 324         tgt_dev = kmem_cache_alloc(scst_tgtd_cachep, GFP_KERNEL);
 325         if (tgt_dev != NULL)
 326                 memset(tgt_dev, 0, sizeof(*tgt_dev));
 327         else {
 328                 TRACE(TRACE_OUT_OF_MEM, "%s",
 329                       "Allocation of scst_tgt_dev failed");
 330                 goto out;
 331         }
 332
 333         tgt_dev->acg_dev = acg_dev;
 334         tgt_dev->sess = sess;
 335         tgt_dev->cmd_count = 0;
 336
 337         if (dev->scsi_dev != NULL) {
 338                 TRACE(TRACE_DEBUG, "host=%d, channel=%d, id=%d, lun=%d, "
 339                       "SCST lun=%Ld", dev->scsi_dev->host->host_no,
 340                       dev->scsi_dev->channel, dev->scsi_dev->id,
 341                       dev->scsi_dev->lun, (uint64_t)tgt_dev->acg_dev->lun);
 342         }
 343         else {
 344                 TRACE_MGMT_DBG("Virtual device SCST lun=%Ld",
 345                       (uint64_t)tgt_dev->acg_dev->lun);
 346         }
 347
 348         spin_lock_init(&tgt_dev->tgt_dev_lock);
 349         INIT_LIST_HEAD(&tgt_dev->UA_list);
 350         spin_lock_init(&tgt_dev->sn_lock);
 351         INIT_LIST_HEAD(&tgt_dev->deferred_cmd_list);
 352         INIT_LIST_HEAD(&tgt_dev->skipped_sn_list);
 353
 354         spin_lock_bh(&scst_temp_UA_lock);
 355         scst_set_sense(scst_temp_UA, sizeof(scst_temp_UA),
 356                 SCST_LOAD_SENSE(scst_sense_reset_UA));
 357         scst_alloc_set_UA(tgt_dev, scst_temp_UA, sizeof(scst_temp_UA));
 358         spin_unlock_bh(&scst_temp_UA_lock);
 359
 360         tm_dbg_init_tgt_dev(tgt_dev, acg_dev);
 361
 362         if (dev->handler && dev->handler->attach_tgt) {
 363                 TRACE_DBG("Calling dev handler's attach_tgt(%p)",
 364                       tgt_dev);
 365                 res = dev->handler->attach_tgt(tgt_dev);
 366                 TRACE_DBG("%s", "Dev handler's attach_tgt() returned");
 367                 if (res != 0) {
 368                         PRINT_ERROR_PR("Device handler's %s attach_tgt() "
 369                             "failed: %d", dev->handler->name, res);
 370                         goto out_free;
 371                 }
 372         }
 373
 374         list_add_tail(&tgt_dev->dev_tgt_dev_list_entry, &dev->dev_tgt_dev_list);
 375         if (dev->dev_reserved)
 376                 __set_bit(SCST_TGT_DEV_RESERVED, &tgt_dev->tgt_dev_flags);
 377
 378         list_add_tail(&tgt_dev->sess_tgt_dev_list_entry,
 379                 &sess->sess_tgt_dev_list);
 380
 381 out:
 382         TRACE_EXIT();
 383         return tgt_dev;
 384
 385 out_free:
 386         kmem_cache_free(scst_tgtd_cachep, tgt_dev);
 387         tgt_dev = NULL;
 388         goto out;
 389 }
 390
 391 static void scst_send_release(struct scst_tgt_dev *tgt_dev);
 392
 393 /*
 394  * No locks supposed to be held, scst_mutex - held.
 395  * The activity is suspended.
 396  */
 397 void scst_reset_tgt_dev(struct scst_tgt_dev *tgt_dev, int nexus_loss)
 398 {
 399         struct scst_device *dev = tgt_dev->acg_dev->dev;
 400
 401         if (dev->dev_reserved &&
 402             !test_bit(SCST_TGT_DEV_RESERVED, &tgt_dev->tgt_dev_flags))
 403         {
 404                 /* This is one who holds the reservation */
 405                 struct scst_tgt_dev *tgt_dev_tmp;
 406                 list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
 407                                     dev_tgt_dev_list_entry)
 408                 {
 409                         clear_bit(SCST_TGT_DEV_RESERVED,
 410                                     &tgt_dev_tmp->tgt_dev_flags);
 411                 }
 412                 dev->dev_reserved = 0;
 413
 414                 scst_send_release(tgt_dev);
 415         }
 416
 417         spin_lock_bh(&scst_temp_UA_lock);
 418         if (nexus_loss) {
 419                 scst_set_sense(scst_temp_UA, sizeof(scst_temp_UA),
 420                         SCST_LOAD_SENSE(scst_sense_nexus_loss_UA));
 421         } else {
 422                 scst_set_sense(scst_temp_UA, sizeof(scst_temp_UA),
 423                         SCST_LOAD_SENSE(scst_sense_reset_UA));
 424         }
 425         scst_check_set_UA(tgt_dev, scst_temp_UA, sizeof(scst_temp_UA));
 426         spin_unlock_bh(&scst_temp_UA_lock);
 427 }
 428
 429 /*
 430  * No locks supposed to be held, scst_mutex - held.
 431  * The activity is suspended.
 432  */
 433 static void scst_free_tgt_dev(struct scst_tgt_dev *tgt_dev)
 434 {
 435         struct scst_device *dev = tgt_dev->acg_dev->dev;
 436
 437         TRACE_ENTRY();
 438
 439         tm_dbg_deinit_tgt_dev(tgt_dev);
 440
 441         list_del(&tgt_dev->dev_tgt_dev_list_entry);
 442         list_del(&tgt_dev->sess_tgt_dev_list_entry);
 443
 444         scst_reset_tgt_dev(tgt_dev, 0);
 445         scst_free_all_UA(tgt_dev);
 446
 447         if (dev->handler && dev->handler->detach_tgt) {
 448                 TRACE_DBG("Calling dev handler's detach_tgt(%p)",
 449                       tgt_dev);
 450                 dev->handler->detach_tgt(tgt_dev);
 451                 TRACE_DBG("%s", "Dev handler's detach_tgt() returned");
 452         }
 453
 454         kmem_cache_free(scst_tgtd_cachep, tgt_dev);
 455
 456         TRACE_EXIT();
 457         return;
 458 }
 459
 460 /* scst_mutex supposed to be held */
 461 int scst_sess_alloc_tgt_devs(struct scst_session *sess)
 462 {
 463         int res = 0;
 464         struct scst_acg_dev *acg_dev;
 465         struct scst_tgt_dev *tgt_dev;
 466
 467         TRACE_ENTRY();
 468
 469         __scst_suspend_activity();
 470
 471         INIT_LIST_HEAD(&sess->sess_tgt_dev_list);
 472         list_for_each_entry(acg_dev, &sess->acg->acg_dev_list,
 473                 acg_dev_list_entry)
 474         {
 475                 tgt_dev = scst_alloc_add_tgt_dev(sess, acg_dev);
 476                 if (tgt_dev == NULL) {
 477                         res = -ENOMEM;
 478                         goto out_free;
 479                 }
 480         }
 481
 482 out_resume:
 483         __scst_resume_activity();
 484
 485         TRACE_EXIT();
 486         return res;
 487
 488 out_free:
 489         scst_sess_free_tgt_devs(sess);
 490         goto out_resume;
 491 }
 492
 493 /* scst_mutex supposed to be held and activity suspended */
 494 void scst_sess_free_tgt_devs(struct scst_session *sess)
 495 {
 496         struct scst_tgt_dev *tgt_dev, *t;
 497
 498         TRACE_ENTRY();
 499
 500         /* The session is going down, no users, so no locks */
 501         list_for_each_entry_safe(tgt_dev, t, &sess->sess_tgt_dev_list,
 502                                  sess_tgt_dev_list_entry)
 503         {
 504                 scst_free_tgt_dev(tgt_dev);
 505         }
 506         INIT_LIST_HEAD(&sess->sess_tgt_dev_list);
 507
 508         TRACE_EXIT();
 509         return;
 510 }
 511
 512 /* scst_mutex supposed to be held */
 513 int scst_acg_add_dev(struct scst_acg *acg, struct scst_device *dev, lun_t lun,
 514         int read_only)
 515 {
 516         int res = 0;
 517         struct scst_acg_dev *acg_dev;
 518         struct scst_tgt_dev *tgt_dev;
 519         struct scst_session *sess;
 520         LIST_HEAD(tmp_tgt_dev_list);
 521
 522         TRACE_ENTRY();
 523
 524         INIT_LIST_HEAD(&tmp_tgt_dev_list);
 525
 526 #ifdef EXTRACHECKS
 527         list_for_each_entry(acg_dev, &acg->acg_dev_list, acg_dev_list_entry) {
 528                 if (acg_dev->dev == dev) {
 529                         PRINT_ERROR_PR("Device is already in group %s",
 530                                 acg->acg_name);
 531                         res = -EINVAL;
 532                         goto out;
 533                 }
 534         }
 535 #endif
 536
 537         acg_dev = scst_alloc_acg_dev(acg, dev, lun);
 538         if (acg_dev == NULL) {
 539                 res = -ENOMEM;
 540                 goto out;
 541         }
 542         acg_dev->rd_only_flag = read_only;
 543
 544         __scst_suspend_activity();
 545
 546         TRACE_DBG("Adding acg_dev %p to acg_dev_list and dev_acg_dev_list",
 547                 acg_dev);
 548         list_add_tail(&acg_dev->acg_dev_list_entry, &acg->acg_dev_list);
 549         list_add_tail(&acg_dev->dev_acg_dev_list_entry, &dev->dev_acg_dev_list);
 550
 551         list_for_each_entry(sess, &acg->acg_sess_list, acg_sess_list_entry)
 552         {
 553                 tgt_dev = scst_alloc_add_tgt_dev(sess, acg_dev);
 554                 if (tgt_dev == NULL) {
 555                         res = -ENOMEM;
 556                         goto out_free;
 557                 }
 558                 list_add_tail(&tgt_dev->extra_tgt_dev_list_entry,
 559                               &tmp_tgt_dev_list);
 560         }
 561
 562 out_resume:
 563         __scst_resume_activity();
 564
 565 out:
 566         TRACE_EXIT_RES(res);
 567         return res;
 568
 569 out_free:
 570         list_for_each_entry(tgt_dev, &tmp_tgt_dev_list,
 571                          extra_tgt_dev_list_entry)
 572         {
 573                 scst_free_tgt_dev(tgt_dev);
 574         }
 575         scst_free_acg_dev(acg_dev);
 576         goto out_resume;
 577 }
 578
 579 /* scst_mutex supposed to be held */
 580 int scst_acg_remove_dev(struct scst_acg *acg, struct scst_device *dev)
 581 {
 582         int res = 0;
 583         struct scst_acg_dev *acg_dev = NULL, *a;
 584         struct scst_tgt_dev *tgt_dev, *tt;
 585
 586         TRACE_ENTRY();
 587
 588         list_for_each_entry(a, &acg->acg_dev_list, acg_dev_list_entry) {
 589                 if (a->dev == dev) {
 590                         acg_dev = a;
 591                         break;
 592                 }
 593         }
 594
 595         if (acg_dev == NULL) {
 596                 PRINT_ERROR_PR("Device is not found in group %s", acg->acg_name);
 597                 res = -EINVAL;
 598                 goto out;
 599         }
 600
 601         __scst_suspend_activity();
 602
 603         list_for_each_entry_safe(tgt_dev, tt, &dev->dev_tgt_dev_list,
 604                  dev_tgt_dev_list_entry)
 605         {
 606                 if (tgt_dev->acg_dev == acg_dev)
 607                         scst_free_tgt_dev(tgt_dev);
 608         }
 609         scst_free_acg_dev(acg_dev);
 610
 611         __scst_resume_activity();
 612
 613 out:
 614         TRACE_EXIT_RES(res);
 615         return res;
 616 }
 617
 618 /* scst_mutex supposed to be held */
 619 int scst_acg_add_name(struct scst_acg *acg, const char *name)
 620 {
 621         int res = 0;
 622         struct scst_acn *n;
 623         int len;
 624         char *nm;
 625
 626         TRACE_ENTRY();
 627
 628         list_for_each_entry(n, &acg->acn_list, acn_list_entry)
 629         {
 630                 if (strcmp(n->name, name) == 0) {
 631                         PRINT_ERROR_PR("Name %s already exists in access "
 632                                 "control group %s", name, acg->acg_name);
 633                         res = -EINVAL;
 634                         goto out;
 635                 }
 636         }
 637
 638         n = kmalloc(sizeof(*n), GFP_KERNEL);
 639         if (n == NULL) {
 640                 PRINT_ERROR_PR("%s", "Unable to allocate scst_acn");
 641                 res = -ENOMEM;
 642                 goto out;
 643         }
 644
 645         len = strlen(name);
 646         nm = kmalloc(len + 1, GFP_KERNEL);
 647         if (nm == NULL) {
 648                 PRINT_ERROR_PR("%s", "Unable to allocate scst_acn->name");
 649                 res = -ENOMEM;
 650                 goto out_free;
 651         }
 652
 653         strcpy(nm, name);
 654         n->name = nm;
 655
 656         list_add_tail(&n->acn_list_entry, &acg->acn_list);
 657
 658 out:
 659         TRACE_EXIT_RES(res);
 660         return res;
 661
 662 out_free:
 663         kfree(n);
 664         goto out;
 665 }
 666
 667 /* scst_mutex supposed to be held */
 668 int scst_acg_remove_name(struct scst_acg *acg, const char *name)
 669 {
 670         int res = -EINVAL;
 671         struct scst_acn *n;
 672
 673         TRACE_ENTRY();
 674
 675         list_for_each_entry(n, &acg->acn_list, acn_list_entry)
 676         {
 677                 if (strcmp(n->name, name) == 0) {
 678                         list_del(&n->acn_list_entry);
 679                         kfree(n->name);
 680                         kfree(n);
 681                         res = 0;
 682                         break;
 683                 }
 684         }
 685
 686         if (res != 0) {
 687                 PRINT_ERROR_PR("Unable to find name %s in access control "
 688                         "group %s", name, acg->acg_name);
 689         }
 690
 691         TRACE_EXIT_RES(res);
 692         return res;
 693 }
 694
 695 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18)
 696 static void scst_req_done(struct scsi_cmnd *scsi_cmd)
 697 {
 698         struct scsi_request *req;
 699
 700         TRACE_ENTRY();
 701
 702         if (scsi_cmd && (req = scsi_cmd->sc_request)) {
 703                 if (req) {
 704                         if (req->sr_bufflen)
 705                                 kfree(req->sr_buffer);
 706                         scsi_release_request(req);
 707                 }
 708         }
 709
 710         TRACE_EXIT();
 711         return;
 712 }
 713
 714 static void scst_send_release(struct scst_tgt_dev *tgt_dev)
 715 {
 716         struct scsi_request *req;
 717         struct scsi_device *scsi_dev;
 718         uint8_t cdb[6];
 719
 720         TRACE_ENTRY();
 721
 722         if (tgt_dev->acg_dev->dev->scsi_dev == NULL)
 723                 goto out;
 724
 725         scsi_dev = tgt_dev->acg_dev->dev->scsi_dev;
 726
 727         req = scsi_allocate_request(scsi_dev, GFP_KERNEL);
 728         if (req == NULL) {
 729                 PRINT_ERROR_PR("Allocation of scsi_request failed: unable "
 730                             "to RELEASE device %d:%d:%d:%d",
 731                             scsi_dev->host->host_no, scsi_dev->channel,
 732                             scsi_dev->id, scsi_dev->lun);
 733                 goto out;
 734         }
 735
 736         memset(cdb, 0, sizeof(cdb));
 737         cdb[0] = RELEASE;
 738         cdb[1] = (scsi_dev->scsi_level <= SCSI_2) ?
 739             ((scsi_dev->lun << 5) & 0xe0) : 0;
 740         memcpy(req->sr_cmnd, cdb, sizeof(cdb));
 741         req->sr_cmd_len = sizeof(cdb);
 742         req->sr_data_direction = SCST_DATA_NONE;
 743         req->sr_use_sg = 0;
 744         req->sr_bufflen = 0;
 745         req->sr_buffer = NULL;
 746         req->sr_request->rq_disk = tgt_dev->acg_dev->dev->rq_disk;
 747         req->sr_sense_buffer[0] = 0;
 748
 749         TRACE(TRACE_DEBUG | TRACE_SCSI, "Sending RELEASE req %p to SCSI "
 750                 "mid-level", req);
 751         scst_do_req(req, req->sr_cmnd, (void *)req->sr_buffer, req->sr_bufflen,
 752                     scst_req_done, SCST_DEFAULT_TIMEOUT, 3);
 753
 754 out:
 755         TRACE_EXIT();
 756         return;
 757 }
 758 #else /* LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) */
 759 static void scst_send_release(struct scst_tgt_dev *tgt_dev)
 760 {
 761         struct scsi_device *scsi_dev;
 762         unsigned char cdb[6];
 763         unsigned char sense[SCSI_SENSE_BUFFERSIZE];
 764         int rc;
 765
 766         TRACE_ENTRY();
 767
 768         if (tgt_dev->acg_dev->dev->scsi_dev == NULL)
 769                 goto out;
 770
 771         scsi_dev = tgt_dev->acg_dev->dev->scsi_dev;
 772
 773         memset(cdb, 0, sizeof(cdb));
 774         cdb[0] = RELEASE;
 775         cdb[1] = (scsi_dev->scsi_level <= SCSI_2) ?
 776             ((scsi_dev->lun << 5) & 0xe0) : 0;
 777
 778         TRACE(TRACE_DEBUG | TRACE_SCSI, "%s", "Sending RELEASE req to SCSI "
 779                 "mid-level");
 780         rc = scsi_execute(scsi_dev, cdb, SCST_DATA_NONE, NULL, 0,
 781                         sense, SCST_DEFAULT_TIMEOUT,
 782                         3, GFP_KERNEL);
 783         if (rc) {
 784                 PRINT_INFO_PR("scsi_execute() failed: %d", rc);
 785                 goto out;
 786         }
 787
 788 out:
 789         TRACE_EXIT();
 790         return;
 791 }
 792 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) */
 793
 794 struct scst_session *scst_alloc_session(struct scst_tgt *tgt, int gfp_mask,
 795         const char *initiator_name)
 796 {
 797         struct scst_session *sess;
 798         int len;
 799         char *nm;
 800
 801         TRACE_ENTRY();
 802
 803         sess = kmem_cache_alloc(scst_sess_cachep, gfp_mask);
 804         if (sess != NULL)
 805                 memset(sess, 0, sizeof(*sess));
 806         else {
 807                 TRACE(TRACE_OUT_OF_MEM, "%s",
 808                       "Allocation of scst_session failed");
 809                 goto out;
 810         }
 811
 812         sess->init_phase = SCST_SESS_IPH_INITING;
 813         atomic_set(&sess->refcnt, 0);
 814         INIT_LIST_HEAD(&sess->sess_tgt_dev_list);
 815         INIT_LIST_HEAD(&sess->search_cmd_list);
 816         sess->tgt = tgt;
 817         INIT_LIST_HEAD(&sess->init_deferred_cmd_list);
 818         INIT_LIST_HEAD(&sess->init_deferred_mcmd_list);
 819
 820         len = strlen(initiator_name);
 821         nm = kmalloc(len + 1, gfp_mask);
 822         if (nm == NULL) {
 823                 PRINT_ERROR_PR("%s", "Unable to allocate sess->initiator_name");
 824                 goto out_free;
 825         }
 826
 827         strcpy(nm, initiator_name);
 828         sess->initiator_name = nm;
 829
 830 out:
 831         TRACE_EXIT();
 832         return sess;
 833
 834 out_free:
 835         kmem_cache_free(scst_sess_cachep, sess);
 836         sess = NULL;
 837         goto out;
 838 }
 839
 840 void scst_free_session(struct scst_session *sess)
 841 {
 842         TRACE_ENTRY();
 843
 844         down(&scst_mutex);
 845         TRACE_DBG("Removing sess %p from the list", sess);
 846         list_del(&sess->sess_list_entry);
 847         TRACE_DBG("Removing session %p from acg %s", sess, sess->acg->acg_name);
 848         list_del(&sess->acg_sess_list_entry);
 849
 850         __scst_suspend_activity();
 851         scst_sess_free_tgt_devs(sess);
 852         __scst_resume_activity();
 853
 854         wake_up_all(&sess->tgt->unreg_waitQ);
 855
 856         up(&scst_mutex);
 857
 858         kfree(sess->initiator_name);
 859         kmem_cache_free(scst_sess_cachep, sess);
 860
 861         TRACE_EXIT();
 862         return;
 863 }
 864
 865 void scst_free_session_callback(struct scst_session *sess)
 866 {
 867         struct semaphore *shm;
 868
 869         TRACE_ENTRY();
 870
 871         TRACE_DBG("Freeing session %p", sess);
 872
 873         shm = sess->shutdown_mutex;
 874
 875         if (sess->unreg_done_fn) {
 876                 TRACE_DBG("Calling unreg_done_fn(%p)", sess);
 877                 sess->unreg_done_fn(sess);
 878                 TRACE_DBG("%s", "unreg_done_fn() returned");
 879         }
 880         scst_free_session(sess);
 881
 882         if (shm)
 883                 up(shm);
 884
 885         TRACE_EXIT();
 886         return;
 887 }
 888
 889 void scst_sched_session_free(struct scst_session *sess)
 890 {
 891         unsigned long flags;
 892
 893         TRACE_ENTRY();
 894
 895         spin_lock_irqsave(&scst_mgmt_lock, flags);
 896         TRACE_DBG("Adding sess %p to scst_sess_mgmt_list", sess);
 897         list_add_tail(&sess->sess_mgmt_list_entry, &scst_sess_mgmt_list);
 898         spin_unlock_irqrestore(&scst_mgmt_lock, flags);
 899
 900         wake_up(&scst_mgmt_waitQ);
 901
 902         TRACE_EXIT();
 903         return;
 904 }
 905
 906 struct scst_cmd *scst_alloc_cmd(int gfp_mask)
 907 {
 908         struct scst_cmd *cmd;
 909
 910         TRACE_ENTRY();
 911
 912         cmd = kmem_cache_alloc(scst_cmd_cachep, gfp_mask);
 913         if (cmd != NULL)
 914                 memset(cmd, 0, sizeof(*cmd));
 915         else {
 916                 TRACE(TRACE_OUT_OF_MEM, "%s", "Allocation of scst_cmd failed");
 917                 goto out;
 918         }
 919
 920         cmd->queue_type = SCST_CMD_QUEUE_UNTAGGED;
 921         cmd->timeout = SCST_DEFAULT_TIMEOUT;
 922         cmd->retries = 1;
 923         cmd->data_len = -1;
 924         cmd->tgt_resp_flags = SCST_TSC_FLAG_STATUS;
 925         cmd->resp_data_len = -1;
 926
 927 out:
 928         TRACE_EXIT();
 929         return cmd;
 930 }
 931
 932 static void scst_destroy_put_cmd(struct scst_cmd *cmd)
 933 {
 934         scst_sess_put(cmd->sess);
 935
 936         /* At this point tgt_dev can be dead, but the pointer remains not-NULL */
 937         if (likely(cmd->tgt_dev != NULL))
 938                 scst_dec_cmd_count();
 939
 940         scst_destroy_cmd(cmd);
 941         return;
 942 }
 943
 944 /* No locks supposed to be held. Must be called only from scst_finish_cmd()! */
 945 void scst_free_cmd(struct scst_cmd *cmd)
 946 {
 947         int destroy = 1;
 948
 949         TRACE_ENTRY();
 950
 951         BUG_ON(cmd->blocking);
 952
 953 #if defined(EXTRACHECKS) && (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18))
 954         if (cmd->scsi_req) {
 955                 PRINT_ERROR_PR("%s: %s", __FUNCTION__, "Cmd with unfreed "
 956                         "scsi_req!");
 957                 scst_release_request(cmd);
 958         }
 959 #endif
 960
 961         TRACE_DBG("Calling target's on_free_cmd(%p)", cmd);
 962         cmd->tgtt->on_free_cmd(cmd);
 963         TRACE_DBG("%s", "Target's on_free_cmd() returned");
 964
 965         if (likely(cmd->dev != NULL)) {
 966                 struct scst_dev_type *handler = cmd->dev->handler;
 967                 if (handler->on_free_cmd != NULL) {
 968                         TRACE_DBG("Calling dev handler %s on_free_cmd(%p)",
 969                               handler->name, cmd);
 970                         handler->on_free_cmd(cmd);
 971                         TRACE_DBG("Dev handler %s on_free_cmd() returned",
 972                                 handler->name);
 973                 }
 974         }
 975
 976         scst_release_space(cmd);
 977
 978         if (likely(cmd->tgt_dev != NULL)) {
 979 #ifdef EXTRACHECKS
 980                 if (cmd->sent_to_midlev == 0) {
 981                         PRINT_ERROR_PR("Finishing not executed cmd (opcode %d, "
 982                              "target %s, lun %Ld, sn %d, expected_sn %d)",
 983                              cmd->cdb[0], cmd->tgtt->name, (uint64_t)cmd->lun,
 984                              cmd->sn, cmd->tgt_dev->expected_sn);
 985                         scst_inc_expected_sn_unblock(cmd->tgt_dev, cmd, 0);
 986                 }
 987 #endif
 988                 if (unlikely(test_bit(SCST_CMD_OUT_OF_SN,
 989                                 &cmd->cmd_flags)))
 990                 {
 991                         spin_lock_bh(&cmd->tgt_dev->sn_lock);
 992                         set_bit(SCST_CMD_CAN_BE_DESTROYED,
 993                                 &cmd->cmd_flags);
 994                         barrier(); /* to reread SCST_CMD_OUT_OF_SN */
 995                         destroy = !test_bit(SCST_CMD_OUT_OF_SN,
 996                                         &cmd->cmd_flags);
 997                         TRACE(TRACE_SCSI_SERIALIZING, "Out of SN "
 998                                 "cmd %p (tag %d, sn %d), destroy=%d", cmd,
 999                                 cmd->tag, cmd->sn, destroy);
1000                         spin_unlock_bh(&cmd->tgt_dev->sn_lock);
1001                 }
1002         }
1003
1004         if (likely(destroy))
1005                 scst_destroy_put_cmd(cmd);
1006
1007         TRACE_EXIT();
1008         return;
1009 }
1010
1011 /* No locks supposed to be held. */
1012 void scst_check_retries(struct scst_tgt *tgt, int processible_env)
1013 {
1014         int need_wake_up = 0;
1015
1016         TRACE_ENTRY();
1017
1018         /*
1019          * We don't worry about overflow of finished_cmds, because we check
1020          * only for its change
1021          */
1022         atomic_inc(&tgt->finished_cmds);
1023         smp_mb__after_atomic_inc();
1024         if (unlikely(tgt->retry_cmds > 0))
1025         {
1026                 struct scst_cmd *c, *tc;
1027                 unsigned long flags;
1028
1029                 TRACE(TRACE_RETRY, "Checking retry cmd list (retry_cmds %d)",
1030                       tgt->retry_cmds);
1031
1032                 spin_lock_irqsave(&tgt->tgt_lock, flags);
1033                 spin_lock(&scst_list_lock);
1034
1035                 list_for_each_entry_safe(c, tc, &tgt->retry_cmd_list,
1036                                 cmd_list_entry)
1037                 {
1038                         tgt->retry_cmds--;
1039
1040                         TRACE(TRACE_RETRY, "Moving retry cmd %p to active cmd "
1041                             "list (retry_cmds left %d)", c, tgt->retry_cmds);
1042                         list_move(&c->cmd_list_entry, &scst_active_cmd_list);
1043
1044                         need_wake_up++;
1045                         if (need_wake_up >= 2) /* "slow start" */
1046                                 break;
1047                 }
1048
1049                 spin_unlock(&scst_list_lock);
1050                 spin_unlock_irqrestore(&tgt->tgt_lock, flags);
1051         }
1052
1053         if (need_wake_up && !processible_env)
1054                 wake_up(&scst_list_waitQ);
1055
1056         TRACE_EXIT();
1057         return;
1058 }
1059
1060 void scst_tgt_retry_timer_fn(unsigned long arg)
1061 {
1062         struct scst_tgt *tgt = (struct scst_tgt*)arg;
1063         unsigned long flags;
1064
1065         TRACE(TRACE_RETRY, "Retry timer expired (retry_cmds %d)",
1066                 tgt->retry_cmds);
1067
1068         spin_lock_irqsave(&tgt->tgt_lock, flags);
1069         tgt->retry_timer_active = 0;
1070         spin_unlock_irqrestore(&tgt->tgt_lock, flags);
1071
1072         scst_check_retries(tgt, 0);
1073
1074         TRACE_EXIT();
1075         return;
1076 }
1077
1078 struct scst_mgmt_cmd *scst_alloc_mgmt_cmd(int gfp_mask)
1079 {
1080         struct scst_mgmt_cmd *mcmd;
1081
1082         TRACE_ENTRY();
1083
1084         mcmd = mempool_alloc(scst_mgmt_mempool, gfp_mask);
1085         if (mcmd == NULL) {
1086                 PRINT_ERROR("%s", "Allocation of management command "
1087                         "failed, some commands and their data could leak");
1088                 goto out;
1089         }
1090         memset(mcmd, 0, sizeof(*mcmd));
1091
1092 out:
1093         TRACE_EXIT();
1094         return mcmd;
1095 }
1096
1097 void scst_free_mgmt_cmd(struct scst_mgmt_cmd *mcmd, int del)
1098 {
1099         unsigned long flags;
1100
1101         TRACE_ENTRY();
1102
1103         spin_lock_irqsave(&scst_list_lock, flags);
1104         if (del)
1105                 list_del(&mcmd->mgmt_cmd_list_entry);
1106         mcmd->sess->sess_cmd_count--;
1107         spin_unlock_irqrestore(&scst_list_lock, flags);
1108
1109         scst_sess_put(mcmd->sess);
1110
1111         if (mcmd->mcmd_tgt_dev != NULL)
1112                 scst_dec_cmd_count();
1113
1114         mempool_free(mcmd, scst_mgmt_mempool);
1115
1116         TRACE_EXIT();
1117         return;
1118 }
1119
1120 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18)
1121 int scst_alloc_request(struct scst_cmd *cmd)
1122 {
1123         int res = 0;
1124         struct scsi_request *req;
1125         int gm = scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL;
1126
1127         TRACE_ENTRY();
1128
1129         /* cmd->dev->scsi_dev must be non-NULL here */
1130         req = scsi_allocate_request(cmd->dev->scsi_dev, gm);
1131         if (req == NULL) {
1132                 TRACE(TRACE_OUT_OF_MEM, "%s",
1133                       "Allocation of scsi_request failed");
1134                 res = -ENOMEM;
1135                 goto out;
1136         }
1137
1138         cmd->scsi_req = req;
1139
1140         memcpy(req->sr_cmnd, cmd->cdb, cmd->cdb_len);
1141         req->sr_cmd_len = cmd->cdb_len;
1142         req->sr_data_direction = cmd->data_direction;
1143         req->sr_use_sg = cmd->sg_cnt;
1144         req->sr_bufflen = cmd->bufflen;
1145         req->sr_buffer = cmd->sg;
1146         req->sr_request->rq_disk = cmd->dev->rq_disk;
1147         req->sr_sense_buffer[0] = 0;
1148
1149         cmd->scsi_req->upper_private_data = cmd;
1150
1151 out:
1152         TRACE_EXIT();
1153         return res;
1154 }
1155
1156 void scst_release_request(struct scst_cmd *cmd)
1157 {
1158         scsi_release_request(cmd->scsi_req);
1159         cmd->scsi_req = NULL;
1160 }
1161 #endif
1162
1163 int scst_alloc_space(struct scst_cmd *cmd)
1164 {
1165         int tgt_sg = cmd->tgt->sg_tablesize;
1166         int ini_sg;
1167         int gfp_mask;
1168         int res = -ENOMEM;
1169         int ini_unchecked_isa_dma, ini_use_clustering;
1170         struct sgv_pool *pool;
1171         struct sgv_pool_obj *sgv;
1172
1173         TRACE_ENTRY();
1174
1175         if (cmd->data_buf_alloced) {
1176                 TRACE_MEM("%s", "data_buf_alloced set, returning");
1177                 BUG_ON(cmd->sg == NULL);
1178                 res = 0;
1179                 goto out;
1180         }
1181
1182         gfp_mask = __GFP_NOWARN;
1183         gfp_mask |= (scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
1184         pool = &scst_sgv.norm;
1185
1186         if (cmd->dev->scsi_dev != NULL) {
1187                 ini_sg = cmd->dev->scsi_dev->host->sg_tablesize;
1188                 ini_unchecked_isa_dma =
1189                         cmd->dev->scsi_dev->host->unchecked_isa_dma;
1190                 ini_use_clustering =
1191                         (cmd->dev->scsi_dev->host->use_clustering ==
1192                                 ENABLE_CLUSTERING);
1193         }
1194         else {
1195                 ini_sg = (1 << 15) /* infinite */;
1196                 ini_unchecked_isa_dma = 0;
1197                 ini_use_clustering = 0;
1198         }
1199
1200         if (cmd->tgtt->use_clustering || ini_use_clustering)
1201         {
1202                 TRACE_MEM("%s", "Use clustering");
1203                 pool = &scst_sgv.norm_clust;
1204         }
1205
1206         if (cmd->tgtt->unchecked_isa_dma || ini_unchecked_isa_dma) {
1207                 TRACE_MEM("%s", "Use ISA DMA memory");
1208                 gfp_mask |= GFP_DMA;
1209                 pool = &scst_sgv.dma;
1210         } else {
1211 #ifdef SCST_HIGHMEM
1212                 gfp_mask |= __GFP_HIGHMEM;
1213                 pool = &scst_sgv.highmem;
1214 #endif
1215         }
1216
1217         sgv = sgv_pool_alloc(pool, cmd->bufflen, gfp_mask, &cmd->sg_cnt);
1218         if (sgv == NULL)
1219                 goto out;
1220
1221         if (unlikely(cmd->sg_cnt > ini_sg)) {
1222                 static int ll;
1223                 if (ll < 10) {
1224                         PRINT_INFO("Unable to complete command due to "
1225                                 "underlying device SG IO count limitation "
1226                                 "(requested %d, available %d)", cmd->sg_cnt,
1227                                 ini_sg);
1228                         ll++;
1229                 }
1230                 goto out_sgv_free;
1231         }
1232
1233         if (unlikely(cmd->sg_cnt > tgt_sg)) {
1234                 static int ll;
1235                 if (ll < 10) {
1236                         PRINT_INFO("Unable to complete command due to "
1237                                 "target device %s SG IO count limitation "
1238                                 "(requested %d, available %d)", cmd->tgtt->name,
1239                                 cmd->sg_cnt, tgt_sg);
1240                         ll++;
1241                 }
1242                 goto out_sgv_free;
1243         }
1244
1245         cmd->sgv = sgv;
1246         cmd->sg = sgv_pool_sg(sgv);
1247
1248         res = 0;
1249
1250 out:
1251         TRACE_EXIT();
1252         return res;
1253
1254 out_sgv_free:
1255         sgv_pool_free(sgv);
1256         cmd->sg_cnt = 0;
1257         goto out;
1258 }
1259
1260 void scst_release_space(struct scst_cmd *cmd)
1261 {
1262         TRACE_ENTRY();
1263
1264         if (cmd->data_buf_alloced) {
1265                 TRACE_MEM("%s", "data_buf_alloced set, returning");
1266                 goto out;
1267         }
1268
1269         if (cmd->sgv) {
1270                 scst_check_restore_sg_buff(cmd);
1271                 sgv_pool_free(cmd->sgv);
1272         }
1273
1274         cmd->sgv = NULL;
1275         cmd->sg_cnt = 0;
1276         cmd->sg = NULL;
1277         cmd->bufflen = 0;
1278         cmd->data_len = 0;
1279
1280 out:
1281         TRACE_EXIT();
1282         return;
1283 }
1284
1285 int __scst_get_buf(struct scst_cmd *cmd, uint8_t **buf)
1286 {
1287         int res = 0;
1288         struct scatterlist *sg = cmd->sg;
1289         int i = cmd->get_sg_buf_entry_num;
1290
1291         TRACE_ENTRY();
1292
1293         *buf = NULL;
1294
1295         if (i >= cmd->sg_cnt)
1296                 goto out;
1297 #ifdef SCST_HIGHMEM
1298         /*
1299          * HIGHMEM pages not merged (clustered), so if it's
1300          * not HIGHMEM page, kmap() is the same as page_address()
1301          */
1302         if (scst_cmd_atomic(cmd)) {
1303                 enum km_type km;
1304                 if (in_softirq())
1305                         km = KM_SOFTIRQ0;
1306                 else
1307                         km = KM_USER0;
1308                 *buf = kmap_atomic(sg[i].page, km);
1309         } else
1310                 *buf = kmap(sg[i].page);
1311 #else
1312         *buf = page_address(sg[i].page);
1313 #endif
1314         res = sg[i].length;
1315         cmd->get_sg_buf_entry_num++;
1316
1317 out:
1318         TRACE_EXIT_RES(res);
1319         return res;
1320 }
1321
1322 static const int SCST_CDB_LENGTH[8] = { 6, 10, 10, -1, 16, 12, -1, -1 };
1323
1324 #define SCST_CDB_GROUP(opcode)   ((opcode >> 5) & 0x7)
1325 #define SCST_GET_CDB_LEN(opcode) SCST_CDB_LENGTH[SCST_CDB_GROUP(opcode)]
1326
1327 int scst_get_cdb_len(const uint8_t *cdb)
1328 {
1329         return SCST_GET_CDB_LEN(cdb[0]);
1330 }
1331
1332 int scst_get_cdb_info(const uint8_t *cdb_p, int dev_type,
1333                       struct scst_info_cdb *info_p)
1334 {
1335         int i, res = 0;
1336         uint8_t op;
1337         const struct scst_sdbops *ptr = NULL;
1338
1339         TRACE_ENTRY();
1340
1341         memset(info_p, 0, sizeof(*info_p));
1342         info_p->direction = SCST_DATA_NONE;
1343         info_p->op_name = "NOOP";
1344         op = *cdb_p;    /* get clear opcode */
1345
1346         TRACE(TRACE_SCSI, "opcode=%02x, cdblen=%d bytes, tblsize=%zd, "
1347                 "dev_type=%d", op, SCST_GET_CDB_LEN(op), SCST_CDB_TBL_SIZE,
1348                 dev_type);
1349
1350         i = scst_scsi_op_list[op];
1351         while (i < SCST_CDB_TBL_SIZE && scst_scsi_op_table[i].ops == op) {
1352                 if (scst_scsi_op_table[i].devkey[dev_type] != SCST_CDB_NOTSUPP) {
1353                         ptr = &scst_scsi_op_table[i];
1354 #if 0
1355                         TRACE(TRACE_SCSI, "op = 0x%02x+'%c%c%c%c%c%c%c%c%c%c'+<%s>",
1356                               ptr->ops, ptr->devkey[0], /* disk     */
1357                               ptr->devkey[1],   /* tape     */
1358                               ptr->devkey[2],   /* printer */
1359                               ptr->devkey[3],   /* cpu      */
1360                               ptr->devkey[4],   /* cdr      */
1361                               ptr->devkey[5],   /* cdrom    */
1362                               ptr->devkey[6],   /* scanner */
1363                               ptr->devkey[7],   /* worm     */
1364                               ptr->devkey[8],   /* changer */
1365                               ptr->devkey[9],   /* commdev */
1366                               ptr->op_name);
1367
1368                         TRACE(TRACE_SCSI,
1369                               "direction=%d size_field_len=%d fixed=%d flag1=%d flag2=%d",
1370                               ptr->direction,
1371                               ptr->size_field_len,
1372                               ptr->fixed, ptr->flag1, ptr->flag2);
1373 #endif
1374                         break;
1375                 }
1376                 i++;
1377         }
1378
1379         if (ptr == NULL) {
1380                 /* opcode not found or now not used !!! */
1381                 TRACE(TRACE_SCSI, "Unknown opcode 0x%x for type %d", op,
1382                       dev_type);
1383                 res = -1;
1384                 goto out;
1385         }
1386
1387         info_p->cdb_len = SCST_GET_CDB_LEN(op);
1388         info_p->op_name = ptr->op_name;
1389         /* 1. direction */
1390         info_p->direction = ptr->direction;
1391         if (info_p->direction == SCST_DATA_NONE)
1392                 goto out;
1393
1394         /* 2. flags */
1395         info_p->flags = ptr->fixed;
1396
1397         /*
1398          * CDB length needed, because we must know offsets:
1399          * 1) for  6-bytes CDB len = 1 byte or 3 bytes(if real transfer exist)
1400          * 2) for 10-bytes CDB len = 1 byte or 2 bytes(0x24,0x25 = 3)
1401          * 3) for 12-bytes CDB len = 1 byte or 4 bytes
1402          */
1403
1404         /* 3. transfer_len */
1405         if (SCST_GET_CDB_LEN(op) == 6) {
1406                 if (ptr->size_field_len == 3) {
1407                         /* length = 3 bytes */
1408                         info_p->transfer_len = (((*(cdb_p + 2)) & 0xff) << 16) +
1409                             (((*(cdb_p + 3)) & 0xff) << 8) +
1410                             ((*(cdb_p + 4)) & 0xff);
1411                         info_p->transfer_len &= 0xffffff;
1412                 } else if (ptr->size_field_len == 1) {
1413                         /*
1414                          * Warning!!! CDB 'READ BLOCK LIMITS'
1415                          * always returns 6-byte block with limits
1416                          * info_p->transfer_len = (int)(*(cdb_p + 4));
1417                          */
1418                         info_p->transfer_len = ((op == READ_BLOCK_LIMITS) ?
1419                                                 SCST_BLOCK_LIMIT_LEN :
1420                                                 *(cdb_p + 4)) & 0xff;
1421                 }
1422         } else if (SCST_GET_CDB_LEN(op) == 10) {
1423                 if (ptr->size_field_len == 3)
1424                         /*
1425                          * SET window usees 3 bytes length SET/GET WINDOW
1426                          * if ((uint8_t)ptr->ops == 0x24 || 0x25)
1427                          */
1428                 {
1429                         info_p->transfer_len = (((*(cdb_p + 6)) & 0xff) << 16) +
1430                             (((*(cdb_p + 7)) & 0xff) << 8) +
1431                             ((*(cdb_p + 8)) & 0xff);
1432                         info_p->transfer_len &= 0xffffff;
1433                 } else if (ptr->size_field_len == 2) {
1434                         info_p->transfer_len = (((*(cdb_p + 7)) & 0xff) << 8) +
1435                             ((*(cdb_p + 8)) & 0xff);
1436                         info_p->transfer_len &= 0xffff;
1437                 } else if (ptr->size_field_len == 1) {
1438                         info_p->transfer_len = (*(cdb_p + 8));
1439
1440                         /* opcode = READ-WRITE UPDATED BLOCK */
1441                         if ((ptr->ops == UPDATE_BLOCK) ||
1442                             (ptr->ops == WRITE_SAME)) {
1443                                 /* the opcode always returns 1 block */
1444                                 info_p->flags |= SCST_TRANSFER_LEN_TYPE_FIXED;
1445                                 info_p->transfer_len = 1;
1446                         }
1447
1448                         if ((ptr->ops == COMPARE) || (ptr->ops == COPY_VERIFY)) {
1449                                 /* ese other place in CDB [3,4],5 */
1450                                 info_p->transfer_len = (*(cdb_p + 5));
1451                         }
1452
1453                         info_p->transfer_len &= 0xff;
1454                 }
1455         } else if (SCST_GET_CDB_LEN(op) == 12) {
1456                 if (ptr->size_field_len == 4) {
1457                         info_p->transfer_len = (((*(cdb_p + 6)) & 0xff) << 24) +
1458                             (((*(cdb_p + 7)) & 0xff) << 16) +
1459                             (((*(cdb_p + 8)) & 0xff) << 8) +
1460                             ((*(cdb_p + 9)) & 0xff);
1461                         info_p->transfer_len &= 0xffffffff;
1462                 } else if (ptr->size_field_len == 3) {
1463                         info_p->transfer_len = (((*(cdb_p + 7)) & 0xff) << 16) +
1464                             (((*(cdb_p + 8)) & 0xff) << 8) +
1465                             ((*(cdb_p + 9)) & 0xff);
1466                         info_p->transfer_len &= 0xffffff;
1467                 } else if (ptr->size_field_len == 2) {
1468                         info_p->transfer_len = (((*(cdb_p + 8)) & 0xff) << 8) +
1469                             ((*(cdb_p + 9)) & 0xff);
1470                         info_p->transfer_len &= 0xffff;
1471                 } else {
1472                         if (ptr->size_field_len == 1) {
1473                                 info_p->transfer_len = (*(cdb_p + 9));
1474                                 info_p->transfer_len &= 0xff;
1475                         }
1476                 }
1477         } else if (SCST_GET_CDB_LEN(op) == 16) {
1478                 if (ptr->size_field_len == 4) {
1479                         info_p->transfer_len =
1480                             (((*(cdb_p + 10)) & 0xff) << 24) +
1481                             (((*(cdb_p + 11)) & 0xff) << 16) +
1482                             (((*(cdb_p + 12)) & 0xff) << 8) +
1483                             ((*(cdb_p + 13)) & 0xff);
1484                 }
1485         }
1486         if (!info_p->transfer_len) {
1487                 TRACE_DBG("Warning! transfer_len 0, direction %d change on %d",
1488                         info_p->direction, SCST_DATA_NONE);
1489                 info_p->direction = SCST_DATA_NONE;
1490         }
1491
1492 out:
1493         TRACE_EXIT();
1494         return res;
1495 }
1496
1497 void scst_scsi_op_list_init(void)
1498 {
1499         int i;
1500         uint8_t op = 0xff;
1501
1502         TRACE_ENTRY();
1503
1504         for (i = 0; i < 256; i++)
1505                 scst_scsi_op_list[i] = SCST_CDB_TBL_SIZE;
1506
1507         for (i = 0; i < SCST_CDB_TBL_SIZE; i++) {
1508                 if (scst_scsi_op_table[i].ops != op) {
1509                         op = scst_scsi_op_table[i].ops;
1510                         scst_scsi_op_list[op] = i;
1511                 }
1512         }
1513
1514         TRACE_EXIT();
1515         return;
1516 }
1517
1518 /*
1519  * Routine to extract a lun number from an 8-byte LUN structure
1520  * in network byte order (BE).
1521  * (see SAM-2, Section 4.12.3 page 40)
1522  * Supports 2 types of lun unpacking: peripheral and logical unit.
1523  */
1524 lun_t scst_unpack_lun(const uint8_t *lun, int len)
1525 {
1526         lun_t res = (lun_t)-1;
1527         int address_method;
1528
1529         TRACE_ENTRY();
1530
1531         TRACE_BUFF_FLAG(TRACE_DEBUG, "Raw LUN", lun, len);
1532
1533         if (len < 2) {
1534                 PRINT_ERROR_PR("Illegal lun length %d, expected 2 bytes or "
1535                         "more", len);
1536                 goto out;
1537         }
1538
1539         if (len > 2) {
1540                 switch(len) {
1541                 case 8:
1542                 {
1543                         if ((*((uint64_t*)lun) &
1544                           __constant_cpu_to_be64(0x0000FFFFFFFFFFFFLL)) != 0)
1545                                 goto out_err;
1546                         break;
1547                 }
1548                 case 4:
1549                         if (*((uint16_t*)&lun[2]) != 0)
1550                                 goto out_err;
1551                         break;
1552                 case 6:
1553                         if (*((uint32_t*)&lun[2]) != 0)
1554                                 goto out_err;
1555                         break;
1556                 default:
1557                         goto out_err;
1558                 }
1559         }
1560
1561         address_method = (*lun) >> 6;   /* high 2 bits of byte 0 */
1562         switch (address_method) {
1563         case 0: /* peripheral device addressing method */
1564 #if 0 /* At least QLA2300 Linux ini uses it as the flat space addressing method */
1565                 if (*lun) {
1566                         PRINT_ERROR_PR("Illegal BUS INDENTIFIER in LUN "
1567                              "peripheral device addressing method 0x%02x, "
1568                              "expected 0", *lun);
1569                         break;
1570                 }
1571                 res = *(lun + 1);
1572                 break;
1573 #else
1574                 /* go through */
1575 #endif
1576
1577         case 1: /* flat space addressing method */
1578                 res = *(lun + 1) | (((*lun) & 0x3f) << 8);
1579                 break;
1580
1581         case 2: /* logical unit addressing method */
1582                 if (*lun & 0x3f) {
1583                         PRINT_ERROR_PR("Illegal BUS NUMBER in LUN logical unit "
1584                                     "addressing method 0x%02x, expected 0",
1585                                     *lun & 0x3f);
1586                         break;
1587                 }
1588                 if (*(lun + 1) & 0xe0) {
1589                         PRINT_ERROR_PR("Illegal TARGET in LUN logical unit "
1590                                     "addressing method 0x%02x, expected 0",
1591                                     (*(lun + 1) & 0xf8) >> 5);
1592                         break;
1593                 }
1594                 res = *(lun + 1) & 0x1f;
1595                 break;
1596
1597         case 3: /* extended logical unit addressing method */
1598         default:
1599                 PRINT_ERROR_PR("Unimplemented LUN addressing method %u",
1600                             address_method);
1601                 break;
1602         }
1603
1604 out:
1605         TRACE_EXIT_RES((int)res);
1606         return res;
1607
1608 out_err:
1609         PRINT_ERROR_PR("%s", "Multi-level LUN unimplemented");
1610         goto out;
1611 }
1612
1613 /* Called under dev_lock and BH off */
1614 void scst_process_reset(struct scst_device *dev,
1615         struct scst_session *originator, struct scst_cmd *exclude_cmd,
1616         struct scst_mgmt_cmd *mcmd)
1617 {
1618         struct scst_tgt_dev *tgt_dev;
1619         struct scst_cmd *cmd, *tcmd;
1620         int wake = 0;
1621
1622         TRACE_ENTRY();
1623
1624         /* Clear RESERVE'ation, if necessary */
1625         if (dev->dev_reserved) {
1626                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
1627                                     dev_tgt_dev_list_entry)
1628                 {
1629                         TRACE(TRACE_MGMT, "Clearing RESERVE'ation for tgt_dev "
1630                                 "lun %d", tgt_dev->acg_dev->lun);
1631                         clear_bit(SCST_TGT_DEV_RESERVED,
1632                                   &tgt_dev->tgt_dev_flags);
1633                 }
1634                 dev->dev_reserved = 0;
1635                 /*
1636                  * There is no need to send RELEASE, since the device is going
1637                  * to be resetted
1638                  */
1639         }
1640
1641         dev->dev_double_ua_possible = 1;
1642         dev->dev_serialized = 1;
1643
1644         /* BH already off */
1645         spin_lock(&scst_temp_UA_lock);
1646         scst_set_sense(scst_temp_UA, sizeof(scst_temp_UA),
1647                 SCST_LOAD_SENSE(scst_sense_reset_UA));
1648         __scst_process_UA(dev, exclude_cmd, scst_temp_UA, sizeof(scst_temp_UA),
1649                 1);
1650         spin_unlock(&scst_temp_UA_lock);
1651
1652         list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
1653                 dev_tgt_dev_list_entry)
1654         {
1655                 struct scst_session *sess = tgt_dev->sess;
1656
1657                 local_bh_disable();
1658                 spin_lock_irq(&scst_list_lock);
1659
1660                 TRACE_DBG("Searching in search cmd list (sess=%p)", sess);
1661                 list_for_each_entry(cmd, &sess->search_cmd_list,
1662                                 search_cmd_list_entry) {
1663                         if (cmd == exclude_cmd)
1664                                 continue;
1665                         if ((cmd->tgt_dev == tgt_dev) ||
1666                             ((cmd->tgt_dev == NULL) &&
1667                              (cmd->lun == tgt_dev->acg_dev->lun))) {
1668                                 scst_abort_cmd(cmd, mcmd,
1669                                         (tgt_dev->sess != originator), 0);
1670                         }
1671                 }
1672                 spin_unlock_irq(&scst_list_lock);
1673                 local_bh_enable();
1674         }
1675
1676         list_for_each_entry_safe(cmd, tcmd, &dev->blocked_cmd_list,
1677                                 blocked_cmd_list_entry) {
1678                 if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
1679                         list_del(&cmd->blocked_cmd_list_entry);
1680                         TRACE_MGMT_DBG("Moving aborted blocked cmd %p "
1681                                 "to active cmd list", cmd);
1682                         spin_lock_irq(&scst_list_lock);
1683                         list_move_tail(&cmd->cmd_list_entry,
1684                                 &scst_active_cmd_list);
1685                         spin_unlock_irq(&scst_list_lock);
1686                         wake = 1;
1687                 }
1688         }
1689
1690         if (wake)
1691                 wake_up(&scst_list_waitQ);
1692
1693         TRACE_EXIT();
1694         return;
1695 }
1696
1697 int scst_set_pending_UA(struct scst_cmd *cmd)
1698 {
1699         int res = 0;
1700         struct scst_tgt_dev_UA *UA_entry;
1701
1702         TRACE_ENTRY();
1703
1704         TRACE(TRACE_MGMT, "Setting pending UA cmd %p", cmd);
1705
1706         spin_lock_bh(&cmd->tgt_dev->tgt_dev_lock);
1707
1708         /* UA list could be cleared behind us, so retest */
1709         if (list_empty(&cmd->tgt_dev->UA_list)) {
1710                 TRACE_DBG("%s",
1711                       "SCST_TGT_DEV_UA_PENDING set, but UA_list empty");
1712                 res = -1;
1713                 goto out_unlock;
1714         }
1715
1716         UA_entry = list_entry(cmd->tgt_dev->UA_list.next, typeof(*UA_entry),
1717                               UA_list_entry);
1718
1719         TRACE_DBG("next %p UA_entry %p",
1720               cmd->tgt_dev->UA_list.next, UA_entry);
1721
1722         scst_set_cmd_error_sense(cmd, UA_entry->UA_sense_buffer,
1723                 sizeof(UA_entry->UA_sense_buffer));
1724
1725         cmd->ua_ignore = 1;
1726
1727         list_del(&UA_entry->UA_list_entry);
1728
1729         mempool_free(UA_entry, scst_ua_mempool);
1730
1731         if (list_empty(&cmd->tgt_dev->UA_list)) {
1732                 clear_bit(SCST_TGT_DEV_UA_PENDING,
1733                           &cmd->tgt_dev->tgt_dev_flags);
1734         }
1735
1736         spin_unlock_bh(&cmd->tgt_dev->tgt_dev_lock);
1737
1738 out:
1739         TRACE_EXIT_RES(res);
1740         return res;
1741
1742 out_unlock:
1743         spin_unlock_bh(&cmd->tgt_dev->tgt_dev_lock);
1744         goto out;
1745 }
1746
1747 /* Called under dev_lock, tgt_dev_lock and BH off */
1748 void scst_alloc_set_UA(struct scst_tgt_dev *tgt_dev,
1749         const uint8_t *sense, int sense_len)
1750 {
1751         struct scst_tgt_dev_UA *UA_entry = NULL;
1752
1753         TRACE_ENTRY();
1754
1755         UA_entry = mempool_alloc(scst_ua_mempool, GFP_ATOMIC);
1756         if (UA_entry == NULL) {
1757                 PRINT_ERROR_PR("%s", "UNIT ATTENTION memory "
1758                      "allocation failed. The UNIT ATTENTION "
1759                      "on some sessions will be missed");
1760                 goto out;
1761         }
1762         memset(UA_entry, 0, sizeof(*UA_entry));
1763
1764         if (sense_len > sizeof(UA_entry->UA_sense_buffer))
1765                 sense_len = sizeof(UA_entry->UA_sense_buffer);
1766         memcpy(UA_entry->UA_sense_buffer, sense, sense_len);
1767         set_bit(SCST_TGT_DEV_UA_PENDING, &tgt_dev->tgt_dev_flags);
1768         smp_mb__after_set_bit();
1769         list_add_tail(&UA_entry->UA_list_entry, &tgt_dev->UA_list);
1770
1771 out:
1772         TRACE_EXIT();
1773         return;
1774 }
1775
1776 /* Called under dev_lock and BH off */
1777 void scst_check_set_UA(struct scst_tgt_dev *tgt_dev,
1778         const uint8_t *sense, int sense_len)
1779 {
1780         int skip_UA = 0;
1781         struct scst_tgt_dev_UA *UA_entry_tmp;
1782
1783         TRACE_ENTRY();
1784
1785         spin_lock(&tgt_dev->tgt_dev_lock);
1786
1787         list_for_each_entry(UA_entry_tmp, &tgt_dev->UA_list,
1788                             UA_list_entry)
1789         {
1790                 if (sense[12] == UA_entry_tmp->UA_sense_buffer[12]) {
1791                         skip_UA = 1;
1792                         break;
1793                 }
1794         }
1795
1796         if (skip_UA == 0)
1797                 scst_alloc_set_UA(tgt_dev, sense, sense_len);
1798
1799         spin_unlock(&tgt_dev->tgt_dev_lock);
1800
1801         TRACE_EXIT();
1802         return;
1803 }
1804
1805 /* Called under dev_lock and BH off */
1806 void __scst_process_UA(struct scst_device *dev,
1807         struct scst_cmd *exclude, const uint8_t *sense, int sense_len,
1808         int internal)
1809 {
1810         struct scst_tgt_dev *tgt_dev, *exclude_tgt_dev = NULL;
1811
1812         TRACE_ENTRY();
1813
1814         TRACE(TRACE_MGMT, "Processing UA dev %p", dev);
1815
1816         if (exclude != NULL)
1817                 exclude_tgt_dev = exclude->tgt_dev;
1818
1819         /* Check for reset UA */
1820         if (!internal && (sense[12] == SCST_SENSE_ASC_UA_RESET)) {
1821                 scst_process_reset(dev, (exclude != NULL) ? exclude->sess : NULL,
1822                         exclude, NULL);
1823         }
1824
1825         list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
1826                                 dev_tgt_dev_list_entry) {
1827                 if (tgt_dev != exclude_tgt_dev)
1828                         scst_check_set_UA(tgt_dev, sense, sense_len);
1829         }
1830
1831         TRACE_EXIT();
1832         return;
1833 }
1834
1835 /* Called under tgt_dev_lock or when tgt_dev is unused */
1836 void scst_free_all_UA(struct scst_tgt_dev *tgt_dev)
1837 {
1838         struct scst_tgt_dev_UA *UA_entry, *t;
1839
1840         TRACE_ENTRY();
1841
1842         list_for_each_entry_safe(UA_entry, t, &tgt_dev->UA_list, UA_list_entry) {
1843                 TRACE_MGMT_DBG("Clearing UA for tgt_dev lun %d",
1844                         tgt_dev->acg_dev->lun);
1845                 list_del(&UA_entry->UA_list_entry);
1846                 kfree(UA_entry);
1847         }
1848         INIT_LIST_HEAD(&tgt_dev->UA_list);
1849         clear_bit(SCST_TGT_DEV_UA_PENDING, &tgt_dev->tgt_dev_flags);
1850
1851         TRACE_EXIT();
1852         return;
1853 }
1854
1855 struct scst_cmd *__scst_check_deferred_commands(struct scst_tgt_dev *tgt_dev,
1856         int expected_sn)
1857 {
1858         struct scst_cmd *cmd = NULL, *tcmd;
1859
1860         if (tgt_dev->def_cmd_count == 0)
1861                 goto out;
1862
1863         spin_lock_bh(&tgt_dev->sn_lock);
1864
1865 restart:
1866         list_for_each_entry(tcmd, &tgt_dev->deferred_cmd_list,
1867                                 sn_cmd_list_entry) {
1868                 if (tcmd->sn == expected_sn) {
1869                         TRACE(TRACE_SCSI_SERIALIZING,
1870                               "Deferred command sn %d found", tcmd->sn);
1871                         tgt_dev->def_cmd_count--;
1872                         list_del(&tcmd->sn_cmd_list_entry);
1873                         cmd = tcmd;
1874                         goto out_unlock;
1875                 }
1876         }
1877
1878         list_for_each_entry(tcmd, &tgt_dev->skipped_sn_list,
1879                                 sn_cmd_list_entry) {
1880                 if (tcmd->sn == expected_sn) {
1881                         /*
1882                          * !! At this point any pointer in tcmd, except      !!
1883                          * !! sn_cmd_list_entry, could be already destroyed  !!
1884                          */
1885                         TRACE(TRACE_SCSI_SERIALIZING,
1886                               "cmd %p (tag %d) with skipped sn %d found", tcmd,
1887                               tcmd->tag, tcmd->sn);
1888                         tgt_dev->def_cmd_count--;
1889                         list_del(&tcmd->sn_cmd_list_entry);
1890                         if (test_bit(SCST_CMD_CAN_BE_DESTROYED,
1891                                         &tcmd->cmd_flags)) {
1892                                 scst_destroy_put_cmd(tcmd);
1893                         } else {
1894                                 smp_mb__before_clear_bit();
1895                                 clear_bit(SCST_CMD_OUT_OF_SN, &tcmd->cmd_flags);
1896                         }
1897                         expected_sn = __scst_inc_expected_sn(tgt_dev);
1898                         goto restart;
1899                 }
1900         }
1901
1902 out_unlock:
1903         spin_unlock_bh(&tgt_dev->sn_lock);
1904
1905 out:
1906         return cmd;
1907 }
1908
1909 /* No locks */
1910 int scst_inc_on_dev_cmd(struct scst_cmd *cmd)
1911 {
1912         int res = 0;
1913         struct scst_device *dev = cmd->dev;
1914
1915         BUG_ON(cmd->blocking);
1916
1917         atomic_inc(&dev->on_dev_count);
1918
1919 #ifdef STRICT_SERIALIZING
1920         spin_lock_bh(&dev->dev_lock);
1921         if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
1922                 goto out_unlock;
1923         if (dev->block_count > 0) {
1924                 scst_dec_on_dev_cmd(cmd);
1925                 TRACE_MGMT_DBG("Delaying cmd %p due to blocking or serializing"
1926                       "(tag %d, dev %p)", cmd, cmd->tag, dev);
1927                 list_add_tail(&cmd->blocked_cmd_list_entry,
1928                               &dev->blocked_cmd_list);
1929                 res = 1;
1930         } else {
1931                 __scst_block_dev(cmd->dev);
1932                 cmd->blocking = 1;
1933         }
1934         spin_unlock_bh(&dev->dev_lock);
1935         goto out;
1936 #else
1937 repeat:
1938         if (unlikely(dev->block_count > 0)) {
1939                 spin_lock_bh(&dev->dev_lock);
1940                 if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
1941                         goto out_unlock;
1942                 barrier(); /* to reread block_count */
1943                 if (dev->block_count > 0) {
1944                         scst_dec_on_dev_cmd(cmd);
1945                         TRACE_MGMT_DBG("Delaying cmd %p due to blocking or "
1946                                 "serializing (tag %d, dev %p)", cmd,
1947                                 cmd->tag, dev);
1948                         list_add_tail(&cmd->blocked_cmd_list_entry,
1949                                       &dev->blocked_cmd_list);
1950                         res = 1;
1951                         spin_unlock_bh(&dev->dev_lock);
1952                         goto out;
1953                 } else {
1954                         TRACE_MGMT_DBG("%s", "Somebody unblocked the device, "
1955                                 "continuing");
1956                 }
1957                 spin_unlock_bh(&dev->dev_lock);
1958         }
1959         if (unlikely(cmd->dev->dev_serialized)) {
1960                 spin_lock_bh(&dev->dev_lock);
1961                 barrier(); /* to reread block_count */
1962                 if (cmd->dev->block_count == 0) {
1963                         TRACE_MGMT_DBG("cmd %p (tag %d), blocking further "
1964                                 "cmds due to serializing (dev %p)", cmd,
1965                                 cmd->tag, dev);
1966                         __scst_block_dev(cmd->dev);
1967                         cmd->blocking = 1;
1968                 } else {
1969                         spin_unlock_bh(&dev->dev_lock);
1970                         goto repeat;
1971                 }
1972                 spin_unlock_bh(&dev->dev_lock);
1973         }
1974 #endif
1975
1976 out:
1977         return res;
1978
1979 out_unlock:
1980         spin_unlock_bh(&dev->dev_lock);
1981         goto out;
1982 }
1983
1984 /* Called under dev_lock */
1985 void scst_unblock_cmds(struct scst_device *dev)
1986 {
1987 #ifdef STRICT_SERIALIZING
1988         struct scst_cmd *cmd, *t;
1989         int found = 0;
1990
1991         TRACE_ENTRY();
1992
1993         list_for_each_entry_safe(cmd, t, &dev->blocked_cmd_list,
1994                                  blocked_cmd_list_entry) {
1995                 unsigned long flags;
1996                 int brk = 0;
1997                 /*
1998                  * Since only one cmd per time is being executed, expected_sn
1999                  * can't change behind us, if the corresponding cmd is in
2000                  * blocked_cmd_list, but we could be called before
2001                  * __scst_inc_expected_sn().
2002                  */
2003                 if (likely(!cmd->internal) && likely(!cmd->retry)) {
2004                         int expected_sn;
2005                         if (cmd->tgt_dev == NULL)
2006                                 BUG();
2007                         expected_sn = cmd->tgt_dev->expected_sn;
2008                         if (cmd->sn == expected_sn)
2009                                 brk = 1;
2010                         else if (cmd->sn != (expected_sn+1))
2011                                 continue;
2012                 }
2013
2014                 list_del(&cmd->blocked_cmd_list_entry);
2015                 TRACE_MGMT_DBG("Moving cmd %p to active cmd list", cmd);
2016                 spin_lock_irqsave(&scst_list_lock, flags);
2017                 list_move(&cmd->cmd_list_entry, &scst_active_cmd_list);
2018                 spin_unlock_irqrestore(&scst_list_lock, flags);
2019                 found = 1;
2020                 if (brk)
2021                         break;
2022         }
2023         if (found)
2024                 wake_up(&scst_list_waitQ);
2025 #else /* STRICT_SERIALIZING */
2026         struct scst_cmd *cmd, *tcmd;
2027         unsigned long flags;
2028
2029         TRACE_ENTRY();
2030
2031         spin_lock_irqsave(&scst_list_lock, flags);
2032         list_for_each_entry_safe(cmd, tcmd, &dev->blocked_cmd_list,
2033                                  blocked_cmd_list_entry) {
2034                 list_del(&cmd->blocked_cmd_list_entry);
2035                 TRACE_MGMT_DBG("Moving blocked cmd %p to active cmd list", cmd);
2036                 list_move_tail(&cmd->cmd_list_entry, &scst_active_cmd_list);
2037                 wake_up(&scst_list_waitQ);
2038         }
2039         spin_unlock_irqrestore(&scst_list_lock, flags);
2040 #endif /* STRICT_SERIALIZING */
2041
2042         TRACE_EXIT();
2043         return;
2044 }
2045
2046 static struct scst_cmd *scst_inc_expected_sn(
2047         struct scst_tgt_dev *tgt_dev, struct scst_cmd *out_of_sn_cmd)
2048 {
2049         struct scst_cmd *res = NULL;
2050
2051         if (out_of_sn_cmd->sn == tgt_dev->expected_sn) {
2052                 __scst_inc_expected_sn(tgt_dev);
2053         } else {
2054                 spin_lock_bh(&tgt_dev->sn_lock);
2055                 tgt_dev->def_cmd_count++;
2056                 set_bit(SCST_CMD_OUT_OF_SN, &out_of_sn_cmd->cmd_flags);
2057                 list_add_tail(&out_of_sn_cmd->sn_cmd_list_entry,
2058                               &tgt_dev->skipped_sn_list);
2059                 TRACE(TRACE_SCSI_SERIALIZING, "out_of_sn_cmd %p with sn %d "
2060                         "added to skipped_sn_list (expected_sn %d)",
2061                         out_of_sn_cmd, out_of_sn_cmd->sn, tgt_dev->expected_sn);
2062                 spin_unlock_bh(&tgt_dev->sn_lock);
2063                 smp_mb(); /* just in case, we need new value of tgt_dev->expected_sn */
2064         }
2065         res = scst_check_deferred_commands(tgt_dev, tgt_dev->expected_sn);
2066         return res;
2067 }
2068
2069 void scst_inc_expected_sn_unblock(struct scst_tgt_dev *tgt_dev,
2070         struct scst_cmd *cmd_sn, int locked)
2071 {
2072         struct scst_cmd *cmd;
2073
2074         TRACE_ENTRY();
2075
2076         cmd = scst_inc_expected_sn(tgt_dev, cmd_sn);
2077         if (cmd != NULL) {
2078                 unsigned long flags = 0;
2079                 if (!locked)
2080                         spin_lock_irqsave(&scst_list_lock, flags);
2081                 TRACE(TRACE_SCSI_SERIALIZING, "cmd %p with sn %d "
2082                         "moved to active cmd list", cmd, cmd->sn);
2083                 list_move(&cmd->cmd_list_entry, &scst_active_cmd_list);
2084                 if (!locked)
2085                         spin_unlock_irqrestore(&scst_list_lock, flags);
2086                 if (!cmd_sn->processible_env)
2087                         wake_up(&scst_list_waitQ);
2088         }
2089
2090         TRACE_EXIT();
2091         return;
2092 }
2093
2094 #ifdef DEBUG
2095 /* Original taken from the XFS code */
2096 unsigned long scst_random(void)
2097 {
2098         static int Inited;
2099         static unsigned long RandomValue;
2100         static spinlock_t lock = SPIN_LOCK_UNLOCKED;
2101         /* cycles pseudo-randomly through all values between 1 and 2^31 - 2 */
2102         register long rv;
2103         register long lo;
2104         register long hi;
2105         unsigned long flags;
2106
2107         spin_lock_irqsave(&lock, flags);
2108         if (!Inited) {
2109                 RandomValue = jiffies;
2110                 Inited = 1;
2111         }
2112         rv = RandomValue;
2113         hi = rv / 127773;
2114         lo = rv % 127773;
2115         rv = 16807 * lo - 2836 * hi;
2116         if (rv <= 0) rv += 2147483647;
2117         RandomValue = rv;
2118         spin_unlock_irqrestore(&lock, flags);
2119         return rv;
2120 }
2121 #endif
2122
2123 #ifdef DEBUG_TM
2124
2125 #define TM_DBG_STATE_ABORT              0
2126 #define TM_DBG_STATE_RESET              1
2127 #define TM_DBG_STATE_OFFLINE            2
2128
2129 #define INIT_TM_DBG_STATE               TM_DBG_STATE_ABORT
2130
2131 static void tm_dbg_timer_fn(unsigned long arg);
2132
2133 /* All serialized by scst_list_lock */
2134 static int tm_dbg_release;
2135 static int tm_dbg_blocked;
2136 static LIST_HEAD(tm_dbg_delayed_cmd_list);
2137 static int tm_dbg_delayed_cmds_count;
2138 static int tm_dbg_passed_cmds_count;
2139 static int tm_dbg_state;
2140 static int tm_dbg_on_state_passes;
2141 static DEFINE_TIMER(tm_dbg_timer, tm_dbg_timer_fn, 0, 0);
2142
2143 static const int tm_dbg_on_state_num_passes[] = { 10, 1, 0x7ffffff };
2144
2145 void tm_dbg_init_tgt_dev(struct scst_tgt_dev *tgt_dev,
2146         struct scst_acg_dev *acg_dev)
2147 {
2148         if ((acg_dev->acg == scst_default_acg) && (acg_dev->lun == 0)) {
2149                 /* Do TM debugging only for LUN 0 */
2150                 tm_dbg_state = INIT_TM_DBG_STATE;
2151                 tm_dbg_on_state_passes =
2152                         tm_dbg_on_state_num_passes[tm_dbg_state];
2153                 __set_bit(SCST_TGT_DEV_UNDER_TM_DBG, &tgt_dev->tgt_dev_flags);
2154                 PRINT_INFO("LUN 0 connected from initiator %s is under "
2155                         "TM debugging", tgt_dev->sess->tgt->tgtt->name);
2156         }
2157 }
2158
2159 void tm_dbg_deinit_tgt_dev(struct scst_tgt_dev *tgt_dev)
2160 {
2161         if (test_bit(SCST_TGT_DEV_UNDER_TM_DBG, &tgt_dev->tgt_dev_flags))
2162                 del_timer_sync(&tm_dbg_timer);
2163 }
2164
2165 static void tm_dbg_timer_fn(unsigned long arg)
2166 {
2167         TRACE_MGMT_DBG("%s: delayed cmd timer expired", __func__);
2168         tm_dbg_release = 1;
2169         smp_mb();
2170         wake_up_all(&scst_list_waitQ);
2171 }
2172
2173 /* Called under scst_list_lock */
2174 static void tm_dbg_delay_cmd(struct scst_cmd *cmd)
2175 {
2176         switch(tm_dbg_state) {
2177         case TM_DBG_STATE_ABORT:
2178                 if (tm_dbg_delayed_cmds_count == 0) {
2179                         unsigned long d = 58*HZ + (scst_random() % (4*HZ));
2180                         TRACE_MGMT_DBG("%s: delaying timed cmd %p (tag %d) "
2181                                 "for %ld.%ld seconds (%ld HZ)", __func__, cmd, cmd->tag,
2182                                 d/HZ, (d%HZ)*100/HZ, d);
2183                         mod_timer(&tm_dbg_timer, jiffies + d);
2184 #if 0
2185                         tm_dbg_blocked = 1;
2186 #endif
2187                 } else {
2188                         TRACE_MGMT_DBG("%s: delaying another timed cmd %p "
2189                                 "(tag %d), delayed_cmds_count=%d", __func__, cmd,
2190                                 cmd->tag, tm_dbg_delayed_cmds_count);
2191                         if (tm_dbg_delayed_cmds_count == 2)
2192                                 tm_dbg_blocked = 0;
2193                 }
2194                 break;
2195
2196         case TM_DBG_STATE_RESET:
2197         case TM_DBG_STATE_OFFLINE:
2198                 TRACE_MGMT_DBG("%s: delaying cmd %p "
2199                         "(tag %d), delayed_cmds_count=%d", __func__, cmd,
2200                         cmd->tag, tm_dbg_delayed_cmds_count);
2201                 tm_dbg_blocked = 1;
2202                 break;
2203
2204         default:
2205                 BUG();
2206         }
2207         list_move_tail(&cmd->cmd_list_entry, &tm_dbg_delayed_cmd_list);
2208         cmd->tm_dbg_delayed = 1;
2209         tm_dbg_delayed_cmds_count++;
2210         return;
2211 }
2212
2213 /* Called under scst_list_lock */
2214 void tm_dbg_check_released_cmds(void)
2215 {
2216         if (tm_dbg_release) {
2217                 struct scst_cmd *cmd, *tc;
2218                 list_for_each_entry_safe_reverse(cmd, tc,
2219                                 &tm_dbg_delayed_cmd_list, cmd_list_entry) {
2220                         TRACE_MGMT_DBG("%s: Releasing timed cmd %p "
2221                                 "(tag %d), delayed_cmds_count=%d", __func__,
2222                                 cmd, cmd->tag, tm_dbg_delayed_cmds_count);
2223                         list_move(&cmd->cmd_list_entry, &scst_active_cmd_list);
2224                 }
2225                 tm_dbg_release = 0;
2226         }
2227 }
2228
2229 static void tm_dbg_change_state(void)
2230 {
2231         tm_dbg_blocked = 0;
2232         if (--tm_dbg_on_state_passes == 0) {
2233                 switch(tm_dbg_state) {
2234                 case TM_DBG_STATE_ABORT:
2235                         TRACE_MGMT_DBG("%s", "Changing "
2236                             "tm_dbg_state to RESET");
2237                         tm_dbg_state =
2238                                 TM_DBG_STATE_RESET;
2239                         tm_dbg_blocked = 0;
2240                         break;
2241                 case TM_DBG_STATE_RESET:
2242                 case TM_DBG_STATE_OFFLINE:
2243                         if (TM_DBG_GO_OFFLINE) {
2244                             TRACE_MGMT_DBG("%s", "Changing "
2245                                     "tm_dbg_state to OFFLINE");
2246                             tm_dbg_state =
2247                                 TM_DBG_STATE_OFFLINE;
2248                         } else {
2249                             TRACE_MGMT_DBG("%s", "Changing "
2250                                     "tm_dbg_state to ABORT");
2251                             tm_dbg_state =
2252                                 TM_DBG_STATE_ABORT;
2253                         }
2254                         break;
2255                 default:
2256                         BUG();
2257                 }
2258                 tm_dbg_on_state_passes =
2259                     tm_dbg_on_state_num_passes[tm_dbg_state];
2260         }
2261
2262         TRACE_MGMT_DBG("%s", "Deleting timer");
2263         del_timer(&tm_dbg_timer);
2264 }
2265
2266 /* Called under scst_list_lock */
2267 int tm_dbg_check_cmd(struct scst_cmd *cmd)
2268 {
2269         int res = 0;
2270
2271         if (cmd->tm_dbg_immut)
2272                 goto out;
2273
2274         if (cmd->tm_dbg_delayed) {
2275                 TRACE_MGMT_DBG("Processing delayed cmd %p (tag %d), "
2276                         "delayed_cmds_count=%d", cmd, cmd->tag,
2277                         tm_dbg_delayed_cmds_count);
2278
2279                 cmd->tm_dbg_immut = 1;
2280                 tm_dbg_delayed_cmds_count--;
2281                 if ((tm_dbg_delayed_cmds_count == 0) &&
2282                     (tm_dbg_state == TM_DBG_STATE_ABORT))
2283                         tm_dbg_change_state();
2284
2285         } else if (cmd->tgt_dev && test_bit(SCST_TGT_DEV_UNDER_TM_DBG,
2286                                         &cmd->tgt_dev->tgt_dev_flags)) {
2287                 /* Delay 50th command */
2288                 if (tm_dbg_blocked || (++tm_dbg_passed_cmds_count % 50) == 0) {
2289                         tm_dbg_delay_cmd(cmd);
2290                         res = 1;
2291                 } else
2292                         cmd->tm_dbg_immut = 1;
2293         }
2294
2295 out:
2296         return res;
2297 }
2298
2299 /* Called under scst_list_lock */
2300 void tm_dbg_release_cmd(struct scst_cmd *cmd)
2301 {
2302         struct scst_cmd *c;
2303         list_for_each_entry(c, &tm_dbg_delayed_cmd_list,
2304                                 cmd_list_entry) {
2305                 if (c == cmd) {
2306                         TRACE_MGMT_DBG("Abort request for "
2307                                 "delayed cmd %p (tag=%d), moving it to "
2308                                 "active cmd list (delayed_cmds_count=%d)",
2309                                 c, c->tag, tm_dbg_delayed_cmds_count);
2310                         list_move(&c->cmd_list_entry, &scst_active_cmd_list);
2311                         wake_up_all(&scst_list_waitQ);
2312                         break;
2313                 }
2314         }
2315 }
2316
2317 /* Called under scst_list_lock */
2318 void tm_dbg_task_mgmt(const char *fn)
2319 {
2320         if (tm_dbg_state != TM_DBG_STATE_OFFLINE) {
2321                 TRACE_MGMT_DBG("%s: freeing %d delayed cmds", fn,
2322                         tm_dbg_delayed_cmds_count);
2323                 tm_dbg_change_state();
2324                 tm_dbg_release = 1;
2325                 smp_mb();
2326                 wake_up_all(&scst_list_waitQ);
2327         } else {
2328                 TRACE_MGMT_DBG("%s: while OFFLINE state, doing nothing", fn);
2329         }
2330 }
2331
2332 int tm_dbg_is_release(void)
2333 {
2334         return tm_dbg_release;
2335 }
2336 #endif /* DEBUG_TM */