drivers/scsi/be2iscsi/be_main.c

   1 /**
   2  * Copyright (C) 2005 - 2010 ServerEngines
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Written by: Jayamohan Kallickal (jayamohank@serverengines.com)
  11  *
  12  * Contact Information:
  13  * linux-drivers@serverengines.com
  14  *
  15  *  ServerEngines
  16  * 209 N. Fair Oaks Ave
  17  * Sunnyvale, CA 94085
  18  *
  19  */
  20 #include <linux/reboot.h>
  21 #include <linux/delay.h>
  22 #include <linux/slab.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/blkdev.h>
  25 #include <linux/pci.h>
  26 #include <linux/string.h>
  27 #include <linux/kernel.h>
  28 #include <linux/semaphore.h>
  29
  30 #include <scsi/libiscsi.h>
  31 #include <scsi/scsi_transport_iscsi.h>
  32 #include <scsi/scsi_transport.h>
  33 #include <scsi/scsi_cmnd.h>
  34 #include <scsi/scsi_device.h>
  35 #include <scsi/scsi_host.h>
  36 #include <scsi/scsi.h>
  37 #include "be_main.h"
  38 #include "be_iscsi.h"
  39 #include "be_mgmt.h"
  40
  41 static unsigned int be_iopoll_budget = 10;
  42 static unsigned int be_max_phys_size = 64;
  43 static unsigned int enable_msix = 1;
  44
  45 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
  46 MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
  47 MODULE_AUTHOR("ServerEngines Corporation");
  48 MODULE_LICENSE("GPL");
  49 module_param(be_iopoll_budget, int, 0);
  50 module_param(enable_msix, int, 0);
  51 module_param(be_max_phys_size, uint, S_IRUGO);
  52 MODULE_PARM_DESC(be_max_phys_size, "Maximum Size (In Kilobytes) of physically"
  53                                    "contiguous memory that can be allocated."
  54                                    "Range is 16 - 128");
  55
  56 static int beiscsi_slave_configure(struct scsi_device *sdev)
  57 {
  58         blk_queue_max_segment_size(sdev->request_queue, 65536);
  59         return 0;
  60 }
  61
  62 static int beiscsi_eh_abort(struct scsi_cmnd *sc)
  63 {
  64         struct iscsi_cls_session *cls_session;
  65         struct iscsi_task *aborted_task = (struct iscsi_task *)sc->SCp.ptr;
  66         struct beiscsi_io_task *aborted_io_task;
  67         struct iscsi_conn *conn;
  68         struct beiscsi_conn *beiscsi_conn;
  69         struct beiscsi_hba *phba;
  70         struct iscsi_session *session;
  71         struct invalidate_command_table *inv_tbl;
  72         unsigned int cid, tag, num_invalidate;
  73
  74         cls_session = starget_to_session(scsi_target(sc->device));
  75         session = cls_session->dd_data;
  76
  77         spin_lock_bh(&session->lock);
  78         if (!aborted_task || !aborted_task->sc) {
  79                 /* we raced */
  80                 spin_unlock_bh(&session->lock);
  81                 return SUCCESS;
  82         }
  83
  84         aborted_io_task = aborted_task->dd_data;
  85         if (!aborted_io_task->scsi_cmnd) {
  86                 /* raced or invalid command */
  87                 spin_unlock_bh(&session->lock);
  88                 return SUCCESS;
  89         }
  90         spin_unlock_bh(&session->lock);
  91         conn = aborted_task->conn;
  92         beiscsi_conn = conn->dd_data;
  93         phba = beiscsi_conn->phba;
  94
  95         /* invalidate iocb */
  96         cid = beiscsi_conn->beiscsi_conn_cid;
  97         inv_tbl = phba->inv_tbl;
  98         memset(inv_tbl, 0x0, sizeof(*inv_tbl));
  99         inv_tbl->cid = cid;
 100         inv_tbl->icd = aborted_io_task->psgl_handle->sgl_index;
 101         num_invalidate = 1;
 102         tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate, cid);
 103         if (!tag) {
 104                 shost_printk(KERN_WARNING, phba->shost,
 105                              "mgmt_invalidate_icds could not be"
 106                              " submitted\n");
 107                 return FAILED;
 108         } else {
 109                 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
 110                                          phba->ctrl.mcc_numtag[tag]);
 111                 free_mcc_tag(&phba->ctrl, tag);
 112         }
 113
 114         return iscsi_eh_abort(sc);
 115 }
 116
 117 static int beiscsi_eh_device_reset(struct scsi_cmnd *sc)
 118 {
 119         struct iscsi_task *abrt_task;
 120         struct beiscsi_io_task *abrt_io_task;
 121         struct iscsi_conn *conn;
 122         struct beiscsi_conn *beiscsi_conn;
 123         struct beiscsi_hba *phba;
 124         struct iscsi_session *session;
 125         struct iscsi_cls_session *cls_session;
 126         struct invalidate_command_table *inv_tbl;
 127         unsigned int cid, tag, i, num_invalidate;
 128         int rc = FAILED;
 129
 130         /* invalidate iocbs */
 131         cls_session = starget_to_session(scsi_target(sc->device));
 132         session = cls_session->dd_data;
 133         spin_lock_bh(&session->lock);
 134         if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN)
 135                 goto unlock;
 136
 137         conn = session->leadconn;
 138         beiscsi_conn = conn->dd_data;
 139         phba = beiscsi_conn->phba;
 140         cid = beiscsi_conn->beiscsi_conn_cid;
 141         inv_tbl = phba->inv_tbl;
 142         memset(inv_tbl, 0x0, sizeof(*inv_tbl) * BE2_CMDS_PER_CXN);
 143         num_invalidate = 0;
 144         for (i = 0; i < conn->session->cmds_max; i++) {
 145                 abrt_task = conn->session->cmds[i];
 146                 abrt_io_task = abrt_task->dd_data;
 147                 if (!abrt_task->sc || abrt_task->state == ISCSI_TASK_FREE)
 148                         continue;
 149
 150                 if (abrt_task->sc->device->lun != abrt_task->sc->device->lun)
 151                         continue;
 152
 153                 inv_tbl->cid = cid;
 154                 inv_tbl->icd = abrt_io_task->psgl_handle->sgl_index;
 155                 num_invalidate++;
 156                 inv_tbl++;
 157         }
 158         spin_unlock_bh(&session->lock);
 159         inv_tbl = phba->inv_tbl;
 160
 161         tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate, cid);
 162         if (!tag) {
 163                 shost_printk(KERN_WARNING, phba->shost,
 164                              "mgmt_invalidate_icds could not be"
 165                              " submitted\n");
 166                 return FAILED;
 167         } else {
 168                 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
 169                                          phba->ctrl.mcc_numtag[tag]);
 170                 free_mcc_tag(&phba->ctrl, tag);
 171         }
 172
 173         return iscsi_eh_device_reset(sc);
 174 unlock:
 175         spin_unlock_bh(&session->lock);
 176         return rc;
 177 }
 178
 179 /*------------------- PCI Driver operations and data ----------------- */
 180 static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
 181         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
 182         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
 183         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
 184         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
 185         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
 186         { 0 }
 187 };
 188 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
 189
 190 static struct scsi_host_template beiscsi_sht = {
 191         .module = THIS_MODULE,
 192         .name = "ServerEngines 10Gbe open-iscsi Initiator Driver",
 193         .proc_name = DRV_NAME,
 194         .queuecommand = iscsi_queuecommand,
 195         .change_queue_depth = iscsi_change_queue_depth,
 196         .slave_configure = beiscsi_slave_configure,
 197         .target_alloc = iscsi_target_alloc,
 198         .eh_abort_handler = beiscsi_eh_abort,
 199         .eh_device_reset_handler = beiscsi_eh_device_reset,
 200         .eh_target_reset_handler = iscsi_eh_session_reset,
 201         .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
 202         .can_queue = BE2_IO_DEPTH,
 203         .this_id = -1,
 204         .max_sectors = BEISCSI_MAX_SECTORS,
 205         .cmd_per_lun = BEISCSI_CMD_PER_LUN,
 206         .use_clustering = ENABLE_CLUSTERING,
 207 };
 208
 209 static struct scsi_transport_template *beiscsi_scsi_transport;
 210
 211 static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
 212 {
 213         struct beiscsi_hba *phba;
 214         struct Scsi_Host *shost;
 215
 216         shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
 217         if (!shost) {
 218                 dev_err(&pcidev->dev, "beiscsi_hba_alloc -"
 219                         "iscsi_host_alloc failed \n");
 220                 return NULL;
 221         }
 222         shost->dma_boundary = pcidev->dma_mask;
 223         shost->max_id = BE2_MAX_SESSIONS;
 224         shost->max_channel = 0;
 225         shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
 226         shost->max_lun = BEISCSI_NUM_MAX_LUN;
 227         shost->transportt = beiscsi_scsi_transport;
 228         phba = iscsi_host_priv(shost);
 229         memset(phba, 0, sizeof(*phba));
 230         phba->shost = shost;
 231         phba->pcidev = pci_dev_get(pcidev);
 232         pci_set_drvdata(pcidev, phba);
 233
 234         if (iscsi_host_add(shost, &phba->pcidev->dev))
 235                 goto free_devices;
 236         return phba;
 237
 238 free_devices:
 239         pci_dev_put(phba->pcidev);
 240         iscsi_host_free(phba->shost);
 241         return NULL;
 242 }
 243
 244 static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
 245 {
 246         if (phba->csr_va) {
 247                 iounmap(phba->csr_va);
 248                 phba->csr_va = NULL;
 249         }
 250         if (phba->db_va) {
 251                 iounmap(phba->db_va);
 252                 phba->db_va = NULL;
 253         }
 254         if (phba->pci_va) {
 255                 iounmap(phba->pci_va);
 256                 phba->pci_va = NULL;
 257         }
 258 }
 259
 260 static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
 261                                 struct pci_dev *pcidev)
 262 {
 263         u8 __iomem *addr;
 264         int pcicfg_reg;
 265
 266         addr = ioremap_nocache(pci_resource_start(pcidev, 2),
 267                                pci_resource_len(pcidev, 2));
 268         if (addr == NULL)
 269                 return -ENOMEM;
 270         phba->ctrl.csr = addr;
 271         phba->csr_va = addr;
 272         phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
 273
 274         addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
 275         if (addr == NULL)
 276                 goto pci_map_err;
 277         phba->ctrl.db = addr;
 278         phba->db_va = addr;
 279         phba->db_pa.u.a64.address =  pci_resource_start(pcidev, 4);
 280
 281         if (phba->generation == BE_GEN2)
 282                 pcicfg_reg = 1;
 283         else
 284                 pcicfg_reg = 0;
 285
 286         addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg),
 287                                pci_resource_len(pcidev, pcicfg_reg));
 288
 289         if (addr == NULL)
 290                 goto pci_map_err;
 291         phba->ctrl.pcicfg = addr;
 292         phba->pci_va = addr;
 293         phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg);
 294         return 0;
 295
 296 pci_map_err:
 297         beiscsi_unmap_pci_function(phba);
 298         return -ENOMEM;
 299 }
 300
 301 static int beiscsi_enable_pci(struct pci_dev *pcidev)
 302 {
 303         int ret;
 304
 305         ret = pci_enable_device(pcidev);
 306         if (ret) {
 307                 dev_err(&pcidev->dev, "beiscsi_enable_pci - enable device "
 308                         "failed. Returning -ENODEV\n");
 309                 return ret;
 310         }
 311
 312         pci_set_master(pcidev);
 313         if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
 314                 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
 315                 if (ret) {
 316                         dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
 317                         pci_disable_device(pcidev);
 318                         return ret;
 319                 }
 320         }
 321         return 0;
 322 }
 323
 324 static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
 325 {
 326         struct be_ctrl_info *ctrl = &phba->ctrl;
 327         struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
 328         struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
 329         int status = 0;
 330
 331         ctrl->pdev = pdev;
 332         status = beiscsi_map_pci_bars(phba, pdev);
 333         if (status)
 334                 return status;
 335         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
 336         mbox_mem_alloc->va = pci_alloc_consistent(pdev,
 337                                                   mbox_mem_alloc->size,
 338                                                   &mbox_mem_alloc->dma);
 339         if (!mbox_mem_alloc->va) {
 340                 beiscsi_unmap_pci_function(phba);
 341                 status = -ENOMEM;
 342                 return status;
 343         }
 344
 345         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
 346         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
 347         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
 348         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
 349         spin_lock_init(&ctrl->mbox_lock);
 350         spin_lock_init(&phba->ctrl.mcc_lock);
 351         spin_lock_init(&phba->ctrl.mcc_cq_lock);
 352
 353         return status;
 354 }
 355
 356 static void beiscsi_get_params(struct beiscsi_hba *phba)
 357 {
 358         phba->params.ios_per_ctrl = (phba->fw_config.iscsi_icd_count
 359                                     - (phba->fw_config.iscsi_cid_count
 360                                     + BE2_TMFS
 361                                     + BE2_NOPOUT_REQ));
 362         phba->params.cxns_per_ctrl = phba->fw_config.iscsi_cid_count;
 363         phba->params.asyncpdus_per_ctrl = phba->fw_config.iscsi_cid_count * 2;
 364         phba->params.icds_per_ctrl = phba->fw_config.iscsi_icd_count;;
 365         phba->params.num_sge_per_io = BE2_SGE;
 366         phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
 367         phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
 368         phba->params.eq_timer = 64;
 369         phba->params.num_eq_entries =
 370             (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2
 371                                     + BE2_TMFS) / 512) + 1) * 512;
 372         phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024)
 373                                 ? 1024 : phba->params.num_eq_entries;
 374         SE_DEBUG(DBG_LVL_8, "phba->params.num_eq_entries=%d \n",
 375                              phba->params.num_eq_entries);
 376         phba->params.num_cq_entries =
 377             (((BE2_CMDS_PER_CXN * 2 +  phba->fw_config.iscsi_cid_count * 2
 378                                     + BE2_TMFS) / 512) + 1) * 512;
 379         phba->params.wrbs_per_cxn = 256;
 380 }
 381
 382 static void hwi_ring_eq_db(struct beiscsi_hba *phba,
 383                            unsigned int id, unsigned int clr_interrupt,
 384                            unsigned int num_processed,
 385                            unsigned char rearm, unsigned char event)
 386 {
 387         u32 val = 0;
 388         val |= id & DB_EQ_RING_ID_MASK;
 389         if (rearm)
 390                 val |= 1 << DB_EQ_REARM_SHIFT;
 391         if (clr_interrupt)
 392                 val |= 1 << DB_EQ_CLR_SHIFT;
 393         if (event)
 394                 val |= 1 << DB_EQ_EVNT_SHIFT;
 395         val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
 396         iowrite32(val, phba->db_va + DB_EQ_OFFSET);
 397 }
 398
 399 /**
 400  * be_isr_mcc - The isr routine of the driver.
 401  * @irq: Not used
 402  * @dev_id: Pointer to host adapter structure
 403  */
 404 static irqreturn_t be_isr_mcc(int irq, void *dev_id)
 405 {
 406         struct beiscsi_hba *phba;
 407         struct be_eq_entry *eqe = NULL;
 408         struct be_queue_info *eq;
 409         struct be_queue_info *mcc;
 410         unsigned int num_eq_processed;
 411         struct be_eq_obj *pbe_eq;
 412         unsigned long flags;
 413
 414         pbe_eq = dev_id;
 415         eq = &pbe_eq->q;
 416         phba =  pbe_eq->phba;
 417         mcc = &phba->ctrl.mcc_obj.cq;
 418         eqe = queue_tail_node(eq);
 419         if (!eqe)
 420                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
 421
 422         num_eq_processed = 0;
 423
 424         while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 425                                 & EQE_VALID_MASK) {
 426                 if (((eqe->dw[offsetof(struct amap_eq_entry,
 427                      resource_id) / 32] &
 428                      EQE_RESID_MASK) >> 16) == mcc->id) {
 429                         spin_lock_irqsave(&phba->isr_lock, flags);
 430                         phba->todo_mcc_cq = 1;
 431                         spin_unlock_irqrestore(&phba->isr_lock, flags);
 432                 }
 433                 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 434                 queue_tail_inc(eq);
 435                 eqe = queue_tail_node(eq);
 436                 num_eq_processed++;
 437         }
 438         if (phba->todo_mcc_cq)
 439                 queue_work(phba->wq, &phba->work_cqs);
 440         if (num_eq_processed)
 441                 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
 442
 443         return IRQ_HANDLED;
 444 }
 445
 446 /**
 447  * be_isr_msix - The isr routine of the driver.
 448  * @irq: Not used
 449  * @dev_id: Pointer to host adapter structure
 450  */
 451 static irqreturn_t be_isr_msix(int irq, void *dev_id)
 452 {
 453         struct beiscsi_hba *phba;
 454         struct be_eq_entry *eqe = NULL;
 455         struct be_queue_info *eq;
 456         struct be_queue_info *cq;
 457         unsigned int num_eq_processed;
 458         struct be_eq_obj *pbe_eq;
 459         unsigned long flags;
 460
 461         pbe_eq = dev_id;
 462         eq = &pbe_eq->q;
 463         cq = pbe_eq->cq;
 464         eqe = queue_tail_node(eq);
 465         if (!eqe)
 466                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
 467
 468         phba = pbe_eq->phba;
 469         num_eq_processed = 0;
 470         if (blk_iopoll_enabled) {
 471                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 472                                         & EQE_VALID_MASK) {
 473                         if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
 474                                 blk_iopoll_sched(&pbe_eq->iopoll);
 475
 476                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 477                         queue_tail_inc(eq);
 478                         eqe = queue_tail_node(eq);
 479                         num_eq_processed++;
 480                 }
 481                 if (num_eq_processed)
 482                         hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1);
 483
 484                 return IRQ_HANDLED;
 485         } else {
 486                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 487                                                 & EQE_VALID_MASK) {
 488                         spin_lock_irqsave(&phba->isr_lock, flags);
 489                         phba->todo_cq = 1;
 490                         spin_unlock_irqrestore(&phba->isr_lock, flags);
 491                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 492                         queue_tail_inc(eq);
 493                         eqe = queue_tail_node(eq);
 494                         num_eq_processed++;
 495                 }
 496                 if (phba->todo_cq)
 497                         queue_work(phba->wq, &phba->work_cqs);
 498
 499                 if (num_eq_processed)
 500                         hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
 501
 502                 return IRQ_HANDLED;
 503         }
 504 }
 505
 506 /**
 507  * be_isr - The isr routine of the driver.
 508  * @irq: Not used
 509  * @dev_id: Pointer to host adapter structure
 510  */
 511 static irqreturn_t be_isr(int irq, void *dev_id)
 512 {
 513         struct beiscsi_hba *phba;
 514         struct hwi_controller *phwi_ctrlr;
 515         struct hwi_context_memory *phwi_context;
 516         struct be_eq_entry *eqe = NULL;
 517         struct be_queue_info *eq;
 518         struct be_queue_info *cq;
 519         struct be_queue_info *mcc;
 520         unsigned long flags, index;
 521         unsigned int num_mcceq_processed, num_ioeq_processed;
 522         struct be_ctrl_info *ctrl;
 523         struct be_eq_obj *pbe_eq;
 524         int isr;
 525
 526         phba = dev_id;
 527         ctrl = &phba->ctrl;;
 528         isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
 529                        (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
 530         if (!isr)
 531                 return IRQ_NONE;
 532
 533         phwi_ctrlr = phba->phwi_ctrlr;
 534         phwi_context = phwi_ctrlr->phwi_ctxt;
 535         pbe_eq = &phwi_context->be_eq[0];
 536
 537         eq = &phwi_context->be_eq[0].q;
 538         mcc = &phba->ctrl.mcc_obj.cq;
 539         index = 0;
 540         eqe = queue_tail_node(eq);
 541         if (!eqe)
 542                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
 543
 544         num_ioeq_processed = 0;
 545         num_mcceq_processed = 0;
 546         if (blk_iopoll_enabled) {
 547                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 548                                         & EQE_VALID_MASK) {
 549                         if (((eqe->dw[offsetof(struct amap_eq_entry,
 550                              resource_id) / 32] &
 551                              EQE_RESID_MASK) >> 16) == mcc->id) {
 552                                 spin_lock_irqsave(&phba->isr_lock, flags);
 553                                 phba->todo_mcc_cq = 1;
 554                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 555                                 num_mcceq_processed++;
 556                         } else {
 557                                 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
 558                                         blk_iopoll_sched(&pbe_eq->iopoll);
 559                                 num_ioeq_processed++;
 560                         }
 561                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 562                         queue_tail_inc(eq);
 563                         eqe = queue_tail_node(eq);
 564                 }
 565                 if (num_ioeq_processed || num_mcceq_processed) {
 566                         if (phba->todo_mcc_cq)
 567                                 queue_work(phba->wq, &phba->work_cqs);
 568
 569                         if ((num_mcceq_processed) && (!num_ioeq_processed))
 570                                 hwi_ring_eq_db(phba, eq->id, 0,
 571                                               (num_ioeq_processed +
 572                                                num_mcceq_processed) , 1, 1);
 573                         else
 574                                 hwi_ring_eq_db(phba, eq->id, 0,
 575                                                (num_ioeq_processed +
 576                                                 num_mcceq_processed), 0, 1);
 577
 578                         return IRQ_HANDLED;
 579                 } else
 580                         return IRQ_NONE;
 581         } else {
 582                 cq = &phwi_context->be_cq[0];
 583                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 584                                                 & EQE_VALID_MASK) {
 585
 586                         if (((eqe->dw[offsetof(struct amap_eq_entry,
 587                              resource_id) / 32] &
 588                              EQE_RESID_MASK) >> 16) != cq->id) {
 589                                 spin_lock_irqsave(&phba->isr_lock, flags);
 590                                 phba->todo_mcc_cq = 1;
 591                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 592                         } else {
 593                                 spin_lock_irqsave(&phba->isr_lock, flags);
 594                                 phba->todo_cq = 1;
 595                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 596                         }
 597                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 598                         queue_tail_inc(eq);
 599                         eqe = queue_tail_node(eq);
 600                         num_ioeq_processed++;
 601                 }
 602                 if (phba->todo_cq || phba->todo_mcc_cq)
 603                         queue_work(phba->wq, &phba->work_cqs);
 604
 605                 if (num_ioeq_processed) {
 606                         hwi_ring_eq_db(phba, eq->id, 0,
 607                                        num_ioeq_processed, 1, 1);
 608                         return IRQ_HANDLED;
 609                 } else
 610                         return IRQ_NONE;
 611         }
 612 }
 613
 614 static int beiscsi_init_irqs(struct beiscsi_hba *phba)
 615 {
 616         struct pci_dev *pcidev = phba->pcidev;
 617         struct hwi_controller *phwi_ctrlr;
 618         struct hwi_context_memory *phwi_context;
 619         int ret, msix_vec, i = 0;
 620         char desc[32];
 621
 622         phwi_ctrlr = phba->phwi_ctrlr;
 623         phwi_context = phwi_ctrlr->phwi_ctxt;
 624
 625         if (phba->msix_enabled) {
 626                 for (i = 0; i < phba->num_cpus; i++) {
 627                         sprintf(desc, "beiscsi_msix_%04x", i);
 628                         msix_vec = phba->msix_entries[i].vector;
 629                         ret = request_irq(msix_vec, be_isr_msix, 0, desc,
 630                                           &phwi_context->be_eq[i]);
 631                 }
 632                 msix_vec = phba->msix_entries[i].vector;
 633                 ret = request_irq(msix_vec, be_isr_mcc, 0, "beiscsi_msix_mcc",
 634                                   &phwi_context->be_eq[i]);
 635         } else {
 636                 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
 637                                   "beiscsi", phba);
 638                 if (ret) {
 639                         shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
 640                                      "Failed to register irq\\n");
 641                         return ret;
 642                 }
 643         }
 644         return 0;
 645 }
 646
 647 static void hwi_ring_cq_db(struct beiscsi_hba *phba,
 648                            unsigned int id, unsigned int num_processed,
 649                            unsigned char rearm, unsigned char event)
 650 {
 651         u32 val = 0;
 652         val |= id & DB_CQ_RING_ID_MASK;
 653         if (rearm)
 654                 val |= 1 << DB_CQ_REARM_SHIFT;
 655         val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
 656         iowrite32(val, phba->db_va + DB_CQ_OFFSET);
 657 }
 658
 659 static unsigned int
 660 beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
 661                           struct beiscsi_hba *phba,
 662                           unsigned short cid,
 663                           struct pdu_base *ppdu,
 664                           unsigned long pdu_len,
 665                           void *pbuffer, unsigned long buf_len)
 666 {
 667         struct iscsi_conn *conn = beiscsi_conn->conn;
 668         struct iscsi_session *session = conn->session;
 669         struct iscsi_task *task;
 670         struct beiscsi_io_task *io_task;
 671         struct iscsi_hdr *login_hdr;
 672
 673         switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
 674                                                 PDUBASE_OPCODE_MASK) {
 675         case ISCSI_OP_NOOP_IN:
 676                 pbuffer = NULL;
 677                 buf_len = 0;
 678                 break;
 679         case ISCSI_OP_ASYNC_EVENT:
 680                 break;
 681         case ISCSI_OP_REJECT:
 682                 WARN_ON(!pbuffer);
 683                 WARN_ON(!(buf_len == 48));
 684                 SE_DEBUG(DBG_LVL_1, "In ISCSI_OP_REJECT\n");
 685                 break;
 686         case ISCSI_OP_LOGIN_RSP:
 687         case ISCSI_OP_TEXT_RSP:
 688                 task = conn->login_task;
 689                 io_task = task->dd_data;
 690                 login_hdr = (struct iscsi_hdr *)ppdu;
 691                 login_hdr->itt = io_task->libiscsi_itt;
 692                 break;
 693         default:
 694                 shost_printk(KERN_WARNING, phba->shost,
 695                              "Unrecognized opcode 0x%x in async msg \n",
 696                              (ppdu->
 697                              dw[offsetof(struct amap_pdu_base, opcode) / 32]
 698                                                 & PDUBASE_OPCODE_MASK));
 699                 return 1;
 700         }
 701
 702         spin_lock_bh(&session->lock);
 703         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
 704         spin_unlock_bh(&session->lock);
 705         return 0;
 706 }
 707
 708 static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
 709 {
 710         struct sgl_handle *psgl_handle;
 711
 712         if (phba->io_sgl_hndl_avbl) {
 713                 SE_DEBUG(DBG_LVL_8,
 714                          "In alloc_io_sgl_handle,io_sgl_alloc_index=%d \n",
 715                          phba->io_sgl_alloc_index);
 716                 psgl_handle = phba->io_sgl_hndl_base[phba->
 717                                                 io_sgl_alloc_index];
 718                 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
 719                 phba->io_sgl_hndl_avbl--;
 720                 if (phba->io_sgl_alloc_index == (phba->params.
 721                                                  ios_per_ctrl - 1))
 722                         phba->io_sgl_alloc_index = 0;
 723                 else
 724                         phba->io_sgl_alloc_index++;
 725         } else
 726                 psgl_handle = NULL;
 727         return psgl_handle;
 728 }
 729
 730 static void
 731 free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
 732 {
 733         SE_DEBUG(DBG_LVL_8, "In free_,io_sgl_free_index=%d \n",
 734                  phba->io_sgl_free_index);
 735         if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
 736                 /*
 737                  * this can happen if clean_task is called on a task that
 738                  * failed in xmit_task or alloc_pdu.
 739                  */
 740                  SE_DEBUG(DBG_LVL_8,
 741                          "Double Free in IO SGL io_sgl_free_index=%d,"
 742                          "value there=%p \n", phba->io_sgl_free_index,
 743                          phba->io_sgl_hndl_base[phba->io_sgl_free_index]);
 744                 return;
 745         }
 746         phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
 747         phba->io_sgl_hndl_avbl++;
 748         if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
 749                 phba->io_sgl_free_index = 0;
 750         else
 751                 phba->io_sgl_free_index++;
 752 }
 753
 754 /**
 755  * alloc_wrb_handle - To allocate a wrb handle
 756  * @phba: The hba pointer
 757  * @cid: The cid to use for allocation
 758  *
 759  * This happens under session_lock until submission to chip
 760  */
 761 struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid)
 762 {
 763         struct hwi_wrb_context *pwrb_context;
 764         struct hwi_controller *phwi_ctrlr;
 765         struct wrb_handle *pwrb_handle, *pwrb_handle_tmp;
 766
 767         phwi_ctrlr = phba->phwi_ctrlr;
 768         pwrb_context = &phwi_ctrlr->wrb_context[cid];
 769         if (pwrb_context->wrb_handles_available >= 2) {
 770                 pwrb_handle = pwrb_context->pwrb_handle_base[
 771                                             pwrb_context->alloc_index];
 772                 pwrb_context->wrb_handles_available--;
 773                 if (pwrb_context->alloc_index ==
 774                                                 (phba->params.wrbs_per_cxn - 1))
 775                         pwrb_context->alloc_index = 0;
 776                 else
 777                         pwrb_context->alloc_index++;
 778                 pwrb_handle_tmp = pwrb_context->pwrb_handle_base[
 779                                                 pwrb_context->alloc_index];
 780                 pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index;
 781         } else
 782                 pwrb_handle = NULL;
 783         return pwrb_handle;
 784 }
 785
 786 /**
 787  * free_wrb_handle - To free the wrb handle back to pool
 788  * @phba: The hba pointer
 789  * @pwrb_context: The context to free from
 790  * @pwrb_handle: The wrb_handle to free
 791  *
 792  * This happens under session_lock until submission to chip
 793  */
 794 static void
 795 free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
 796                 struct wrb_handle *pwrb_handle)
 797 {
 798         pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle;
 799         pwrb_context->wrb_handles_available++;
 800         if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))
 801                 pwrb_context->free_index = 0;
 802         else
 803                 pwrb_context->free_index++;
 804
 805         SE_DEBUG(DBG_LVL_8,
 806                  "FREE WRB: pwrb_handle=%p free_index=0x%x"
 807                  "wrb_handles_available=%d \n",
 808                  pwrb_handle, pwrb_context->free_index,
 809                  pwrb_context->wrb_handles_available);
 810 }
 811
 812 static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
 813 {
 814         struct sgl_handle *psgl_handle;
 815
 816         if (phba->eh_sgl_hndl_avbl) {
 817                 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
 818                 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
 819                 SE_DEBUG(DBG_LVL_8, "mgmt_sgl_alloc_index=%d=0x%x \n",
 820                          phba->eh_sgl_alloc_index, phba->eh_sgl_alloc_index);
 821                 phba->eh_sgl_hndl_avbl--;
 822                 if (phba->eh_sgl_alloc_index ==
 823                     (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
 824                      1))
 825                         phba->eh_sgl_alloc_index = 0;
 826                 else
 827                         phba->eh_sgl_alloc_index++;
 828         } else
 829                 psgl_handle = NULL;
 830         return psgl_handle;
 831 }
 832
 833 void
 834 free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
 835 {
 836
 837         SE_DEBUG(DBG_LVL_8, "In  free_mgmt_sgl_handle,eh_sgl_free_index=%d \n",
 838                              phba->eh_sgl_free_index);
 839         if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
 840                 /*
 841                  * this can happen if clean_task is called on a task that
 842                  * failed in xmit_task or alloc_pdu.
 843                  */
 844                 SE_DEBUG(DBG_LVL_8,
 845                          "Double Free in eh SGL ,eh_sgl_free_index=%d \n",
 846                          phba->eh_sgl_free_index);
 847                 return;
 848         }
 849         phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
 850         phba->eh_sgl_hndl_avbl++;
 851         if (phba->eh_sgl_free_index ==
 852             (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
 853                 phba->eh_sgl_free_index = 0;
 854         else
 855                 phba->eh_sgl_free_index++;
 856 }
 857
 858 static void
 859 be_complete_io(struct beiscsi_conn *beiscsi_conn,
 860                struct iscsi_task *task, struct sol_cqe *psol)
 861 {
 862         struct beiscsi_io_task *io_task = task->dd_data;
 863         struct be_status_bhs *sts_bhs =
 864                                 (struct be_status_bhs *)io_task->cmd_bhs;
 865         struct iscsi_conn *conn = beiscsi_conn->conn;
 866         unsigned int sense_len;
 867         unsigned char *sense;
 868         u32 resid = 0, exp_cmdsn, max_cmdsn;
 869         u8 rsp, status, flags;
 870
 871         exp_cmdsn = (psol->
 872                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
 873                         & SOL_EXP_CMD_SN_MASK);
 874         max_cmdsn = ((psol->
 875                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
 876                         & SOL_EXP_CMD_SN_MASK) +
 877                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
 878                                 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
 879         rsp = ((psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 32]
 880                                                 & SOL_RESP_MASK) >> 16);
 881         status = ((psol->dw[offsetof(struct amap_sol_cqe, i_sts) / 32]
 882                                                 & SOL_STS_MASK) >> 8);
 883         flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
 884                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
 885
 886         task->sc->result = (DID_OK << 16) | status;
 887         if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
 888                 task->sc->result = DID_ERROR << 16;
 889                 goto unmap;
 890         }
 891
 892         /* bidi not initially supported */
 893         if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
 894                 resid = (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) /
 895                                 32] & SOL_RES_CNT_MASK);
 896
 897                 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
 898                         task->sc->result = DID_ERROR << 16;
 899
 900                 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
 901                         scsi_set_resid(task->sc, resid);
 902                         if (!status && (scsi_bufflen(task->sc) - resid <
 903                             task->sc->underflow))
 904                                 task->sc->result = DID_ERROR << 16;
 905                 }
 906         }
 907
 908         if (status == SAM_STAT_CHECK_CONDITION) {
 909                 unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
 910                 sense = sts_bhs->sense_info + sizeof(unsigned short);
 911                 sense_len =  cpu_to_be16(*slen);
 912                 memcpy(task->sc->sense_buffer, sense,
 913                        min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
 914         }
 915
 916         if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ) {
 917                 if (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
 918                                                         & SOL_RES_CNT_MASK)
 919                          conn->rxdata_octets += (psol->
 920                              dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
 921                              & SOL_RES_CNT_MASK);
 922         }
 923 unmap:
 924         scsi_dma_unmap(io_task->scsi_cmnd);
 925         iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
 926 }
 927
 928 static void
 929 be_complete_logout(struct beiscsi_conn *beiscsi_conn,
 930                    struct iscsi_task *task, struct sol_cqe *psol)
 931 {
 932         struct iscsi_logout_rsp *hdr;
 933         struct beiscsi_io_task *io_task = task->dd_data;
 934         struct iscsi_conn *conn = beiscsi_conn->conn;
 935
 936         hdr = (struct iscsi_logout_rsp *)task->hdr;
 937         hdr->opcode = ISCSI_OP_LOGOUT_RSP;
 938         hdr->t2wait = 5;
 939         hdr->t2retain = 0;
 940         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
 941                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
 942         hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
 943                                         32] & SOL_RESP_MASK);
 944         hdr->exp_cmdsn = cpu_to_be32(psol->
 945                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
 946                                         & SOL_EXP_CMD_SN_MASK);
 947         hdr->max_cmdsn = be32_to_cpu((psol->
 948                          dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
 949                                         & SOL_EXP_CMD_SN_MASK) +
 950                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
 951                                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
 952         hdr->dlength[0] = 0;
 953         hdr->dlength[1] = 0;
 954         hdr->dlength[2] = 0;
 955         hdr->hlength = 0;
 956         hdr->itt = io_task->libiscsi_itt;
 957         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
 958 }
 959
 960 static void
 961 be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
 962                 struct iscsi_task *task, struct sol_cqe *psol)
 963 {
 964         struct iscsi_tm_rsp *hdr;
 965         struct iscsi_conn *conn = beiscsi_conn->conn;
 966         struct beiscsi_io_task *io_task = task->dd_data;
 967
 968         hdr = (struct iscsi_tm_rsp *)task->hdr;
 969         hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
 970         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
 971                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
 972         hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
 973                                         32] & SOL_RESP_MASK);
 974         hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
 975                                     i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
 976         hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
 977                         i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
 978                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
 979                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
 980         hdr->itt = io_task->libiscsi_itt;
 981         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
 982 }
 983
 984 static void
 985 hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
 986                        struct beiscsi_hba *phba, struct sol_cqe *psol)
 987 {
 988         struct hwi_wrb_context *pwrb_context;
 989         struct wrb_handle *pwrb_handle = NULL;
 990         struct hwi_controller *phwi_ctrlr;
 991         struct iscsi_task *task;
 992         struct beiscsi_io_task *io_task;
 993         struct iscsi_conn *conn = beiscsi_conn->conn;
 994         struct iscsi_session *session = conn->session;
 995
 996         phwi_ctrlr = phba->phwi_ctrlr;
 997         pwrb_context = &phwi_ctrlr->wrb_context[((psol->
 998                                 dw[offsetof(struct amap_sol_cqe, cid) / 32] &
 999                                 SOL_CID_MASK) >> 6) -
1000                                 phba->fw_config.iscsi_cid_start];
1001         pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
1002                                 dw[offsetof(struct amap_sol_cqe, wrb_index) /
1003                                 32] & SOL_WRB_INDEX_MASK) >> 16)];
1004         task = pwrb_handle->pio_handle;
1005
1006         io_task = task->dd_data;
1007         spin_lock(&phba->mgmt_sgl_lock);
1008         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
1009         spin_unlock(&phba->mgmt_sgl_lock);
1010         spin_lock_bh(&session->lock);
1011         free_wrb_handle(phba, pwrb_context, pwrb_handle);
1012         spin_unlock_bh(&session->lock);
1013 }
1014
1015 static void
1016 be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
1017                        struct iscsi_task *task, struct sol_cqe *psol)
1018 {
1019         struct iscsi_nopin *hdr;
1020         struct iscsi_conn *conn = beiscsi_conn->conn;
1021         struct beiscsi_io_task *io_task = task->dd_data;
1022
1023         hdr = (struct iscsi_nopin *)task->hdr;
1024         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1025                         & SOL_FLAGS_MASK) >> 24) | 0x80;
1026         hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
1027                                      i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
1028         hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
1029                         i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
1030                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1031                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1032         hdr->opcode = ISCSI_OP_NOOP_IN;
1033         hdr->itt = io_task->libiscsi_itt;
1034         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1035 }
1036
1037 static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
1038                              struct beiscsi_hba *phba, struct sol_cqe *psol)
1039 {
1040         struct hwi_wrb_context *pwrb_context;
1041         struct wrb_handle *pwrb_handle;
1042         struct iscsi_wrb *pwrb = NULL;
1043         struct hwi_controller *phwi_ctrlr;
1044         struct iscsi_task *task;
1045         unsigned int type;
1046         struct iscsi_conn *conn = beiscsi_conn->conn;
1047         struct iscsi_session *session = conn->session;
1048
1049         phwi_ctrlr = phba->phwi_ctrlr;
1050         pwrb_context = &phwi_ctrlr->wrb_context[((psol->dw[offsetof
1051                                 (struct amap_sol_cqe, cid) / 32]
1052                                 & SOL_CID_MASK) >> 6) -
1053                                 phba->fw_config.iscsi_cid_start];
1054         pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
1055                                 dw[offsetof(struct amap_sol_cqe, wrb_index) /
1056                                 32] & SOL_WRB_INDEX_MASK) >> 16)];
1057         task = pwrb_handle->pio_handle;
1058         pwrb = pwrb_handle->pwrb;
1059         type = (pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] &
1060                                  WRB_TYPE_MASK) >> 28;
1061
1062         spin_lock_bh(&session->lock);
1063         switch (type) {
1064         case HWH_TYPE_IO:
1065         case HWH_TYPE_IO_RD:
1066                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
1067                      ISCSI_OP_NOOP_OUT)
1068                         be_complete_nopin_resp(beiscsi_conn, task, psol);
1069                 else
1070                         be_complete_io(beiscsi_conn, task, psol);
1071                 break;
1072
1073         case HWH_TYPE_LOGOUT:
1074                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT)
1075                         be_complete_logout(beiscsi_conn, task, psol);
1076                 else
1077                         be_complete_tmf(beiscsi_conn, task, psol);
1078
1079                 break;
1080
1081         case HWH_TYPE_LOGIN:
1082                 SE_DEBUG(DBG_LVL_1,
1083                          "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd"
1084                          "- Solicited path \n");
1085                 break;
1086
1087         case HWH_TYPE_NOP:
1088                 be_complete_nopin_resp(beiscsi_conn, task, psol);
1089                 break;
1090
1091         default:
1092                 shost_printk(KERN_WARNING, phba->shost,
1093                                 "In hwi_complete_cmd, unknown type = %d"
1094                                 "wrb_index 0x%x CID 0x%x\n", type,
1095                                 ((psol->dw[offsetof(struct amap_iscsi_wrb,
1096                                 type) / 32] & SOL_WRB_INDEX_MASK) >> 16),
1097                                 ((psol->dw[offsetof(struct amap_sol_cqe,
1098                                 cid) / 32] & SOL_CID_MASK) >> 6));
1099                 break;
1100         }
1101
1102         spin_unlock_bh(&session->lock);
1103 }
1104
1105 static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
1106                                           *pasync_ctx, unsigned int is_header,
1107                                           unsigned int host_write_ptr)
1108 {
1109         if (is_header)
1110                 return &pasync_ctx->async_entry[host_write_ptr].
1111                     header_busy_list;
1112         else
1113                 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
1114 }
1115
1116 static struct async_pdu_handle *
1117 hwi_get_async_handle(struct beiscsi_hba *phba,
1118                      struct beiscsi_conn *beiscsi_conn,
1119                      struct hwi_async_pdu_context *pasync_ctx,
1120                      struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
1121 {
1122         struct be_bus_address phys_addr;
1123         struct list_head *pbusy_list;
1124         struct async_pdu_handle *pasync_handle = NULL;
1125         int buffer_len = 0;
1126         unsigned char buffer_index = -1;
1127         unsigned char is_header = 0;
1128
1129         phys_addr.u.a32.address_lo =
1130             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_lo) / 32] -
1131             ((pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1132                                                 & PDUCQE_DPL_MASK) >> 16);
1133         phys_addr.u.a32.address_hi =
1134             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_hi) / 32];
1135
1136         phys_addr.u.a64.address =
1137                         *((unsigned long long *)(&phys_addr.u.a64.address));
1138
1139         switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
1140                         & PDUCQE_CODE_MASK) {
1141         case UNSOL_HDR_NOTIFY:
1142                 is_header = 1;
1143
1144                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 1,
1145                         (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1146                         index) / 32] & PDUCQE_INDEX_MASK));
1147
1148                 buffer_len = (unsigned int)(phys_addr.u.a64.address -
1149                                 pasync_ctx->async_header.pa_base.u.a64.address);
1150
1151                 buffer_index = buffer_len /
1152                                 pasync_ctx->async_header.buffer_size;
1153
1154                 break;
1155         case UNSOL_DATA_NOTIFY:
1156                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 0, (pdpdu_cqe->
1157                                         dw[offsetof(struct amap_i_t_dpdu_cqe,
1158                                         index) / 32] & PDUCQE_INDEX_MASK));
1159                 buffer_len = (unsigned long)(phys_addr.u.a64.address -
1160                                         pasync_ctx->async_data.pa_base.u.
1161                                         a64.address);
1162                 buffer_index = buffer_len / pasync_ctx->async_data.buffer_size;
1163                 break;
1164         default:
1165                 pbusy_list = NULL;
1166                 shost_printk(KERN_WARNING, phba->shost,
1167                         "Unexpected code=%d \n",
1168                          pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1169                                         code) / 32] & PDUCQE_CODE_MASK);
1170                 return NULL;
1171         }
1172
1173         WARN_ON(!(buffer_index <= pasync_ctx->async_data.num_entries));
1174         WARN_ON(list_empty(pbusy_list));
1175         list_for_each_entry(pasync_handle, pbusy_list, link) {
1176                 WARN_ON(pasync_handle->consumed);
1177                 if (pasync_handle->index == buffer_index)
1178                         break;
1179         }
1180
1181         WARN_ON(!pasync_handle);
1182
1183         pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid -
1184                                              phba->fw_config.iscsi_cid_start;
1185         pasync_handle->is_header = is_header;
1186         pasync_handle->buffer_len = ((pdpdu_cqe->
1187                         dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1188                         & PDUCQE_DPL_MASK) >> 16);
1189
1190         *pcq_index = (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1191                         index) / 32] & PDUCQE_INDEX_MASK);
1192         return pasync_handle;
1193 }
1194
1195 static unsigned int
1196 hwi_update_async_writables(struct hwi_async_pdu_context *pasync_ctx,
1197                            unsigned int is_header, unsigned int cq_index)
1198 {
1199         struct list_head *pbusy_list;
1200         struct async_pdu_handle *pasync_handle;
1201         unsigned int num_entries, writables = 0;
1202         unsigned int *pep_read_ptr, *pwritables;
1203
1204
1205         if (is_header) {
1206                 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
1207                 pwritables = &pasync_ctx->async_header.writables;
1208                 num_entries = pasync_ctx->async_header.num_entries;
1209         } else {
1210                 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
1211                 pwritables = &pasync_ctx->async_data.writables;
1212                 num_entries = pasync_ctx->async_data.num_entries;
1213         }
1214
1215         while ((*pep_read_ptr) != cq_index) {
1216                 (*pep_read_ptr)++;
1217                 *pep_read_ptr = (*pep_read_ptr) % num_entries;
1218
1219                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
1220                                                      *pep_read_ptr);
1221                 if (writables == 0)
1222                         WARN_ON(list_empty(pbusy_list));
1223
1224                 if (!list_empty(pbusy_list)) {
1225                         pasync_handle = list_entry(pbusy_list->next,
1226                                                    struct async_pdu_handle,
1227                                                    link);
1228                         WARN_ON(!pasync_handle);
1229                         pasync_handle->consumed = 1;
1230                 }
1231
1232                 writables++;
1233         }
1234
1235         if (!writables) {
1236                 SE_DEBUG(DBG_LVL_1,
1237                          "Duplicate notification received - index 0x%x!!\n",
1238                          cq_index);
1239                 WARN_ON(1);
1240         }
1241
1242         *pwritables = *pwritables + writables;
1243         return 0;
1244 }
1245
1246 static unsigned int hwi_free_async_msg(struct beiscsi_hba *phba,
1247                                        unsigned int cri)
1248 {
1249         struct hwi_controller *phwi_ctrlr;
1250         struct hwi_async_pdu_context *pasync_ctx;
1251         struct async_pdu_handle *pasync_handle, *tmp_handle;
1252         struct list_head *plist;
1253         unsigned int i = 0;
1254
1255         phwi_ctrlr = phba->phwi_ctrlr;
1256         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1257
1258         plist  = &pasync_ctx->async_entry[cri].wait_queue.list;
1259
1260         list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
1261                 list_del(&pasync_handle->link);
1262
1263                 if (i == 0) {
1264                         list_add_tail(&pasync_handle->link,
1265                                       &pasync_ctx->async_header.free_list);
1266                         pasync_ctx->async_header.free_entries++;
1267                         i++;
1268                 } else {
1269                         list_add_tail(&pasync_handle->link,
1270                                       &pasync_ctx->async_data.free_list);
1271                         pasync_ctx->async_data.free_entries++;
1272                         i++;
1273                 }
1274         }
1275
1276         INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
1277         pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
1278         pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1279         return 0;
1280 }
1281
1282 static struct phys_addr *
1283 hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
1284                      unsigned int is_header, unsigned int host_write_ptr)
1285 {
1286         struct phys_addr *pasync_sge = NULL;
1287
1288         if (is_header)
1289                 pasync_sge = pasync_ctx->async_header.ring_base;
1290         else
1291                 pasync_sge = pasync_ctx->async_data.ring_base;
1292
1293         return pasync_sge + host_write_ptr;
1294 }
1295
1296 static void hwi_post_async_buffers(struct beiscsi_hba *phba,
1297                                    unsigned int is_header)
1298 {
1299         struct hwi_controller *phwi_ctrlr;
1300         struct hwi_async_pdu_context *pasync_ctx;
1301         struct async_pdu_handle *pasync_handle;
1302         struct list_head *pfree_link, *pbusy_list;
1303         struct phys_addr *pasync_sge;
1304         unsigned int ring_id, num_entries;
1305         unsigned int host_write_num;
1306         unsigned int writables;
1307         unsigned int i = 0;
1308         u32 doorbell = 0;
1309
1310         phwi_ctrlr = phba->phwi_ctrlr;
1311         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1312
1313         if (is_header) {
1314                 num_entries = pasync_ctx->async_header.num_entries;
1315                 writables = min(pasync_ctx->async_header.writables,
1316                                 pasync_ctx->async_header.free_entries);
1317                 pfree_link = pasync_ctx->async_header.free_list.next;
1318                 host_write_num = pasync_ctx->async_header.host_write_ptr;
1319                 ring_id = phwi_ctrlr->default_pdu_hdr.id;
1320         } else {
1321                 num_entries = pasync_ctx->async_data.num_entries;
1322                 writables = min(pasync_ctx->async_data.writables,
1323                                 pasync_ctx->async_data.free_entries);
1324                 pfree_link = pasync_ctx->async_data.free_list.next;
1325                 host_write_num = pasync_ctx->async_data.host_write_ptr;
1326                 ring_id = phwi_ctrlr->default_pdu_data.id;
1327         }
1328
1329         writables = (writables / 8) * 8;
1330         if (writables) {
1331                 for (i = 0; i < writables; i++) {
1332                         pbusy_list =
1333                             hwi_get_async_busy_list(pasync_ctx, is_header,
1334                                                     host_write_num);
1335                         pasync_handle =
1336                             list_entry(pfree_link, struct async_pdu_handle,
1337                                                                 link);
1338                         WARN_ON(!pasync_handle);
1339                         pasync_handle->consumed = 0;
1340
1341                         pfree_link = pfree_link->next;
1342
1343                         pasync_sge = hwi_get_ring_address(pasync_ctx,
1344                                                 is_header, host_write_num);
1345
1346                         pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
1347                         pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
1348
1349                         list_move(&pasync_handle->link, pbusy_list);
1350
1351                         host_write_num++;
1352                         host_write_num = host_write_num % num_entries;
1353                 }
1354
1355                 if (is_header) {
1356                         pasync_ctx->async_header.host_write_ptr =
1357                                                         host_write_num;
1358                         pasync_ctx->async_header.free_entries -= writables;
1359                         pasync_ctx->async_header.writables -= writables;
1360                         pasync_ctx->async_header.busy_entries += writables;
1361                 } else {
1362                         pasync_ctx->async_data.host_write_ptr = host_write_num;
1363                         pasync_ctx->async_data.free_entries -= writables;
1364                         pasync_ctx->async_data.writables -= writables;
1365                         pasync_ctx->async_data.busy_entries += writables;
1366                 }
1367
1368                 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1369                 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1370                 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1371                 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
1372                                         << DB_DEF_PDU_CQPROC_SHIFT;
1373
1374                 iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET);
1375         }
1376 }
1377
1378 static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
1379                                          struct beiscsi_conn *beiscsi_conn,
1380                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1381 {
1382         struct hwi_controller *phwi_ctrlr;
1383         struct hwi_async_pdu_context *pasync_ctx;
1384         struct async_pdu_handle *pasync_handle = NULL;
1385         unsigned int cq_index = -1;
1386
1387         phwi_ctrlr = phba->phwi_ctrlr;
1388         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1389
1390         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1391                                              pdpdu_cqe, &cq_index);
1392         BUG_ON(pasync_handle->is_header != 0);
1393         if (pasync_handle->consumed == 0)
1394                 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1395                                            cq_index);
1396
1397         hwi_free_async_msg(phba, pasync_handle->cri);
1398         hwi_post_async_buffers(phba, pasync_handle->is_header);
1399 }
1400
1401 static unsigned int
1402 hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
1403                   struct beiscsi_hba *phba,
1404                   struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
1405 {
1406         struct list_head *plist;
1407         struct async_pdu_handle *pasync_handle;
1408         void *phdr = NULL;
1409         unsigned int hdr_len = 0, buf_len = 0;
1410         unsigned int status, index = 0, offset = 0;
1411         void *pfirst_buffer = NULL;
1412         unsigned int num_buf = 0;
1413
1414         plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1415
1416         list_for_each_entry(pasync_handle, plist, link) {
1417                 if (index == 0) {
1418                         phdr = pasync_handle->pbuffer;
1419                         hdr_len = pasync_handle->buffer_len;
1420                 } else {
1421                         buf_len = pasync_handle->buffer_len;
1422                         if (!num_buf) {
1423                                 pfirst_buffer = pasync_handle->pbuffer;
1424                                 num_buf++;
1425                         }
1426                         memcpy(pfirst_buffer + offset,
1427                                pasync_handle->pbuffer, buf_len);
1428                         offset = buf_len;
1429                 }
1430                 index++;
1431         }
1432
1433         status = beiscsi_process_async_pdu(beiscsi_conn, phba,
1434                                            (beiscsi_conn->beiscsi_conn_cid -
1435                                             phba->fw_config.iscsi_cid_start),
1436                                             phdr, hdr_len, pfirst_buffer,
1437                                             buf_len);
1438
1439         if (status == 0)
1440                 hwi_free_async_msg(phba, cri);
1441         return 0;
1442 }
1443
1444 static unsigned int
1445 hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
1446                      struct beiscsi_hba *phba,
1447                      struct async_pdu_handle *pasync_handle)
1448 {
1449         struct hwi_async_pdu_context *pasync_ctx;
1450         struct hwi_controller *phwi_ctrlr;
1451         unsigned int bytes_needed = 0, status = 0;
1452         unsigned short cri = pasync_handle->cri;
1453         struct pdu_base *ppdu;
1454
1455         phwi_ctrlr = phba->phwi_ctrlr;
1456         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1457
1458         list_del(&pasync_handle->link);
1459         if (pasync_handle->is_header) {
1460                 pasync_ctx->async_header.busy_entries--;
1461                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1462                         hwi_free_async_msg(phba, cri);
1463                         BUG();
1464                 }
1465
1466                 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1467                 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
1468                 pasync_ctx->async_entry[cri].wait_queue.hdr_len =
1469                                 (unsigned short)pasync_handle->buffer_len;
1470                 list_add_tail(&pasync_handle->link,
1471                               &pasync_ctx->async_entry[cri].wait_queue.list);
1472
1473                 ppdu = pasync_handle->pbuffer;
1474                 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
1475                         data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
1476                         0xFFFF0000) | ((be16_to_cpu((ppdu->
1477                         dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
1478                         & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
1479
1480                 if (status == 0) {
1481                         pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
1482                             bytes_needed;
1483
1484                         if (bytes_needed == 0)
1485                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1486                                                            pasync_ctx, cri);
1487                 }
1488         } else {
1489                 pasync_ctx->async_data.busy_entries--;
1490                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1491                         list_add_tail(&pasync_handle->link,
1492                                       &pasync_ctx->async_entry[cri].wait_queue.
1493                                       list);
1494                         pasync_ctx->async_entry[cri].wait_queue.
1495                                 bytes_received +=
1496                                 (unsigned short)pasync_handle->buffer_len;
1497
1498                         if (pasync_ctx->async_entry[cri].wait_queue.
1499                             bytes_received >=
1500                             pasync_ctx->async_entry[cri].wait_queue.
1501                             bytes_needed)
1502                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1503                                                            pasync_ctx, cri);
1504                 }
1505         }
1506         return status;
1507 }
1508
1509 static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
1510                                          struct beiscsi_hba *phba,
1511                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1512 {
1513         struct hwi_controller *phwi_ctrlr;
1514         struct hwi_async_pdu_context *pasync_ctx;
1515         struct async_pdu_handle *pasync_handle = NULL;
1516         unsigned int cq_index = -1;
1517
1518         phwi_ctrlr = phba->phwi_ctrlr;
1519         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1520         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1521                                              pdpdu_cqe, &cq_index);
1522
1523         if (pasync_handle->consumed == 0)
1524                 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1525                                            cq_index);
1526         hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
1527         hwi_post_async_buffers(phba, pasync_handle->is_header);
1528 }
1529
1530 static void  beiscsi_process_mcc_isr(struct beiscsi_hba *phba)
1531 {
1532         struct be_queue_info *mcc_cq;
1533         struct  be_mcc_compl *mcc_compl;
1534         unsigned int num_processed = 0;
1535
1536         mcc_cq = &phba->ctrl.mcc_obj.cq;
1537         mcc_compl = queue_tail_node(mcc_cq);
1538         mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1539         while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) {
1540
1541                 if (num_processed >= 32) {
1542                         hwi_ring_cq_db(phba, mcc_cq->id,
1543                                         num_processed, 0, 0);
1544                         num_processed = 0;
1545                 }
1546                 if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) {
1547                         /* Interpret flags as an async trailer */
1548                         if (is_link_state_evt(mcc_compl->flags))
1549                                 /* Interpret compl as a async link evt */
1550                                 beiscsi_async_link_state_process(phba,
1551                                 (struct be_async_event_link_state *) mcc_compl);
1552                         else
1553                                 SE_DEBUG(DBG_LVL_1,
1554                                         " Unsupported Async Event, flags"
1555                                         " = 0x%08x \n", mcc_compl->flags);
1556                 } else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) {
1557                         be_mcc_compl_process_isr(&phba->ctrl, mcc_compl);
1558                         atomic_dec(&phba->ctrl.mcc_obj.q.used);
1559                 }
1560
1561                 mcc_compl->flags = 0;
1562                 queue_tail_inc(mcc_cq);
1563                 mcc_compl = queue_tail_node(mcc_cq);
1564                 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1565                 num_processed++;
1566         }
1567
1568         if (num_processed > 0)
1569                 hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1, 0);
1570
1571 }
1572
1573 static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
1574 {
1575         struct be_queue_info *cq;
1576         struct sol_cqe *sol;
1577         struct dmsg_cqe *dmsg;
1578         unsigned int num_processed = 0;
1579         unsigned int tot_nump = 0;
1580         struct beiscsi_conn *beiscsi_conn;
1581         struct beiscsi_endpoint *beiscsi_ep;
1582         struct iscsi_endpoint *ep;
1583         struct beiscsi_hba *phba;
1584
1585         cq = pbe_eq->cq;
1586         sol = queue_tail_node(cq);
1587         phba = pbe_eq->phba;
1588
1589         while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
1590                CQE_VALID_MASK) {
1591                 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
1592
1593                 ep = phba->ep_array[(u32) ((sol->
1594                                    dw[offsetof(struct amap_sol_cqe, cid) / 32] &
1595                                    SOL_CID_MASK) >> 6) -
1596                                    phba->fw_config.iscsi_cid_start];
1597
1598                 beiscsi_ep = ep->dd_data;
1599                 beiscsi_conn = beiscsi_ep->conn;
1600
1601                 if (num_processed >= 32) {
1602                         hwi_ring_cq_db(phba, cq->id,
1603                                         num_processed, 0, 0);
1604                         tot_nump += num_processed;
1605                         num_processed = 0;
1606                 }
1607
1608                 switch ((u32) sol->dw[offsetof(struct amap_sol_cqe, code) /
1609                         32] & CQE_CODE_MASK) {
1610                 case SOL_CMD_COMPLETE:
1611                         hwi_complete_cmd(beiscsi_conn, phba, sol);
1612                         break;
1613                 case DRIVERMSG_NOTIFY:
1614                         SE_DEBUG(DBG_LVL_8, "Received DRIVERMSG_NOTIFY \n");
1615                         dmsg = (struct dmsg_cqe *)sol;
1616                         hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
1617                         break;
1618                 case UNSOL_HDR_NOTIFY:
1619                         SE_DEBUG(DBG_LVL_8, "Received UNSOL_HDR_ NOTIFY\n");
1620                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
1621                                              (struct i_t_dpdu_cqe *)sol);
1622                         break;
1623                 case UNSOL_DATA_NOTIFY:
1624                         SE_DEBUG(DBG_LVL_8, "Received UNSOL_DATA_NOTIFY\n");
1625                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
1626                                              (struct i_t_dpdu_cqe *)sol);
1627                         break;
1628                 case CXN_INVALIDATE_INDEX_NOTIFY:
1629                 case CMD_INVALIDATED_NOTIFY:
1630                 case CXN_INVALIDATE_NOTIFY:
1631                         SE_DEBUG(DBG_LVL_1,
1632                                  "Ignoring CQ Error notification for cmd/cxn"
1633                                  "invalidate\n");
1634                         break;
1635                 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
1636                 case CMD_KILLED_INVALID_STATSN_RCVD:
1637                 case CMD_KILLED_INVALID_R2T_RCVD:
1638                 case CMD_CXN_KILLED_LUN_INVALID:
1639                 case CMD_CXN_KILLED_ICD_INVALID:
1640                 case CMD_CXN_KILLED_ITT_INVALID:
1641                 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
1642                 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
1643                         SE_DEBUG(DBG_LVL_1,
1644                                  "CQ Error notification for cmd.. "
1645                                  "code %d cid 0x%x\n",
1646                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1647                                  32] & CQE_CODE_MASK,
1648                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1649                                  32] & SOL_CID_MASK));
1650                         break;
1651                 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
1652                         SE_DEBUG(DBG_LVL_1,
1653                                  "Digest error on def pdu ring, dropping..\n");
1654                         hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
1655                                              (struct i_t_dpdu_cqe *) sol);
1656                         break;
1657                 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
1658                 case CXN_KILLED_BURST_LEN_MISMATCH:
1659                 case CXN_KILLED_AHS_RCVD:
1660                 case CXN_KILLED_HDR_DIGEST_ERR:
1661                 case CXN_KILLED_UNKNOWN_HDR:
1662                 case CXN_KILLED_STALE_ITT_TTT_RCVD:
1663                 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
1664                 case CXN_KILLED_TIMED_OUT:
1665                 case CXN_KILLED_FIN_RCVD:
1666                 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
1667                 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
1668                 case CXN_KILLED_OVER_RUN_RESIDUAL:
1669                 case CXN_KILLED_UNDER_RUN_RESIDUAL:
1670                 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
1671                         SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset CID "
1672                                  "0x%x...\n",
1673                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1674                                  32] & CQE_CODE_MASK,
1675                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1676                                  32] & CQE_CID_MASK));
1677                         iscsi_conn_failure(beiscsi_conn->conn,
1678                                            ISCSI_ERR_CONN_FAILED);
1679                         break;
1680                 case CXN_KILLED_RST_SENT:
1681                 case CXN_KILLED_RST_RCVD:
1682                         SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset"
1683                                 "received/sent on CID 0x%x...\n",
1684                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1685                                  32] & CQE_CODE_MASK,
1686                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1687                                  32] & CQE_CID_MASK));
1688                         iscsi_conn_failure(beiscsi_conn->conn,
1689                                            ISCSI_ERR_CONN_FAILED);
1690                         break;
1691                 default:
1692                         SE_DEBUG(DBG_LVL_1, "CQ Error Invalid code= %d "
1693                                  "received on CID 0x%x...\n",
1694                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1695                                  32] & CQE_CODE_MASK,
1696                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1697                                  32] & CQE_CID_MASK));
1698                         break;
1699                 }
1700
1701                 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
1702                 queue_tail_inc(cq);
1703                 sol = queue_tail_node(cq);
1704                 num_processed++;
1705         }
1706
1707         if (num_processed > 0) {
1708                 tot_nump += num_processed;
1709                 hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0);
1710         }
1711         return tot_nump;
1712 }
1713
1714 void beiscsi_process_all_cqs(struct work_struct *work)
1715 {
1716         unsigned long flags;
1717         struct hwi_controller *phwi_ctrlr;
1718         struct hwi_context_memory *phwi_context;
1719         struct be_eq_obj *pbe_eq;
1720         struct beiscsi_hba *phba =
1721             container_of(work, struct beiscsi_hba, work_cqs);
1722
1723         phwi_ctrlr = phba->phwi_ctrlr;
1724         phwi_context = phwi_ctrlr->phwi_ctxt;
1725         if (phba->msix_enabled)
1726                 pbe_eq = &phwi_context->be_eq[phba->num_cpus];
1727         else
1728                 pbe_eq = &phwi_context->be_eq[0];
1729
1730         if (phba->todo_mcc_cq) {
1731                 spin_lock_irqsave(&phba->isr_lock, flags);
1732                 phba->todo_mcc_cq = 0;
1733                 spin_unlock_irqrestore(&phba->isr_lock, flags);
1734                 beiscsi_process_mcc_isr(phba);
1735         }
1736
1737         if (phba->todo_cq) {
1738                 spin_lock_irqsave(&phba->isr_lock, flags);
1739                 phba->todo_cq = 0;
1740                 spin_unlock_irqrestore(&phba->isr_lock, flags);
1741                 beiscsi_process_cq(pbe_eq);
1742         }
1743 }
1744
1745 static int be_iopoll(struct blk_iopoll *iop, int budget)
1746 {
1747         static unsigned int ret;
1748         struct beiscsi_hba *phba;
1749         struct be_eq_obj *pbe_eq;
1750
1751         pbe_eq = container_of(iop, struct be_eq_obj, iopoll);
1752         ret = beiscsi_process_cq(pbe_eq);
1753         if (ret < budget) {
1754                 phba = pbe_eq->phba;
1755                 blk_iopoll_complete(iop);
1756                 SE_DEBUG(DBG_LVL_8, "rearm pbe_eq->q.id =%d\n", pbe_eq->q.id);
1757                 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
1758         }
1759         return ret;
1760 }
1761
1762 static void
1763 hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
1764               unsigned int num_sg, struct beiscsi_io_task *io_task)
1765 {
1766         struct iscsi_sge *psgl;
1767         unsigned short sg_len, index;
1768         unsigned int sge_len = 0;
1769         unsigned long long addr;
1770         struct scatterlist *l_sg;
1771         unsigned int offset;
1772
1773         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
1774                                       io_task->bhs_pa.u.a32.address_lo);
1775         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
1776                                       io_task->bhs_pa.u.a32.address_hi);
1777
1778         l_sg = sg;
1779         for (index = 0; (index < num_sg) && (index < 2); index++,
1780                                                          sg = sg_next(sg)) {
1781                 if (index == 0) {
1782                         sg_len = sg_dma_len(sg);
1783                         addr = (u64) sg_dma_address(sg);
1784                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
1785                                                         (addr & 0xFFFFFFFF));
1786                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
1787                                                         (addr >> 32));
1788                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
1789                                                         sg_len);
1790                         sge_len = sg_len;
1791                 } else {
1792                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
1793                                                         pwrb, sge_len);
1794                         sg_len = sg_dma_len(sg);
1795                         addr = (u64) sg_dma_address(sg);
1796                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
1797                                                         (addr & 0xFFFFFFFF));
1798                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
1799                                                         (addr >> 32));
1800                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
1801                                                         sg_len);
1802                 }
1803         }
1804         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
1805         memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
1806
1807         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
1808
1809         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1810                         io_task->bhs_pa.u.a32.address_hi);
1811         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1812                         io_task->bhs_pa.u.a32.address_lo);
1813
1814         if (num_sg == 1) {
1815                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
1816                                                                 1);
1817                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
1818                                                                 0);
1819         } else if (num_sg == 2) {
1820                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
1821                                                                 0);
1822                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
1823                                                                 1);
1824         } else {
1825                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
1826                                                                 0);
1827                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
1828                                                                 0);
1829         }
1830         sg = l_sg;
1831         psgl++;
1832         psgl++;
1833         offset = 0;
1834         for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
1835                 sg_len = sg_dma_len(sg);
1836                 addr = (u64) sg_dma_address(sg);
1837                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1838                                                 (addr & 0xFFFFFFFF));
1839                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1840                                                 (addr >> 32));
1841                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
1842                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
1843                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
1844                 offset += sg_len;
1845         }
1846         psgl--;
1847         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
1848 }
1849
1850 static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
1851 {
1852         struct iscsi_sge *psgl;
1853         unsigned long long addr;
1854         struct beiscsi_io_task *io_task = task->dd_data;
1855         struct beiscsi_conn *beiscsi_conn = io_task->conn;
1856         struct beiscsi_hba *phba = beiscsi_conn->phba;
1857
1858         io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
1859         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
1860                                 io_task->bhs_pa.u.a32.address_lo);
1861         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
1862                                 io_task->bhs_pa.u.a32.address_hi);
1863
1864         if (task->data) {
1865                 if (task->data_count) {
1866                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
1867                         addr = (u64) pci_map_single(phba->pcidev,
1868                                                     task->data,
1869                                                     task->data_count, 1);
1870                 } else {
1871                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
1872                         addr = 0;
1873                 }
1874                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
1875                                                 (addr & 0xFFFFFFFF));
1876                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
1877                                                 (addr >> 32));
1878                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
1879                                                 task->data_count);
1880
1881                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
1882         } else {
1883                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
1884                 addr = 0;
1885         }
1886
1887         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
1888
1889         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
1890
1891         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1892                       io_task->bhs_pa.u.a32.address_hi);
1893         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1894                       io_task->bhs_pa.u.a32.address_lo);
1895         if (task->data) {
1896                 psgl++;
1897                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
1898                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
1899                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
1900                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
1901                 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
1902                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
1903
1904                 psgl++;
1905                 if (task->data) {
1906                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1907                                                 (addr & 0xFFFFFFFF));
1908                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1909                                                 (addr >> 32));
1910                 }
1911                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
1912         }
1913         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
1914 }
1915
1916 static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
1917 {
1918         unsigned int num_cq_pages, num_async_pdu_buf_pages;
1919         unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
1920         unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
1921
1922         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
1923                                       sizeof(struct sol_cqe));
1924         num_async_pdu_buf_pages =
1925                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1926                                        phba->params.defpdu_hdr_sz);
1927         num_async_pdu_buf_sgl_pages =
1928                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1929                                        sizeof(struct phys_addr));
1930         num_async_pdu_data_pages =
1931                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1932                                        phba->params.defpdu_data_sz);
1933         num_async_pdu_data_sgl_pages =
1934                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1935                                        sizeof(struct phys_addr));
1936
1937         phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
1938
1939         phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
1940                                                  BE_ISCSI_PDU_HEADER_SIZE;
1941         phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
1942                                             sizeof(struct hwi_context_memory);
1943
1944
1945         phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
1946             * (phba->params.wrbs_per_cxn)
1947             * phba->params.cxns_per_ctrl;
1948         wrb_sz_per_cxn =  sizeof(struct wrb_handle) *
1949                                  (phba->params.wrbs_per_cxn);
1950         phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
1951                                 phba->params.cxns_per_ctrl);
1952
1953         phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
1954                 phba->params.icds_per_ctrl;
1955         phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
1956                 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
1957
1958         phba->mem_req[HWI_MEM_ASYNC_HEADER_BUF] =
1959                 num_async_pdu_buf_pages * PAGE_SIZE;
1960         phba->mem_req[HWI_MEM_ASYNC_DATA_BUF] =
1961                 num_async_pdu_data_pages * PAGE_SIZE;
1962         phba->mem_req[HWI_MEM_ASYNC_HEADER_RING] =
1963                 num_async_pdu_buf_sgl_pages * PAGE_SIZE;
1964         phba->mem_req[HWI_MEM_ASYNC_DATA_RING] =
1965                 num_async_pdu_data_sgl_pages * PAGE_SIZE;
1966         phba->mem_req[HWI_MEM_ASYNC_HEADER_HANDLE] =
1967                 phba->params.asyncpdus_per_ctrl *
1968                 sizeof(struct async_pdu_handle);
1969         phba->mem_req[HWI_MEM_ASYNC_DATA_HANDLE] =
1970                 phba->params.asyncpdus_per_ctrl *
1971                 sizeof(struct async_pdu_handle);
1972         phba->mem_req[HWI_MEM_ASYNC_PDU_CONTEXT] =
1973                 sizeof(struct hwi_async_pdu_context) +
1974                 (phba->params.cxns_per_ctrl * sizeof(struct hwi_async_entry));
1975 }
1976
1977 static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
1978 {
1979         struct be_mem_descriptor *mem_descr;
1980         dma_addr_t bus_add;
1981         struct mem_array *mem_arr, *mem_arr_orig;
1982         unsigned int i, j, alloc_size, curr_alloc_size;
1983
1984         phba->phwi_ctrlr = kmalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
1985         if (!phba->phwi_ctrlr)
1986                 return -ENOMEM;
1987
1988         phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
1989                                  GFP_KERNEL);
1990         if (!phba->init_mem) {
1991                 kfree(phba->phwi_ctrlr);
1992                 return -ENOMEM;
1993         }
1994
1995         mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT,
1996                                GFP_KERNEL);
1997         if (!mem_arr_orig) {
1998                 kfree(phba->init_mem);
1999                 kfree(phba->phwi_ctrlr);
2000                 return -ENOMEM;
2001         }
2002
2003         mem_descr = phba->init_mem;
2004         for (i = 0; i < SE_MEM_MAX; i++) {
2005                 j = 0;
2006                 mem_arr = mem_arr_orig;
2007                 alloc_size = phba->mem_req[i];
2008                 memset(mem_arr, 0, sizeof(struct mem_array) *
2009                        BEISCSI_MAX_FRAGS_INIT);
2010                 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
2011                 do {
2012                         mem_arr->virtual_address = pci_alloc_consistent(
2013                                                         phba->pcidev,
2014                                                         curr_alloc_size,
2015                                                         &bus_add);
2016                         if (!mem_arr->virtual_address) {
2017                                 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
2018                                         goto free_mem;
2019                                 if (curr_alloc_size -
2020                                         rounddown_pow_of_two(curr_alloc_size))
2021                                         curr_alloc_size = rounddown_pow_of_two
2022                                                              (curr_alloc_size);
2023                                 else
2024                                         curr_alloc_size = curr_alloc_size / 2;
2025                         } else {
2026                                 mem_arr->bus_address.u.
2027                                     a64.address = (__u64) bus_add;
2028                                 mem_arr->size = curr_alloc_size;
2029                                 alloc_size -= curr_alloc_size;
2030                                 curr_alloc_size = min(be_max_phys_size *
2031                                                       1024, alloc_size);
2032                                 j++;
2033                                 mem_arr++;
2034                         }
2035                 } while (alloc_size);
2036                 mem_descr->num_elements = j;
2037                 mem_descr->size_in_bytes = phba->mem_req[i];
2038                 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j,
2039                                                GFP_KERNEL);
2040                 if (!mem_descr->mem_array)
2041                         goto free_mem;
2042
2043                 memcpy(mem_descr->mem_array, mem_arr_orig,
2044                        sizeof(struct mem_array) * j);
2045                 mem_descr++;
2046         }
2047         kfree(mem_arr_orig);
2048         return 0;
2049 free_mem:
2050         mem_descr->num_elements = j;
2051         while ((i) || (j)) {
2052                 for (j = mem_descr->num_elements; j > 0; j--) {
2053                         pci_free_consistent(phba->pcidev,
2054                                             mem_descr->mem_array[j - 1].size,
2055                                             mem_descr->mem_array[j - 1].
2056                                             virtual_address,
2057                                             mem_descr->mem_array[j - 1].
2058                                             bus_address.u.a64.address);
2059                 }
2060                 if (i) {
2061                         i--;
2062                         kfree(mem_descr->mem_array);
2063                         mem_descr--;
2064                 }
2065         }
2066         kfree(mem_arr_orig);
2067         kfree(phba->init_mem);
2068         kfree(phba->phwi_ctrlr);
2069         return -ENOMEM;
2070 }
2071
2072 static int beiscsi_get_memory(struct beiscsi_hba *phba)
2073 {
2074         beiscsi_find_mem_req(phba);
2075         return beiscsi_alloc_mem(phba);
2076 }
2077
2078 static void iscsi_init_global_templates(struct beiscsi_hba *phba)
2079 {
2080         struct pdu_data_out *pdata_out;
2081         struct pdu_nop_out *pnop_out;
2082         struct be_mem_descriptor *mem_descr;
2083
2084         mem_descr = phba->init_mem;
2085         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
2086         pdata_out =
2087             (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
2088         memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2089
2090         AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
2091                       IIOC_SCSI_DATA);
2092
2093         pnop_out =
2094             (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
2095                                    virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
2096
2097         memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2098         AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
2099         AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
2100         AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
2101 }
2102
2103 static void beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
2104 {
2105         struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
2106         struct wrb_handle *pwrb_handle;
2107         struct hwi_controller *phwi_ctrlr;
2108         struct hwi_wrb_context *pwrb_context;
2109         struct iscsi_wrb *pwrb;
2110         unsigned int num_cxn_wrbh;
2111         unsigned int num_cxn_wrb, j, idx, index;
2112
2113         mem_descr_wrbh = phba->init_mem;
2114         mem_descr_wrbh += HWI_MEM_WRBH;
2115
2116         mem_descr_wrb = phba->init_mem;
2117         mem_descr_wrb += HWI_MEM_WRB;
2118
2119         idx = 0;
2120         pwrb_handle = mem_descr_wrbh->mem_array[idx].virtual_address;
2121         num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
2122                         ((sizeof(struct wrb_handle)) *
2123                          phba->params.wrbs_per_cxn));
2124         phwi_ctrlr = phba->phwi_ctrlr;
2125
2126         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2127                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2128                 pwrb_context->pwrb_handle_base =
2129                                 kzalloc(sizeof(struct wrb_handle *) *
2130                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2131                 pwrb_context->pwrb_handle_basestd =
2132                                 kzalloc(sizeof(struct wrb_handle *) *
2133                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2134                 if (num_cxn_wrbh) {
2135                         pwrb_context->alloc_index = 0;
2136                         pwrb_context->wrb_handles_available = 0;
2137                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2138                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2139                                 pwrb_context->pwrb_handle_basestd[j] =
2140                                                                 pwrb_handle;
2141                                 pwrb_context->wrb_handles_available++;
2142                                 pwrb_handle->wrb_index = j;
2143                                 pwrb_handle++;
2144                         }
2145                         pwrb_context->free_index = 0;
2146                         num_cxn_wrbh--;
2147                 } else {
2148                         idx++;
2149                         pwrb_handle =
2150                             mem_descr_wrbh->mem_array[idx].virtual_address;
2151                         num_cxn_wrbh =
2152                             ((mem_descr_wrbh->mem_array[idx].size) /
2153                              ((sizeof(struct wrb_handle)) *
2154                               phba->params.wrbs_per_cxn));
2155                         pwrb_context->alloc_index = 0;
2156                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2157                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2158                                 pwrb_context->pwrb_handle_basestd[j] =
2159                                     pwrb_handle;
2160                                 pwrb_context->wrb_handles_available++;
2161                                 pwrb_handle->wrb_index = j;
2162                                 pwrb_handle++;
2163                         }
2164                         pwrb_context->free_index = 0;
2165                         num_cxn_wrbh--;
2166                 }
2167         }
2168         idx = 0;
2169         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2170         num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
2171                       ((sizeof(struct iscsi_wrb) *
2172                         phba->params.wrbs_per_cxn));
2173         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2174                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2175                 if (num_cxn_wrb) {
2176                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2177                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2178                                 pwrb_handle->pwrb = pwrb;
2179                                 pwrb++;
2180                         }
2181                         num_cxn_wrb--;
2182                 } else {
2183                         idx++;
2184                         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2185                         num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
2186                                       ((sizeof(struct iscsi_wrb) *
2187                                         phba->params.wrbs_per_cxn));
2188                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2189                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2190                                 pwrb_handle->pwrb = pwrb;
2191                                 pwrb++;
2192                         }
2193                         num_cxn_wrb--;
2194                 }
2195         }
2196 }
2197
2198 static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
2199 {
2200         struct hwi_controller *phwi_ctrlr;
2201         struct hba_parameters *p = &phba->params;
2202         struct hwi_async_pdu_context *pasync_ctx;
2203         struct async_pdu_handle *pasync_header_h, *pasync_data_h;
2204         unsigned int index;
2205         struct be_mem_descriptor *mem_descr;
2206
2207         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2208         mem_descr += HWI_MEM_ASYNC_PDU_CONTEXT;
2209
2210         phwi_ctrlr = phba->phwi_ctrlr;
2211         phwi_ctrlr->phwi_ctxt->pasync_ctx = (struct hwi_async_pdu_context *)
2212                                 mem_descr->mem_array[0].virtual_address;
2213         pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx;
2214         memset(pasync_ctx, 0, sizeof(*pasync_ctx));
2215
2216         pasync_ctx->async_header.num_entries = p->asyncpdus_per_ctrl;
2217         pasync_ctx->async_header.buffer_size = p->defpdu_hdr_sz;
2218         pasync_ctx->async_data.buffer_size = p->defpdu_data_sz;
2219         pasync_ctx->async_data.num_entries = p->asyncpdus_per_ctrl;
2220
2221         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2222         mem_descr += HWI_MEM_ASYNC_HEADER_BUF;
2223         if (mem_descr->mem_array[0].virtual_address) {
2224                 SE_DEBUG(DBG_LVL_8,
2225                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_BUF"
2226                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
2227         } else
2228                 shost_printk(KERN_WARNING, phba->shost,
2229                              "No Virtual address \n");
2230
2231         pasync_ctx->async_header.va_base =
2232                         mem_descr->mem_array[0].virtual_address;
2233
2234         pasync_ctx->async_header.pa_base.u.a64.address =
2235                         mem_descr->mem_array[0].bus_address.u.a64.address;
2236
2237         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2238         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2239         if (mem_descr->mem_array[0].virtual_address) {
2240                 SE_DEBUG(DBG_LVL_8,
2241                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_RING"
2242                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
2243         } else
2244                 shost_printk(KERN_WARNING, phba->shost,
2245                             "No Virtual address \n");
2246         pasync_ctx->async_header.ring_base =
2247                         mem_descr->mem_array[0].virtual_address;
2248
2249         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2250         mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE;
2251         if (mem_descr->mem_array[0].virtual_address) {
2252                 SE_DEBUG(DBG_LVL_8,
2253                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_HANDLE"
2254                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
2255         } else
2256                 shost_printk(KERN_WARNING, phba->shost,
2257                             "No Virtual address \n");
2258
2259         pasync_ctx->async_header.handle_base =
2260                         mem_descr->mem_array[0].virtual_address;
2261         pasync_ctx->async_header.writables = 0;
2262         INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
2263
2264         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2265         mem_descr += HWI_MEM_ASYNC_DATA_BUF;
2266         if (mem_descr->mem_array[0].virtual_address) {
2267                 SE_DEBUG(DBG_LVL_8,
2268                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_BUF"
2269                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
2270         } else
2271                 shost_printk(KERN_WARNING, phba->shost,
2272                             "No Virtual address \n");
2273         pasync_ctx->async_data.va_base =
2274                         mem_descr->mem_array[0].virtual_address;
2275         pasync_ctx->async_data.pa_base.u.a64.address =
2276                         mem_descr->mem_array[0].bus_address.u.a64.address;
2277
2278         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2279         mem_descr += HWI_MEM_ASYNC_DATA_RING;
2280         if (mem_descr->mem_array[0].virtual_address) {
2281                 SE_DEBUG(DBG_LVL_8,
2282                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_RING"
2283                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
2284         } else
2285                 shost_printk(KERN_WARNING, phba->shost,
2286                              "No Virtual address \n");
2287
2288         pasync_ctx->async_data.ring_base =
2289                         mem_descr->mem_array[0].virtual_address;
2290
2291         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2292         mem_descr += HWI_MEM_ASYNC_DATA_HANDLE;
2293         if (!mem_descr->mem_array[0].virtual_address)
2294                 shost_printk(KERN_WARNING, phba->shost,
2295                             "No Virtual address \n");
2296
2297         pasync_ctx->async_data.handle_base =
2298                         mem_descr->mem_array[0].virtual_address;
2299         pasync_ctx->async_data.writables = 0;
2300         INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
2301
2302         pasync_header_h =
2303                 (struct async_pdu_handle *)pasync_ctx->async_header.handle_base;
2304         pasync_data_h =
2305                 (struct async_pdu_handle *)pasync_ctx->async_data.handle_base;
2306
2307         for (index = 0; index < p->asyncpdus_per_ctrl; index++) {
2308                 pasync_header_h->cri = -1;
2309                 pasync_header_h->index = (char)index;
2310                 INIT_LIST_HEAD(&pasync_header_h->link);
2311                 pasync_header_h->pbuffer =
2312                         (void *)((unsigned long)
2313                         (pasync_ctx->async_header.va_base) +
2314                         (p->defpdu_hdr_sz * index));
2315
2316                 pasync_header_h->pa.u.a64.address =
2317                         pasync_ctx->async_header.pa_base.u.a64.address +
2318                         (p->defpdu_hdr_sz * index);
2319
2320                 list_add_tail(&pasync_header_h->link,
2321                                 &pasync_ctx->async_header.free_list);
2322                 pasync_header_h++;
2323                 pasync_ctx->async_header.free_entries++;
2324                 pasync_ctx->async_header.writables++;
2325
2326                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].wait_queue.list);
2327                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
2328                                header_busy_list);
2329                 pasync_data_h->cri = -1;
2330                 pasync_data_h->index = (char)index;
2331                 INIT_LIST_HEAD(&pasync_data_h->link);
2332                 pasync_data_h->pbuffer =
2333                         (void *)((unsigned long)
2334                         (pasync_ctx->async_data.va_base) +
2335                         (p->defpdu_data_sz * index));
2336
2337                 pasync_data_h->pa.u.a64.address =
2338                     pasync_ctx->async_data.pa_base.u.a64.address +
2339                     (p->defpdu_data_sz * index);
2340
2341                 list_add_tail(&pasync_data_h->link,
2342                               &pasync_ctx->async_data.free_list);
2343                 pasync_data_h++;
2344                 pasync_ctx->async_data.free_entries++;
2345                 pasync_ctx->async_data.writables++;
2346
2347                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].data_busy_list);
2348         }
2349
2350         pasync_ctx->async_header.host_write_ptr = 0;
2351         pasync_ctx->async_header.ep_read_ptr = -1;
2352         pasync_ctx->async_data.host_write_ptr = 0;
2353         pasync_ctx->async_data.ep_read_ptr = -1;
2354 }
2355
2356 static int
2357 be_sgl_create_contiguous(void *virtual_address,
2358                          u64 physical_address, u32 length,
2359                          struct be_dma_mem *sgl)
2360 {
2361         WARN_ON(!virtual_address);
2362         WARN_ON(!physical_address);
2363         WARN_ON(!length > 0);
2364         WARN_ON(!sgl);
2365
2366         sgl->va = virtual_address;
2367         sgl->dma = physical_address;
2368         sgl->size = length;
2369
2370         return 0;
2371 }
2372
2373 static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
2374 {
2375         memset(sgl, 0, sizeof(*sgl));
2376 }
2377
2378 static void
2379 hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
2380                      struct mem_array *pmem, struct be_dma_mem *sgl)
2381 {
2382         if (sgl->va)
2383                 be_sgl_destroy_contiguous(sgl);
2384
2385         be_sgl_create_contiguous(pmem->virtual_address,
2386                                  pmem->bus_address.u.a64.address,
2387                                  pmem->size, sgl);
2388 }
2389
2390 static void
2391 hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
2392                            struct mem_array *pmem, struct be_dma_mem *sgl)
2393 {
2394         if (sgl->va)
2395                 be_sgl_destroy_contiguous(sgl);
2396
2397         be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
2398                                  pmem->bus_address.u.a64.address,
2399                                  pmem->size, sgl);
2400 }
2401
2402 static int be_fill_queue(struct be_queue_info *q,
2403                 u16 len, u16 entry_size, void *vaddress)
2404 {
2405         struct be_dma_mem *mem = &q->dma_mem;
2406
2407         memset(q, 0, sizeof(*q));
2408         q->len = len;
2409         q->entry_size = entry_size;
2410         mem->size = len * entry_size;
2411         mem->va = vaddress;
2412         if (!mem->va)
2413                 return -ENOMEM;
2414         memset(mem->va, 0, mem->size);
2415         return 0;
2416 }
2417
2418 static int beiscsi_create_eqs(struct beiscsi_hba *phba,
2419                              struct hwi_context_memory *phwi_context)
2420 {
2421         unsigned int i, num_eq_pages;
2422         int ret, eq_for_mcc;
2423         struct be_queue_info *eq;
2424         struct be_dma_mem *mem;
2425         void *eq_vaddress;
2426         dma_addr_t paddr;
2427
2428         num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
2429                                       sizeof(struct be_eq_entry));
2430
2431         if (phba->msix_enabled)
2432                 eq_for_mcc = 1;
2433         else
2434                 eq_for_mcc = 0;
2435         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
2436                 eq = &phwi_context->be_eq[i].q;
2437                 mem = &eq->dma_mem;
2438                 phwi_context->be_eq[i].phba = phba;
2439                 eq_vaddress = pci_alloc_consistent(phba->pcidev,
2440                                                      num_eq_pages * PAGE_SIZE,
2441                                                      &paddr);
2442                 if (!eq_vaddress)
2443                         goto create_eq_error;
2444
2445                 mem->va = eq_vaddress;
2446                 ret = be_fill_queue(eq, phba->params.num_eq_entries,
2447                                     sizeof(struct be_eq_entry), eq_vaddress);
2448                 if (ret) {
2449                         shost_printk(KERN_ERR, phba->shost,
2450                                      "be_fill_queue Failed for EQ \n");
2451                         goto create_eq_error;
2452                 }
2453
2454                 mem->dma = paddr;
2455                 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
2456                                             phwi_context->cur_eqd);
2457                 if (ret) {
2458                         shost_printk(KERN_ERR, phba->shost,
2459                                      "beiscsi_cmd_eq_create"
2460                                      "Failedfor EQ \n");
2461                         goto create_eq_error;
2462                 }
2463                 SE_DEBUG(DBG_LVL_8, "eqid = %d\n", phwi_context->be_eq[i].q.id);
2464         }
2465         return 0;
2466 create_eq_error:
2467         for (i = 0; i < (phba->num_cpus + 1); i++) {
2468                 eq = &phwi_context->be_eq[i].q;
2469                 mem = &eq->dma_mem;
2470                 if (mem->va)
2471                         pci_free_consistent(phba->pcidev, num_eq_pages
2472                                             * PAGE_SIZE,
2473                                             mem->va, mem->dma);
2474         }
2475         return ret;
2476 }
2477
2478 static int beiscsi_create_cqs(struct beiscsi_hba *phba,
2479                              struct hwi_context_memory *phwi_context)
2480 {
2481         unsigned int i, num_cq_pages;
2482         int ret;
2483         struct be_queue_info *cq, *eq;
2484         struct be_dma_mem *mem;
2485         struct be_eq_obj *pbe_eq;
2486         void *cq_vaddress;
2487         dma_addr_t paddr;
2488
2489         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2490                                       sizeof(struct sol_cqe));
2491
2492         for (i = 0; i < phba->num_cpus; i++) {
2493                 cq = &phwi_context->be_cq[i];
2494                 eq = &phwi_context->be_eq[i].q;
2495                 pbe_eq = &phwi_context->be_eq[i];
2496                 pbe_eq->cq = cq;
2497                 pbe_eq->phba = phba;
2498                 mem = &cq->dma_mem;
2499                 cq_vaddress = pci_alloc_consistent(phba->pcidev,
2500                                                      num_cq_pages * PAGE_SIZE,
2501                                                      &paddr);
2502                 if (!cq_vaddress)
2503                         goto create_cq_error;
2504                 ret = be_fill_queue(cq, phba->params.num_cq_entries,
2505                                     sizeof(struct sol_cqe), cq_vaddress);
2506                 if (ret) {
2507                         shost_printk(KERN_ERR, phba->shost,
2508                                      "be_fill_queue Failed for ISCSI CQ \n");
2509                         goto create_cq_error;
2510                 }
2511
2512                 mem->dma = paddr;
2513                 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,
2514                                             false, 0);
2515                 if (ret) {
2516                         shost_printk(KERN_ERR, phba->shost,
2517                                      "beiscsi_cmd_eq_create"
2518                                      "Failed for ISCSI CQ \n");
2519                         goto create_cq_error;
2520                 }
2521                 SE_DEBUG(DBG_LVL_8, "iscsi cq_id is %d for eq_id %d\n",
2522                                                  cq->id, eq->id);
2523                 SE_DEBUG(DBG_LVL_8, "ISCSI CQ CREATED\n");
2524         }
2525         return 0;
2526
2527 create_cq_error:
2528         for (i = 0; i < phba->num_cpus; i++) {
2529                 cq = &phwi_context->be_cq[i];
2530                 mem = &cq->dma_mem;
2531                 if (mem->va)
2532                         pci_free_consistent(phba->pcidev, num_cq_pages
2533                                             * PAGE_SIZE,
2534                                             mem->va, mem->dma);
2535         }
2536         return ret;
2537
2538 }
2539
2540 static int
2541 beiscsi_create_def_hdr(struct beiscsi_hba *phba,
2542                        struct hwi_context_memory *phwi_context,
2543                        struct hwi_controller *phwi_ctrlr,
2544                        unsigned int def_pdu_ring_sz)
2545 {
2546         unsigned int idx;
2547         int ret;
2548         struct be_queue_info *dq, *cq;
2549         struct be_dma_mem *mem;
2550         struct be_mem_descriptor *mem_descr;
2551         void *dq_vaddress;
2552
2553         idx = 0;
2554         dq = &phwi_context->be_def_hdrq;
2555         cq = &phwi_context->be_cq[0];
2556         mem = &dq->dma_mem;
2557         mem_descr = phba->init_mem;
2558         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2559         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2560         ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
2561                             sizeof(struct phys_addr),
2562                             sizeof(struct phys_addr), dq_vaddress);
2563         if (ret) {
2564                 shost_printk(KERN_ERR, phba->shost,
2565                              "be_fill_queue Failed for DEF PDU HDR\n");
2566                 return ret;
2567         }
2568         mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address;
2569         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
2570                                               def_pdu_ring_sz,
2571                                               phba->params.defpdu_hdr_sz);
2572         if (ret) {
2573                 shost_printk(KERN_ERR, phba->shost,
2574                              "be_cmd_create_default_pdu_queue Failed DEFHDR\n");
2575                 return ret;
2576         }
2577         phwi_ctrlr->default_pdu_hdr.id = phwi_context->be_def_hdrq.id;
2578         SE_DEBUG(DBG_LVL_8, "iscsi def pdu id is %d\n",
2579                  phwi_context->be_def_hdrq.id);
2580         hwi_post_async_buffers(phba, 1);
2581         return 0;
2582 }
2583
2584 static int
2585 beiscsi_create_def_data(struct beiscsi_hba *phba,
2586                         struct hwi_context_memory *phwi_context,
2587                         struct hwi_controller *phwi_ctrlr,
2588                         unsigned int def_pdu_ring_sz)
2589 {
2590         unsigned int idx;
2591         int ret;
2592         struct be_queue_info *dataq, *cq;
2593         struct be_dma_mem *mem;
2594         struct be_mem_descriptor *mem_descr;
2595         void *dq_vaddress;
2596
2597         idx = 0;
2598         dataq = &phwi_context->be_def_dataq;
2599         cq = &phwi_context->be_cq[0];
2600         mem = &dataq->dma_mem;
2601         mem_descr = phba->init_mem;
2602         mem_descr += HWI_MEM_ASYNC_DATA_RING;
2603         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2604         ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
2605                             sizeof(struct phys_addr),
2606                             sizeof(struct phys_addr), dq_vaddress);
2607         if (ret) {
2608                 shost_printk(KERN_ERR, phba->shost,
2609                              "be_fill_queue Failed for DEF PDU DATA\n");
2610                 return ret;
2611         }
2612         mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address;
2613         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
2614                                               def_pdu_ring_sz,
2615                                               phba->params.defpdu_data_sz);
2616         if (ret) {
2617                 shost_printk(KERN_ERR, phba->shost,
2618                              "be_cmd_create_default_pdu_queue Failed"
2619                              " for DEF PDU DATA\n");
2620                 return ret;
2621         }
2622         phwi_ctrlr->default_pdu_data.id = phwi_context->be_def_dataq.id;
2623         SE_DEBUG(DBG_LVL_8, "iscsi def data id is %d\n",
2624                  phwi_context->be_def_dataq.id);
2625         hwi_post_async_buffers(phba, 0);
2626         SE_DEBUG(DBG_LVL_8, "DEFAULT PDU DATA RING CREATED \n");
2627         return 0;
2628 }
2629
2630 static int
2631 beiscsi_post_pages(struct beiscsi_hba *phba)
2632 {
2633         struct be_mem_descriptor *mem_descr;
2634         struct mem_array *pm_arr;
2635         unsigned int page_offset, i;
2636         struct be_dma_mem sgl;
2637         int status;
2638
2639         mem_descr = phba->init_mem;
2640         mem_descr += HWI_MEM_SGE;
2641         pm_arr = mem_descr->mem_array;
2642
2643         page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
2644                         phba->fw_config.iscsi_icd_start) / PAGE_SIZE;
2645         for (i = 0; i < mem_descr->num_elements; i++) {
2646                 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
2647                 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
2648                                                 page_offset,
2649                                                 (pm_arr->size / PAGE_SIZE));
2650                 page_offset += pm_arr->size / PAGE_SIZE;
2651                 if (status != 0) {
2652                         shost_printk(KERN_ERR, phba->shost,
2653                                      "post sgl failed.\n");
2654                         return status;
2655                 }
2656                 pm_arr++;
2657         }
2658         SE_DEBUG(DBG_LVL_8, "POSTED PAGES \n");
2659         return 0;
2660 }
2661
2662 static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q)
2663 {
2664         struct be_dma_mem *mem = &q->dma_mem;
2665         if (mem->va)
2666                 pci_free_consistent(phba->pcidev, mem->size,
2667                         mem->va, mem->dma);
2668 }
2669
2670 static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q,
2671                 u16 len, u16 entry_size)
2672 {
2673         struct be_dma_mem *mem = &q->dma_mem;
2674
2675         memset(q, 0, sizeof(*q));
2676         q->len = len;
2677         q->entry_size = entry_size;
2678         mem->size = len * entry_size;
2679         mem->va = pci_alloc_consistent(phba->pcidev, mem->size, &mem->dma);
2680         if (!mem->va)
2681                 return -1;
2682         memset(mem->va, 0, mem->size);
2683         return 0;
2684 }
2685
2686 static int
2687 beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
2688                          struct hwi_context_memory *phwi_context,
2689                          struct hwi_controller *phwi_ctrlr)
2690 {
2691         unsigned int wrb_mem_index, offset, size, num_wrb_rings;
2692         u64 pa_addr_lo;
2693         unsigned int idx, num, i;
2694         struct mem_array *pwrb_arr;
2695         void *wrb_vaddr;
2696         struct be_dma_mem sgl;
2697         struct be_mem_descriptor *mem_descr;
2698         int status;
2699
2700         idx = 0;
2701         mem_descr = phba->init_mem;
2702         mem_descr += HWI_MEM_WRB;
2703         pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
2704                            GFP_KERNEL);
2705         if (!pwrb_arr) {
2706                 shost_printk(KERN_ERR, phba->shost,
2707                              "Memory alloc failed in create wrb ring.\n");
2708                 return -ENOMEM;
2709         }
2710         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2711         pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
2712         num_wrb_rings = mem_descr->mem_array[idx].size /
2713                 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
2714
2715         for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
2716                 if (num_wrb_rings) {
2717                         pwrb_arr[num].virtual_address = wrb_vaddr;
2718                         pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
2719                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2720                                             sizeof(struct iscsi_wrb);
2721                         wrb_vaddr += pwrb_arr[num].size;
2722                         pa_addr_lo += pwrb_arr[num].size;
2723                         num_wrb_rings--;
2724                 } else {
2725                         idx++;
2726                         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2727                         pa_addr_lo = mem_descr->mem_array[idx].\
2728                                         bus_address.u.a64.address;
2729                         num_wrb_rings = mem_descr->mem_array[idx].size /
2730                                         (phba->params.wrbs_per_cxn *
2731                                         sizeof(struct iscsi_wrb));
2732                         pwrb_arr[num].virtual_address = wrb_vaddr;
2733                         pwrb_arr[num].bus_address.u.a64.address\
2734                                                 = pa_addr_lo;
2735                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2736                                                  sizeof(struct iscsi_wrb);
2737                         wrb_vaddr += pwrb_arr[num].size;
2738                         pa_addr_lo   += pwrb_arr[num].size;
2739                         num_wrb_rings--;
2740                 }
2741         }
2742         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2743                 wrb_mem_index = 0;
2744                 offset = 0;
2745                 size = 0;
2746
2747                 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
2748                 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
2749                                             &phwi_context->be_wrbq[i]);
2750                 if (status != 0) {
2751                         shost_printk(KERN_ERR, phba->shost,
2752                                      "wrbq create failed.");
2753                         return status;
2754                 }
2755                 phwi_ctrlr->wrb_context[i * 2].cid = phwi_context->be_wrbq[i].
2756                                                                    id;
2757         }
2758         kfree(pwrb_arr);
2759         return 0;
2760 }
2761
2762 static void free_wrb_handles(struct beiscsi_hba *phba)
2763 {
2764         unsigned int index;
2765         struct hwi_controller *phwi_ctrlr;
2766         struct hwi_wrb_context *pwrb_context;
2767
2768         phwi_ctrlr = phba->phwi_ctrlr;
2769         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2770                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2771                 kfree(pwrb_context->pwrb_handle_base);
2772                 kfree(pwrb_context->pwrb_handle_basestd);
2773         }
2774 }
2775
2776 static void be_mcc_queues_destroy(struct beiscsi_hba *phba)
2777 {
2778         struct be_queue_info *q;
2779         struct be_ctrl_info *ctrl = &phba->ctrl;
2780
2781         q = &phba->ctrl.mcc_obj.q;
2782         if (q->created)
2783                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ);
2784         be_queue_free(phba, q);
2785
2786         q = &phba->ctrl.mcc_obj.cq;
2787         if (q->created)
2788                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
2789         be_queue_free(phba, q);
2790 }
2791
2792 static void hwi_cleanup(struct beiscsi_hba *phba)
2793 {
2794         struct be_queue_info *q;
2795         struct be_ctrl_info *ctrl = &phba->ctrl;
2796         struct hwi_controller *phwi_ctrlr;
2797         struct hwi_context_memory *phwi_context;
2798         int i, eq_num;
2799
2800         phwi_ctrlr = phba->phwi_ctrlr;
2801         phwi_context = phwi_ctrlr->phwi_ctxt;
2802         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2803                 q = &phwi_context->be_wrbq[i];
2804                 if (q->created)
2805                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
2806         }
2807         free_wrb_handles(phba);
2808
2809         q = &phwi_context->be_def_hdrq;
2810         if (q->created)
2811                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
2812
2813         q = &phwi_context->be_def_dataq;
2814         if (q->created)
2815                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
2816
2817         beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
2818
2819         for (i = 0; i < (phba->num_cpus); i++) {
2820                 q = &phwi_context->be_cq[i];
2821                 if (q->created)
2822                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
2823         }
2824         if (phba->msix_enabled)
2825                 eq_num = 1;
2826         else
2827                 eq_num = 0;
2828         for (i = 0; i < (phba->num_cpus + eq_num); i++) {
2829                 q = &phwi_context->be_eq[i].q;
2830                 if (q->created)
2831                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
2832         }
2833         be_mcc_queues_destroy(phba);
2834 }
2835
2836 static int be_mcc_queues_create(struct beiscsi_hba *phba,
2837                                 struct hwi_context_memory *phwi_context)
2838 {
2839         struct be_queue_info *q, *cq;
2840         struct be_ctrl_info *ctrl = &phba->ctrl;
2841
2842         /* Alloc MCC compl queue */
2843         cq = &phba->ctrl.mcc_obj.cq;
2844         if (be_queue_alloc(phba, cq, MCC_CQ_LEN,
2845                         sizeof(struct be_mcc_compl)))
2846                 goto err;
2847         /* Ask BE to create MCC compl queue; */
2848         if (phba->msix_enabled) {
2849                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq
2850                                          [phba->num_cpus].q, false, true, 0))
2851                 goto mcc_cq_free;
2852         } else {
2853                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q,
2854                                           false, true, 0))
2855                 goto mcc_cq_free;
2856         }
2857
2858         /* Alloc MCC queue */
2859         q = &phba->ctrl.mcc_obj.q;
2860         if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2861                 goto mcc_cq_destroy;
2862
2863         /* Ask BE to create MCC queue */
2864         if (beiscsi_cmd_mccq_create(phba, q, cq))
2865                 goto mcc_q_free;
2866
2867         return 0;
2868
2869 mcc_q_free:
2870         be_queue_free(phba, q);
2871 mcc_cq_destroy:
2872         beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ);
2873 mcc_cq_free:
2874         be_queue_free(phba, cq);
2875 err:
2876         return -1;
2877 }
2878
2879 static int find_num_cpus(void)
2880 {
2881         int  num_cpus = 0;
2882
2883         num_cpus = num_online_cpus();
2884         if (num_cpus >= MAX_CPUS)
2885                 num_cpus = MAX_CPUS - 1;
2886
2887         SE_DEBUG(DBG_LVL_8, "num_cpus = %d \n", num_cpus);
2888         return num_cpus;
2889 }
2890
2891 static int hwi_init_port(struct beiscsi_hba *phba)
2892 {
2893         struct hwi_controller *phwi_ctrlr;
2894         struct hwi_context_memory *phwi_context;
2895         unsigned int def_pdu_ring_sz;
2896         struct be_ctrl_info *ctrl = &phba->ctrl;
2897         int status;
2898
2899         def_pdu_ring_sz =
2900                 phba->params.asyncpdus_per_ctrl * sizeof(struct phys_addr);
2901         phwi_ctrlr = phba->phwi_ctrlr;
2902         phwi_context = phwi_ctrlr->phwi_ctxt;
2903         phwi_context->max_eqd = 0;
2904         phwi_context->min_eqd = 0;
2905         phwi_context->cur_eqd = 64;
2906         be_cmd_fw_initialize(&phba->ctrl);
2907
2908         status = beiscsi_create_eqs(phba, phwi_context);
2909         if (status != 0) {
2910                 shost_printk(KERN_ERR, phba->shost, "EQ not created \n");
2911                 goto error;
2912         }
2913
2914         status = be_mcc_queues_create(phba, phwi_context);
2915         if (status != 0)
2916                 goto error;
2917
2918         status = mgmt_check_supported_fw(ctrl, phba);
2919         if (status != 0) {
2920                 shost_printk(KERN_ERR, phba->shost,
2921                              "Unsupported fw version \n");
2922                 goto error;
2923         }
2924
2925         status = beiscsi_create_cqs(phba, phwi_context);
2926         if (status != 0) {
2927                 shost_printk(KERN_ERR, phba->shost, "CQ not created\n");
2928                 goto error;
2929         }
2930
2931         status = beiscsi_create_def_hdr(phba, phwi_context, phwi_ctrlr,
2932                                         def_pdu_ring_sz);
2933         if (status != 0) {
2934                 shost_printk(KERN_ERR, phba->shost,
2935                              "Default Header not created\n");
2936                 goto error;
2937         }
2938
2939         status = beiscsi_create_def_data(phba, phwi_context,
2940                                          phwi_ctrlr, def_pdu_ring_sz);
2941         if (status != 0) {
2942                 shost_printk(KERN_ERR, phba->shost,
2943                              "Default Data not created\n");
2944                 goto error;
2945         }
2946
2947         status = beiscsi_post_pages(phba);
2948         if (status != 0) {
2949                 shost_printk(KERN_ERR, phba->shost, "Post SGL Pages Failed\n");
2950                 goto error;
2951         }
2952
2953         status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr);
2954         if (status != 0) {
2955                 shost_printk(KERN_ERR, phba->shost,
2956                              "WRB Rings not created\n");
2957                 goto error;
2958         }
2959
2960         SE_DEBUG(DBG_LVL_8, "hwi_init_port success\n");
2961         return 0;
2962
2963 error:
2964         shost_printk(KERN_ERR, phba->shost, "hwi_init_port failed");
2965         hwi_cleanup(phba);
2966         return -ENOMEM;
2967 }
2968
2969 static int hwi_init_controller(struct beiscsi_hba *phba)
2970 {
2971         struct hwi_controller *phwi_ctrlr;
2972
2973         phwi_ctrlr = phba->phwi_ctrlr;
2974         if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
2975                 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
2976                     init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
2977                 SE_DEBUG(DBG_LVL_8, " phwi_ctrlr->phwi_ctxt=%p \n",
2978                          phwi_ctrlr->phwi_ctxt);
2979         } else {
2980                 shost_printk(KERN_ERR, phba->shost,
2981                              "HWI_MEM_ADDN_CONTEXT is more than one element."
2982                              "Failing to load\n");
2983                 return -ENOMEM;
2984         }
2985
2986         iscsi_init_global_templates(phba);
2987         beiscsi_init_wrb_handle(phba);
2988         hwi_init_async_pdu_ctx(phba);
2989         if (hwi_init_port(phba) != 0) {
2990                 shost_printk(KERN_ERR, phba->shost,
2991                              "hwi_init_controller failed\n");
2992                 return -ENOMEM;
2993         }
2994         return 0;
2995 }
2996
2997 static void beiscsi_free_mem(struct beiscsi_hba *phba)
2998 {
2999         struct be_mem_descriptor *mem_descr;
3000         int i, j;
3001
3002         mem_descr = phba->init_mem;
3003         i = 0;
3004         j = 0;
3005         for (i = 0; i < SE_MEM_MAX; i++) {
3006                 for (j = mem_descr->num_elements; j > 0; j--) {
3007                         pci_free_consistent(phba->pcidev,
3008                           mem_descr->mem_array[j - 1].size,
3009                           mem_descr->mem_array[j - 1].virtual_address,
3010                           mem_descr->mem_array[j - 1].bus_address.
3011                                 u.a64.address);
3012                 }
3013                 kfree(mem_descr->mem_array);
3014                 mem_descr++;
3015         }
3016         kfree(phba->init_mem);
3017         kfree(phba->phwi_ctrlr);
3018 }
3019
3020 static int beiscsi_init_controller(struct beiscsi_hba *phba)
3021 {
3022         int ret = -ENOMEM;
3023
3024         ret = beiscsi_get_memory(phba);
3025         if (ret < 0) {
3026                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe -"
3027                              "Failed in beiscsi_alloc_memory \n");
3028                 return ret;
3029         }
3030
3031         ret = hwi_init_controller(phba);
3032         if (ret)
3033                 goto free_init;
3034         SE_DEBUG(DBG_LVL_8, "Return success from beiscsi_init_controller");
3035         return 0;
3036
3037 free_init:
3038         beiscsi_free_mem(phba);
3039         return -ENOMEM;
3040 }
3041
3042 static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
3043 {
3044         struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg;
3045         struct sgl_handle *psgl_handle;
3046         struct iscsi_sge *pfrag;
3047         unsigned int arr_index, i, idx;
3048
3049         phba->io_sgl_hndl_avbl = 0;
3050         phba->eh_sgl_hndl_avbl = 0;
3051
3052         mem_descr_sglh = phba->init_mem;
3053         mem_descr_sglh += HWI_MEM_SGLH;
3054         if (1 == mem_descr_sglh->num_elements) {
3055                 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3056                                                  phba->params.ios_per_ctrl,
3057                                                  GFP_KERNEL);
3058                 if (!phba->io_sgl_hndl_base) {
3059                         shost_printk(KERN_ERR, phba->shost,
3060                                      "Mem Alloc Failed. Failing to load\n");
3061                         return -ENOMEM;
3062                 }
3063                 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3064                                                  (phba->params.icds_per_ctrl -
3065                                                  phba->params.ios_per_ctrl),
3066                                                  GFP_KERNEL);
3067                 if (!phba->eh_sgl_hndl_base) {
3068                         kfree(phba->io_sgl_hndl_base);
3069                         shost_printk(KERN_ERR, phba->shost,
3070                                      "Mem Alloc Failed. Failing to load\n");
3071                         return -ENOMEM;
3072                 }
3073         } else {
3074                 shost_printk(KERN_ERR, phba->shost,
3075                              "HWI_MEM_SGLH is more than one element."
3076                              "Failing to load\n");
3077                 return -ENOMEM;
3078         }
3079
3080         arr_index = 0;
3081         idx = 0;
3082         while (idx < mem_descr_sglh->num_elements) {
3083                 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;
3084
3085                 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /
3086                       sizeof(struct sgl_handle)); i++) {
3087                         if (arr_index < phba->params.ios_per_ctrl) {
3088                                 phba->io_sgl_hndl_base[arr_index] = psgl_handle;
3089                                 phba->io_sgl_hndl_avbl++;
3090                                 arr_index++;
3091                         } else {
3092                                 phba->eh_sgl_hndl_base[arr_index -
3093                                         phba->params.ios_per_ctrl] =
3094                                                                 psgl_handle;
3095                                 arr_index++;
3096                                 phba->eh_sgl_hndl_avbl++;
3097                         }
3098                         psgl_handle++;
3099                 }
3100                 idx++;
3101         }
3102         SE_DEBUG(DBG_LVL_8,
3103                  "phba->io_sgl_hndl_avbl=%d"
3104                  "phba->eh_sgl_hndl_avbl=%d \n",
3105                  phba->io_sgl_hndl_avbl,
3106                  phba->eh_sgl_hndl_avbl);
3107         mem_descr_sg = phba->init_mem;
3108         mem_descr_sg += HWI_MEM_SGE;
3109         SE_DEBUG(DBG_LVL_8, "\n mem_descr_sg->num_elements=%d \n",
3110                  mem_descr_sg->num_elements);
3111         arr_index = 0;
3112         idx = 0;
3113         while (idx < mem_descr_sg->num_elements) {
3114                 pfrag = mem_descr_sg->mem_array[idx].virtual_address;
3115
3116                 for (i = 0;
3117                      i < (mem_descr_sg->mem_array[idx].size) /
3118                      (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);
3119                      i++) {
3120                         if (arr_index < phba->params.ios_per_ctrl)
3121                                 psgl_handle = phba->io_sgl_hndl_base[arr_index];
3122                         else
3123                                 psgl_handle = phba->eh_sgl_hndl_base[arr_index -
3124                                                 phba->params.ios_per_ctrl];
3125                         psgl_handle->pfrag = pfrag;
3126                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0);
3127                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0);
3128                         pfrag += phba->params.num_sge_per_io;
3129                         psgl_handle->sgl_index =
3130                                 phba->fw_config.iscsi_icd_start + arr_index++;
3131                 }
3132                 idx++;
3133         }
3134         phba->io_sgl_free_index = 0;
3135         phba->io_sgl_alloc_index = 0;
3136         phba->eh_sgl_free_index = 0;
3137         phba->eh_sgl_alloc_index = 0;
3138         return 0;
3139 }
3140
3141 static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
3142 {
3143         int i, new_cid;
3144
3145         phba->cid_array = kzalloc(sizeof(void *) * phba->params.cxns_per_ctrl,
3146                                   GFP_KERNEL);
3147         if (!phba->cid_array) {
3148                 shost_printk(KERN_ERR, phba->shost,
3149                              "Failed to allocate memory in "
3150                              "hba_setup_cid_tbls\n");
3151                 return -ENOMEM;
3152         }
3153         phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) *
3154                                  phba->params.cxns_per_ctrl * 2, GFP_KERNEL);
3155         if (!phba->ep_array) {
3156                 shost_printk(KERN_ERR, phba->shost,
3157                              "Failed to allocate memory in "
3158                              "hba_setup_cid_tbls \n");
3159                 kfree(phba->cid_array);
3160                 return -ENOMEM;
3161         }
3162         new_cid = phba->fw_config.iscsi_cid_start;
3163         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3164                 phba->cid_array[i] = new_cid;
3165                 new_cid += 2;
3166         }
3167         phba->avlbl_cids = phba->params.cxns_per_ctrl;
3168         return 0;
3169 }
3170
3171 static unsigned char hwi_enable_intr(struct beiscsi_hba *phba)
3172 {
3173         struct be_ctrl_info *ctrl = &phba->ctrl;
3174         struct hwi_controller *phwi_ctrlr;
3175         struct hwi_context_memory *phwi_context;
3176         struct be_queue_info *eq;
3177         u8 __iomem *addr;
3178         u32 reg, i;
3179         u32 enabled;
3180
3181         phwi_ctrlr = phba->phwi_ctrlr;
3182         phwi_context = phwi_ctrlr->phwi_ctxt;
3183
3184         addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
3185                         PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
3186         reg = ioread32(addr);
3187         SE_DEBUG(DBG_LVL_8, "reg =x%08x \n", reg);
3188
3189         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3190         if (!enabled) {
3191                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3192                 SE_DEBUG(DBG_LVL_8, "reg =x%08x addr=%p \n", reg, addr);
3193                 iowrite32(reg, addr);
3194                 if (!phba->msix_enabled) {
3195                         eq = &phwi_context->be_eq[0].q;
3196                         SE_DEBUG(DBG_LVL_8, "eq->id=%d \n", eq->id);
3197                         hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3198                 } else {
3199                         for (i = 0; i <= phba->num_cpus; i++) {
3200                                 eq = &phwi_context->be_eq[i].q;
3201                                 SE_DEBUG(DBG_LVL_8, "eq->id=%d \n", eq->id);
3202                                 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3203                         }
3204                 }
3205         }
3206         return true;
3207 }
3208
3209 static void hwi_disable_intr(struct beiscsi_hba *phba)
3210 {
3211         struct be_ctrl_info *ctrl = &phba->ctrl;
3212
3213         u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
3214         u32 reg = ioread32(addr);
3215
3216         u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3217         if (enabled) {
3218                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3219                 iowrite32(reg, addr);
3220         } else
3221                 shost_printk(KERN_WARNING, phba->shost,
3222                              "In hwi_disable_intr, Already Disabled \n");
3223 }
3224
3225 static int beiscsi_init_port(struct beiscsi_hba *phba)
3226 {
3227         int ret;
3228
3229         ret = beiscsi_init_controller(phba);
3230         if (ret < 0) {
3231                 shost_printk(KERN_ERR, phba->shost,
3232                              "beiscsi_dev_probe - Failed in"
3233                              "beiscsi_init_controller \n");
3234                 return ret;
3235         }
3236         ret = beiscsi_init_sgl_handle(phba);
3237         if (ret < 0) {
3238                 shost_printk(KERN_ERR, phba->shost,
3239                              "beiscsi_dev_probe - Failed in"
3240                              "beiscsi_init_sgl_handle \n");
3241                 goto do_cleanup_ctrlr;
3242         }
3243
3244         if (hba_setup_cid_tbls(phba)) {
3245                 shost_printk(KERN_ERR, phba->shost,
3246                              "Failed in hba_setup_cid_tbls\n");
3247                 kfree(phba->io_sgl_hndl_base);
3248                 kfree(phba->eh_sgl_hndl_base);
3249                 goto do_cleanup_ctrlr;
3250         }
3251
3252         return ret;
3253
3254 do_cleanup_ctrlr:
3255         hwi_cleanup(phba);
3256         return ret;
3257 }
3258
3259 static void hwi_purge_eq(struct beiscsi_hba *phba)
3260 {
3261         struct hwi_controller *phwi_ctrlr;
3262         struct hwi_context_memory *phwi_context;
3263         struct be_queue_info *eq;
3264         struct be_eq_entry *eqe = NULL;
3265         int i, eq_msix;
3266         unsigned int num_processed;
3267
3268         phwi_ctrlr = phba->phwi_ctrlr;
3269         phwi_context = phwi_ctrlr->phwi_ctxt;
3270         if (phba->msix_enabled)
3271                 eq_msix = 1;
3272         else
3273                 eq_msix = 0;
3274
3275         for (i = 0; i < (phba->num_cpus + eq_msix); i++) {
3276                 eq = &phwi_context->be_eq[i].q;
3277                 eqe = queue_tail_node(eq);
3278                 num_processed = 0;
3279                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
3280                                         & EQE_VALID_MASK) {
3281                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
3282                         queue_tail_inc(eq);
3283                         eqe = queue_tail_node(eq);
3284                         num_processed++;
3285                 }
3286
3287                 if (num_processed)
3288                         hwi_ring_eq_db(phba, eq->id, 1, num_processed, 1, 1);
3289         }
3290 }
3291
3292 static void beiscsi_clean_port(struct beiscsi_hba *phba)
3293 {
3294         unsigned char mgmt_status;
3295
3296         mgmt_status = mgmt_epfw_cleanup(phba, CMD_CONNECTION_CHUTE_0);
3297         if (mgmt_status)
3298                 shost_printk(KERN_WARNING, phba->shost,
3299                              "mgmt_epfw_cleanup FAILED \n");
3300
3301         hwi_purge_eq(phba);
3302         hwi_cleanup(phba);
3303         kfree(phba->io_sgl_hndl_base);
3304         kfree(phba->eh_sgl_hndl_base);
3305         kfree(phba->cid_array);
3306         kfree(phba->ep_array);
3307 }
3308
3309 void
3310 beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
3311                            struct beiscsi_offload_params *params)
3312 {
3313         struct wrb_handle *pwrb_handle;
3314         struct iscsi_target_context_update_wrb *pwrb = NULL;
3315         struct be_mem_descriptor *mem_descr;
3316         struct beiscsi_hba *phba = beiscsi_conn->phba;
3317         u32 doorbell = 0;
3318
3319         /*
3320          * We can always use 0 here because it is reserved by libiscsi for
3321          * login/startup related tasks.
3322          */
3323         pwrb_handle = alloc_wrb_handle(phba, (beiscsi_conn->beiscsi_conn_cid -
3324                                        phba->fw_config.iscsi_cid_start));
3325         pwrb = (struct iscsi_target_context_update_wrb *)pwrb_handle->pwrb;
3326         memset(pwrb, 0, sizeof(*pwrb));
3327         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3328                       max_burst_length, pwrb, params->dw[offsetof
3329                       (struct amap_beiscsi_offload_params,
3330                       max_burst_length) / 32]);
3331         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3332                       max_send_data_segment_length, pwrb,
3333                       params->dw[offsetof(struct amap_beiscsi_offload_params,
3334                       max_send_data_segment_length) / 32]);
3335         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3336                       first_burst_length,
3337                       pwrb,
3338                       params->dw[offsetof(struct amap_beiscsi_offload_params,
3339                       first_burst_length) / 32]);
3340
3341         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, erl, pwrb,
3342                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3343                       erl) / 32] & OFFLD_PARAMS_ERL));
3344         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, dde, pwrb,
3345                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3346                       dde) / 32] & OFFLD_PARAMS_DDE) >> 2);
3347         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, hde, pwrb,
3348                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3349                       hde) / 32] & OFFLD_PARAMS_HDE) >> 3);
3350         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ir2t, pwrb,
3351                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3352                       ir2t) / 32] & OFFLD_PARAMS_IR2T) >> 4);
3353         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, imd, pwrb,
3354                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3355                        imd) / 32] & OFFLD_PARAMS_IMD) >> 5);
3356         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, stat_sn,
3357                       pwrb,
3358                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3359                       exp_statsn) / 32] + 1));
3360         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, type, pwrb,
3361                       0x7);
3362         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, wrb_idx,
3363                       pwrb, pwrb_handle->wrb_index);
3364         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ptr2nextwrb,
3365                       pwrb, pwrb_handle->nxt_wrb_index);
3366         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3367                         session_state, pwrb, 0);
3368         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, compltonack,
3369                       pwrb, 1);
3370         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, notpredblq,
3371                       pwrb, 0);
3372         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, mode, pwrb,
3373                       0);
3374
3375         mem_descr = phba->init_mem;
3376         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
3377
3378         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3379                         pad_buffer_addr_hi, pwrb,
3380                       mem_descr->mem_array[0].bus_address.u.a32.address_hi);
3381         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3382                         pad_buffer_addr_lo, pwrb,
3383                       mem_descr->mem_array[0].bus_address.u.a32.address_lo);
3384
3385         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_target_context_update_wrb));
3386
3387         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
3388         doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)
3389                              << DB_DEF_PDU_WRB_INDEX_SHIFT;
3390         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3391
3392         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3393 }
3394
3395 static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
3396                               int *index, int *age)
3397 {
3398         *index = (int)itt;
3399         if (age)
3400                 *age = conn->session->age;
3401 }
3402
3403 /**
3404  * beiscsi_alloc_pdu - allocates pdu and related resources
3405  * @task: libiscsi task
3406  * @opcode: opcode of pdu for task
3407  *
3408  * This is called with the session lock held. It will allocate
3409  * the wrb and sgl if needed for the command. And it will prep
3410  * the pdu's itt. beiscsi_parse_pdu will later translate
3411  * the pdu itt to the libiscsi task itt.
3412  */
3413 static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
3414 {
3415         struct beiscsi_io_task *io_task = task->dd_data;
3416         struct iscsi_conn *conn = task->conn;
3417         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3418         struct beiscsi_hba *phba = beiscsi_conn->phba;
3419         struct hwi_wrb_context *pwrb_context;
3420         struct hwi_controller *phwi_ctrlr;
3421         itt_t itt;
3422         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
3423         dma_addr_t paddr;
3424
3425         io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool,
3426                                           GFP_KERNEL, &paddr);
3427         if (!io_task->cmd_bhs)
3428                 return -ENOMEM;
3429         io_task->bhs_pa.u.a64.address = paddr;
3430         io_task->libiscsi_itt = (itt_t)task->itt;
3431         io_task->pwrb_handle = alloc_wrb_handle(phba,
3432                                                 beiscsi_conn->beiscsi_conn_cid -
3433                                                 phba->fw_config.iscsi_cid_start
3434                                                 );
3435         io_task->conn = beiscsi_conn;
3436
3437         task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
3438         task->hdr_max = sizeof(struct be_cmd_bhs);
3439
3440         if (task->sc) {
3441                 spin_lock(&phba->io_sgl_lock);
3442                 io_task->psgl_handle = alloc_io_sgl_handle(phba);
3443                 spin_unlock(&phba->io_sgl_lock);
3444                 if (!io_task->psgl_handle)
3445                         goto free_hndls;
3446         } else {
3447                 io_task->scsi_cmnd = NULL;
3448                 if ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
3449                         if (!beiscsi_conn->login_in_progress) {
3450                                 spin_lock(&phba->mgmt_sgl_lock);
3451                                 io_task->psgl_handle = (struct sgl_handle *)
3452                                                 alloc_mgmt_sgl_handle(phba);
3453                                 spin_unlock(&phba->mgmt_sgl_lock);
3454                                 if (!io_task->psgl_handle)
3455                                         goto free_hndls;
3456
3457                                 beiscsi_conn->login_in_progress = 1;
3458                                 beiscsi_conn->plogin_sgl_handle =
3459                                                         io_task->psgl_handle;
3460                         } else {
3461                                 io_task->psgl_handle =
3462                                                 beiscsi_conn->plogin_sgl_handle;
3463                         }
3464                 } else {
3465                         spin_lock(&phba->mgmt_sgl_lock);
3466                         io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);
3467                         spin_unlock(&phba->mgmt_sgl_lock);
3468                         if (!io_task->psgl_handle)
3469                                 goto free_hndls;
3470                 }
3471         }
3472         itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->
3473                                  wrb_index << 16) | (unsigned int)
3474                                 (io_task->psgl_handle->sgl_index));
3475         io_task->pwrb_handle->pio_handle = task;
3476
3477         io_task->cmd_bhs->iscsi_hdr.itt = itt;
3478         return 0;
3479
3480 free_hndls:
3481         phwi_ctrlr = phba->phwi_ctrlr;
3482         pwrb_context = &phwi_ctrlr->wrb_context[
3483                         beiscsi_conn->beiscsi_conn_cid -
3484                         phba->fw_config.iscsi_cid_start];
3485         free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
3486         io_task->pwrb_handle = NULL;
3487         pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3488                       io_task->bhs_pa.u.a64.address);
3489         SE_DEBUG(DBG_LVL_1, "Alloc of SGL_ICD Failed \n");
3490         return -ENOMEM;
3491 }
3492
3493 static void beiscsi_cleanup_task(struct iscsi_task *task)
3494 {
3495         struct beiscsi_io_task *io_task = task->dd_data;
3496         struct iscsi_conn *conn = task->conn;
3497         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3498         struct beiscsi_hba *phba = beiscsi_conn->phba;
3499         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
3500         struct hwi_wrb_context *pwrb_context;
3501         struct hwi_controller *phwi_ctrlr;
3502
3503         phwi_ctrlr = phba->phwi_ctrlr;
3504         pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid
3505                         - phba->fw_config.iscsi_cid_start];
3506         if (io_task->pwrb_handle) {
3507                 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
3508                 io_task->pwrb_handle = NULL;
3509         }
3510
3511         if (io_task->cmd_bhs) {
3512                 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3513                               io_task->bhs_pa.u.a64.address);
3514         }
3515
3516         if (task->sc) {
3517                 if (io_task->psgl_handle) {
3518                         spin_lock(&phba->io_sgl_lock);
3519                         free_io_sgl_handle(phba, io_task->psgl_handle);
3520                         spin_unlock(&phba->io_sgl_lock);
3521                         io_task->psgl_handle = NULL;
3522                 }
3523         } else {
3524                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN)
3525                         return;
3526                 if (io_task->psgl_handle) {
3527                         spin_lock(&phba->mgmt_sgl_lock);
3528                         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
3529                         spin_unlock(&phba->mgmt_sgl_lock);
3530                         io_task->psgl_handle = NULL;
3531                 }
3532         }
3533 }
3534
3535 static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
3536                           unsigned int num_sg, unsigned int xferlen,
3537                           unsigned int writedir)
3538 {
3539
3540         struct beiscsi_io_task *io_task = task->dd_data;
3541         struct iscsi_conn *conn = task->conn;
3542         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3543         struct beiscsi_hba *phba = beiscsi_conn->phba;
3544         struct iscsi_wrb *pwrb = NULL;
3545         unsigned int doorbell = 0;
3546
3547         pwrb = io_task->pwrb_handle->pwrb;
3548         io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
3549         io_task->bhs_len = sizeof(struct be_cmd_bhs);
3550
3551         if (writedir) {
3552                 memset(&io_task->cmd_bhs->iscsi_data_pdu, 0, 48);
3553                 AMAP_SET_BITS(struct amap_pdu_data_out, itt,
3554                               &io_task->cmd_bhs->iscsi_data_pdu,
3555                               (unsigned int)io_task->cmd_bhs->iscsi_hdr.itt);
3556                 AMAP_SET_BITS(struct amap_pdu_data_out, opcode,
3557                               &io_task->cmd_bhs->iscsi_data_pdu,
3558                               ISCSI_OPCODE_SCSI_DATA_OUT);
3559                 AMAP_SET_BITS(struct amap_pdu_data_out, final_bit,
3560                               &io_task->cmd_bhs->iscsi_data_pdu, 1);
3561                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3562                               INI_WR_CMD);
3563                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
3564         } else {
3565                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3566                               INI_RD_CMD);
3567                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
3568         }
3569         memcpy(&io_task->cmd_bhs->iscsi_data_pdu.
3570                dw[offsetof(struct amap_pdu_data_out, lun) / 32],
3571                io_task->cmd_bhs->iscsi_hdr.lun, sizeof(struct scsi_lun));
3572
3573         AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
3574                       cpu_to_be16((unsigned short)io_task->cmd_bhs->iscsi_hdr.
3575                                   lun[0]));
3576         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen);
3577         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
3578                       io_task->pwrb_handle->wrb_index);
3579         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
3580                       be32_to_cpu(task->cmdsn));
3581         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
3582                       io_task->psgl_handle->sgl_index);
3583
3584         hwi_write_sgl(pwrb, sg, num_sg, io_task);
3585
3586         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
3587                       io_task->pwrb_handle->nxt_wrb_index);
3588         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
3589
3590         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
3591         doorbell |= (io_task->pwrb_handle->wrb_index &
3592                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
3593         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3594
3595         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3596         return 0;
3597 }
3598
3599 static int beiscsi_mtask(struct iscsi_task *task)
3600 {
3601         struct beiscsi_io_task *io_task = task->dd_data;
3602         struct iscsi_conn *conn = task->conn;
3603         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3604         struct beiscsi_hba *phba = beiscsi_conn->phba;
3605         struct iscsi_wrb *pwrb = NULL;
3606         unsigned int doorbell = 0;
3607         unsigned int cid;
3608
3609         cid = beiscsi_conn->beiscsi_conn_cid;
3610         pwrb = io_task->pwrb_handle->pwrb;
3611         memset(pwrb, 0, sizeof(*pwrb));
3612         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
3613                       be32_to_cpu(task->cmdsn));
3614         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
3615                       io_task->pwrb_handle->wrb_index);
3616         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
3617                       io_task->psgl_handle->sgl_index);
3618
3619         switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
3620         case ISCSI_OP_LOGIN:
3621                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3622                               TGT_DM_CMD);
3623                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3624                 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
3625                 hwi_write_buffer(pwrb, task);
3626                 break;
3627         case ISCSI_OP_NOOP_OUT:
3628                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3629                               INI_RD_CMD);
3630                 if (task->hdr->ttt == ISCSI_RESERVED_TAG)
3631                         AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3632                 else
3633                         AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 1);
3634                 hwi_write_buffer(pwrb, task);
3635                 break;
3636         case ISCSI_OP_TEXT:
3637                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3638                               TGT_DM_CMD);
3639                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3640                 hwi_write_buffer(pwrb, task);
3641                 break;
3642         case ISCSI_OP_SCSI_TMFUNC:
3643                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3644                               INI_TMF_CMD);
3645                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3646                 hwi_write_buffer(pwrb, task);
3647                 break;
3648         case ISCSI_OP_LOGOUT:
3649                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3650                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3651                               HWH_TYPE_LOGOUT);
3652                 hwi_write_buffer(pwrb, task);
3653                 break;
3654
3655         default:
3656                 SE_DEBUG(DBG_LVL_1, "opcode =%d Not supported \n",
3657                          task->hdr->opcode & ISCSI_OPCODE_MASK);
3658                 return -EINVAL;
3659         }
3660
3661         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb,
3662                       task->data_count);
3663         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
3664                       io_task->pwrb_handle->nxt_wrb_index);
3665         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
3666
3667         doorbell |= cid & DB_WRB_POST_CID_MASK;
3668         doorbell |= (io_task->pwrb_handle->wrb_index &
3669                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
3670         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3671         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3672         return 0;
3673 }
3674
3675 static int beiscsi_task_xmit(struct iscsi_task *task)
3676 {
3677         struct beiscsi_io_task *io_task = task->dd_data;
3678         struct scsi_cmnd *sc = task->sc;
3679         struct scatterlist *sg;
3680         int num_sg;
3681         unsigned int  writedir = 0, xferlen = 0;
3682
3683         if (!sc)
3684                 return beiscsi_mtask(task);
3685
3686         io_task->scsi_cmnd = sc;
3687         num_sg = scsi_dma_map(sc);
3688         if (num_sg < 0) {
3689                 SE_DEBUG(DBG_LVL_1, " scsi_dma_map Failed\n")
3690                 return num_sg;
3691         }
3692         SE_DEBUG(DBG_LVL_4, "xferlen=0x%08x scmd=%p num_sg=%d sernum=%lu\n",
3693                   (scsi_bufflen(sc)), sc, num_sg, sc->serial_number);
3694         xferlen = scsi_bufflen(sc);
3695         sg = scsi_sglist(sc);
3696         if (sc->sc_data_direction == DMA_TO_DEVICE) {
3697                 writedir = 1;
3698                 SE_DEBUG(DBG_LVL_4, "task->imm_count=0x%08x \n",
3699                          task->imm_count);
3700         } else
3701                 writedir = 0;
3702         return beiscsi_iotask(task, sg, num_sg, xferlen, writedir);
3703 }
3704
3705 static void beiscsi_remove(struct pci_dev *pcidev)
3706 {
3707         struct beiscsi_hba *phba = NULL;
3708         struct hwi_controller *phwi_ctrlr;
3709         struct hwi_context_memory *phwi_context;
3710         struct be_eq_obj *pbe_eq;
3711         unsigned int i, msix_vec;
3712
3713         phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev);
3714         if (!phba) {
3715                 dev_err(&pcidev->dev, "beiscsi_remove called with no phba \n");
3716                 return;
3717         }
3718
3719         phwi_ctrlr = phba->phwi_ctrlr;
3720         phwi_context = phwi_ctrlr->phwi_ctxt;
3721         hwi_disable_intr(phba);
3722         if (phba->msix_enabled) {
3723                 for (i = 0; i <= phba->num_cpus; i++) {
3724                         msix_vec = phba->msix_entries[i].vector;
3725                         free_irq(msix_vec, &phwi_context->be_eq[i]);
3726                 }
3727         } else
3728                 if (phba->pcidev->irq)
3729                         free_irq(phba->pcidev->irq, phba);
3730         pci_disable_msix(phba->pcidev);
3731         destroy_workqueue(phba->wq);
3732         if (blk_iopoll_enabled)
3733                 for (i = 0; i < phba->num_cpus; i++) {
3734                         pbe_eq = &phwi_context->be_eq[i];
3735                         blk_iopoll_disable(&pbe_eq->iopoll);
3736                 }
3737
3738         beiscsi_clean_port(phba);
3739         beiscsi_free_mem(phba);
3740         beiscsi_unmap_pci_function(phba);
3741         pci_free_consistent(phba->pcidev,
3742                             phba->ctrl.mbox_mem_alloced.size,
3743                             phba->ctrl.mbox_mem_alloced.va,
3744                             phba->ctrl.mbox_mem_alloced.dma);
3745         iscsi_host_remove(phba->shost);
3746         pci_dev_put(phba->pcidev);
3747         iscsi_host_free(phba->shost);
3748 }
3749
3750 static void beiscsi_msix_enable(struct beiscsi_hba *phba)
3751 {
3752         int i, status;
3753
3754         for (i = 0; i <= phba->num_cpus; i++)
3755                 phba->msix_entries[i].entry = i;
3756
3757         status = pci_enable_msix(phba->pcidev, phba->msix_entries,
3758                                  (phba->num_cpus + 1));
3759         if (!status)
3760                 phba->msix_enabled = true;
3761
3762         return;
3763 }
3764
3765 static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
3766                                 const struct pci_device_id *id)
3767 {
3768         struct beiscsi_hba *phba = NULL;
3769         struct hwi_controller *phwi_ctrlr;
3770         struct hwi_context_memory *phwi_context;
3771         struct be_eq_obj *pbe_eq;
3772         int ret, msix_vec, num_cpus, i;
3773
3774         ret = beiscsi_enable_pci(pcidev);
3775         if (ret < 0) {
3776                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3777                              "Failed to enable pci device \n");
3778                 return ret;
3779         }
3780
3781         phba = beiscsi_hba_alloc(pcidev);
3782         if (!phba) {
3783                 dev_err(&pcidev->dev, "beiscsi_dev_probe-"
3784                         " Failed in beiscsi_hba_alloc \n");
3785                 goto disable_pci;
3786         }
3787
3788         switch (pcidev->device) {
3789         case BE_DEVICE_ID1:
3790         case OC_DEVICE_ID1:
3791         case OC_DEVICE_ID2:
3792                 phba->generation = BE_GEN2;
3793                 break;
3794         case BE_DEVICE_ID2:
3795         case OC_DEVICE_ID3:
3796                 phba->generation = BE_GEN3;
3797                 break;
3798         default:
3799                 phba->generation = 0;
3800         }
3801
3802         if (enable_msix)
3803                 num_cpus = find_num_cpus();
3804         else
3805                 num_cpus = 1;
3806         phba->num_cpus = num_cpus;
3807         SE_DEBUG(DBG_LVL_8, "num_cpus = %d \n", phba->num_cpus);
3808
3809         if (enable_msix)
3810                 beiscsi_msix_enable(phba);
3811         ret = be_ctrl_init(phba, pcidev);
3812         if (ret) {
3813                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3814                                 "Failed in be_ctrl_init\n");
3815                 goto hba_free;
3816         }
3817
3818         spin_lock_init(&phba->io_sgl_lock);
3819         spin_lock_init(&phba->mgmt_sgl_lock);
3820         spin_lock_init(&phba->isr_lock);
3821         ret = mgmt_get_fw_config(&phba->ctrl, phba);
3822         if (ret != 0) {
3823                 shost_printk(KERN_ERR, phba->shost,
3824                              "Error getting fw config\n");
3825                 goto free_port;
3826         }
3827         phba->shost->max_id = phba->fw_config.iscsi_cid_count;
3828         beiscsi_get_params(phba);
3829         phba->shost->can_queue = phba->params.ios_per_ctrl;
3830         ret = beiscsi_init_port(phba);
3831         if (ret < 0) {
3832                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3833                              "Failed in beiscsi_init_port\n");
3834                 goto free_port;
3835         }
3836
3837         for (i = 0; i < MAX_MCC_CMD ; i++) {
3838                 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
3839                 phba->ctrl.mcc_tag[i] = i + 1;
3840                 phba->ctrl.mcc_numtag[i + 1] = 0;
3841                 phba->ctrl.mcc_tag_available++;
3842         }
3843
3844         phba->ctrl.mcc_alloc_index = phba->ctrl.mcc_free_index = 0;
3845
3846         snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_q_irq%u",
3847                  phba->shost->host_no);
3848         phba->wq = create_workqueue(phba->wq_name);
3849         if (!phba->wq) {
3850                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3851                                 "Failed to allocate work queue\n");
3852                 goto free_twq;
3853         }
3854
3855         INIT_WORK(&phba->work_cqs, beiscsi_process_all_cqs);
3856
3857         phwi_ctrlr = phba->phwi_ctrlr;
3858         phwi_context = phwi_ctrlr->phwi_ctxt;
3859         if (blk_iopoll_enabled) {
3860                 for (i = 0; i < phba->num_cpus; i++) {
3861                         pbe_eq = &phwi_context->be_eq[i];
3862                         blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
3863                                         be_iopoll);
3864                         blk_iopoll_enable(&pbe_eq->iopoll);
3865                 }
3866         }
3867         ret = beiscsi_init_irqs(phba);
3868         if (ret < 0) {
3869                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3870                              "Failed to beiscsi_init_irqs\n");
3871                 goto free_blkenbld;
3872         }
3873         ret = hwi_enable_intr(phba);
3874         if (ret < 0) {
3875                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3876                              "Failed to hwi_enable_intr\n");
3877                 goto free_ctrlr;
3878         }
3879         SE_DEBUG(DBG_LVL_8, "\n\n\n SUCCESS - DRIVER LOADED \n\n\n");
3880         return 0;
3881
3882 free_ctrlr:
3883         if (phba->msix_enabled) {
3884                 for (i = 0; i <= phba->num_cpus; i++) {
3885                         msix_vec = phba->msix_entries[i].vector;
3886                         free_irq(msix_vec, &phwi_context->be_eq[i]);
3887                 }
3888         } else
3889                 if (phba->pcidev->irq)
3890                         free_irq(phba->pcidev->irq, phba);
3891         pci_disable_msix(phba->pcidev);
3892 free_blkenbld:
3893         destroy_workqueue(phba->wq);
3894         if (blk_iopoll_enabled)
3895                 for (i = 0; i < phba->num_cpus; i++) {
3896                         pbe_eq = &phwi_context->be_eq[i];
3897                         blk_iopoll_disable(&pbe_eq->iopoll);
3898                 }
3899 free_twq:
3900         beiscsi_clean_port(phba);
3901         beiscsi_free_mem(phba);
3902 free_port:
3903         pci_free_consistent(phba->pcidev,
3904                             phba->ctrl.mbox_mem_alloced.size,
3905                             phba->ctrl.mbox_mem_alloced.va,
3906                            phba->ctrl.mbox_mem_alloced.dma);
3907         beiscsi_unmap_pci_function(phba);
3908 hba_free:
3909         iscsi_host_remove(phba->shost);
3910         pci_dev_put(phba->pcidev);
3911         iscsi_host_free(phba->shost);
3912 disable_pci:
3913         pci_disable_device(pcidev);
3914         return ret;
3915 }
3916
3917 struct iscsi_transport beiscsi_iscsi_transport = {
3918         .owner = THIS_MODULE,
3919         .name = DRV_NAME,
3920         .caps = CAP_RECOVERY_L0 | CAP_HDRDGST | CAP_TEXT_NEGO |
3921                 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD,
3922         .param_mask = ISCSI_MAX_RECV_DLENGTH |
3923                 ISCSI_MAX_XMIT_DLENGTH |
3924                 ISCSI_HDRDGST_EN |
3925                 ISCSI_DATADGST_EN |
3926                 ISCSI_INITIAL_R2T_EN |
3927                 ISCSI_MAX_R2T |
3928                 ISCSI_IMM_DATA_EN |
3929                 ISCSI_FIRST_BURST |
3930                 ISCSI_MAX_BURST |
3931                 ISCSI_PDU_INORDER_EN |
3932                 ISCSI_DATASEQ_INORDER_EN |
3933                 ISCSI_ERL |
3934                 ISCSI_CONN_PORT |
3935                 ISCSI_CONN_ADDRESS |
3936                 ISCSI_EXP_STATSN |
3937                 ISCSI_PERSISTENT_PORT |
3938                 ISCSI_PERSISTENT_ADDRESS |
3939                 ISCSI_TARGET_NAME | ISCSI_TPGT |
3940                 ISCSI_USERNAME | ISCSI_PASSWORD |
3941                 ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN |
3942                 ISCSI_FAST_ABORT | ISCSI_ABORT_TMO |
3943                 ISCSI_LU_RESET_TMO |
3944                 ISCSI_PING_TMO | ISCSI_RECV_TMO |
3945                 ISCSI_IFACE_NAME | ISCSI_INITIATOR_NAME,
3946         .host_param_mask = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS |
3947                                 ISCSI_HOST_INITIATOR_NAME,
3948         .create_session = beiscsi_session_create,
3949         .destroy_session = beiscsi_session_destroy,
3950         .create_conn = beiscsi_conn_create,
3951         .bind_conn = beiscsi_conn_bind,
3952         .destroy_conn = iscsi_conn_teardown,
3953         .set_param = beiscsi_set_param,
3954         .get_conn_param = beiscsi_conn_get_param,
3955         .get_session_param = iscsi_session_get_param,
3956         .get_host_param = beiscsi_get_host_param,
3957         .start_conn = beiscsi_conn_start,
3958         .stop_conn = beiscsi_conn_stop,
3959         .send_pdu = iscsi_conn_send_pdu,
3960         .xmit_task = beiscsi_task_xmit,
3961         .cleanup_task = beiscsi_cleanup_task,
3962         .alloc_pdu = beiscsi_alloc_pdu,
3963         .parse_pdu_itt = beiscsi_parse_pdu,
3964         .get_stats = beiscsi_conn_get_stats,
3965         .ep_connect = beiscsi_ep_connect,
3966         .ep_poll = beiscsi_ep_poll,
3967         .ep_disconnect = beiscsi_ep_disconnect,
3968         .session_recovery_timedout = iscsi_session_recovery_timedout,
3969 };
3970
3971 static struct pci_driver beiscsi_pci_driver = {
3972         .name = DRV_NAME,
3973         .probe = beiscsi_dev_probe,
3974         .remove = beiscsi_remove,
3975         .id_table = beiscsi_pci_id_table
3976 };
3977
3978
3979 static int __init beiscsi_module_init(void)
3980 {
3981         int ret;
3982
3983         beiscsi_scsi_transport =
3984                         iscsi_register_transport(&beiscsi_iscsi_transport);
3985         if (!beiscsi_scsi_transport) {
3986                 SE_DEBUG(DBG_LVL_1,
3987                          "beiscsi_module_init - Unable to  register beiscsi"
3988                          "transport.\n");
3989                 return -ENOMEM;
3990         }
3991         SE_DEBUG(DBG_LVL_8, "In beiscsi_module_init, tt=%p \n",
3992                  &beiscsi_iscsi_transport);
3993
3994         ret = pci_register_driver(&beiscsi_pci_driver);
3995         if (ret) {
3996                 SE_DEBUG(DBG_LVL_1,
3997                          "beiscsi_module_init - Unable to  register"
3998                          "beiscsi pci driver.\n");
3999                 goto unregister_iscsi_transport;
4000         }
4001         return 0;
4002
4003 unregister_iscsi_transport:
4004         iscsi_unregister_transport(&beiscsi_iscsi_transport);
4005         return ret;
4006 }
4007
4008 static void __exit beiscsi_module_exit(void)
4009 {
4010         pci_unregister_driver(&beiscsi_pci_driver);
4011         iscsi_unregister_transport(&beiscsi_iscsi_transport);
4012 }
4013
4014 module_init(beiscsi_module_init);
4015 module_exit(beiscsi_module_exit);