drivers/dma/ioat/dma.c

   1 /*
   2  * Intel I/OAT DMA Linux driver
   3  * Copyright(c) 2004 - 2009 Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  */
  22
  23 /*
  24  * This driver supports an Intel I/OAT DMA engine, which does asynchronous
  25  * copy operations.
  26  */
  27
  28 #include <linux/init.h>
  29 #include <linux/module.h>
  30 #include <linux/pci.h>
  31 #include <linux/interrupt.h>
  32 #include <linux/dmaengine.h>
  33 #include <linux/delay.h>
  34 #include <linux/dma-mapping.h>
  35 #include <linux/workqueue.h>
  36 #include <linux/i7300_idle.h>
  37 #include "dma.h"
  38 #include "registers.h"
  39 #include "hw.h"
  40
  41 int ioat_pending_level = 4;
  42 module_param(ioat_pending_level, int, 0644);
  43 MODULE_PARM_DESC(ioat_pending_level,
  44                  "high-water mark for pushing ioat descriptors (default: 4)");
  45
  46 /* internal functions */
  47 static void ioat1_cleanup(struct ioat_dma_chan *ioat);
  48 static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat);
  49
  50 /**
  51  * ioat_dma_do_interrupt - handler used for single vector interrupt mode
  52  * @irq: interrupt id
  53  * @data: interrupt data
  54  */
  55 static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
  56 {
  57         struct ioatdma_device *instance = data;
  58         struct ioat_chan_common *chan;
  59         unsigned long attnstatus;
  60         int bit;
  61         u8 intrctrl;
  62
  63         intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
  64
  65         if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
  66                 return IRQ_NONE;
  67
  68         if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
  69                 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
  70                 return IRQ_NONE;
  71         }
  72
  73         attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
  74         for_each_bit(bit, &attnstatus, BITS_PER_LONG) {
  75                 chan = ioat_chan_by_index(instance, bit);
  76                 tasklet_schedule(&chan->cleanup_task);
  77         }
  78
  79         writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
  80         return IRQ_HANDLED;
  81 }
  82
  83 /**
  84  * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode
  85  * @irq: interrupt id
  86  * @data: interrupt data
  87  */
  88 static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
  89 {
  90         struct ioat_chan_common *chan = data;
  91
  92         tasklet_schedule(&chan->cleanup_task);
  93
  94         return IRQ_HANDLED;
  95 }
  96
  97 static void ioat1_cleanup_tasklet(unsigned long data);
  98
  99 /* common channel initialization */
 100 void ioat_init_channel(struct ioatdma_device *device,
 101                        struct ioat_chan_common *chan, int idx,
 102                        work_func_t work_fn, void (*tasklet)(unsigned long),
 103                        unsigned long tasklet_data)
 104 {
 105         struct dma_device *dma = &device->common;
 106
 107         chan->device = device;
 108         chan->reg_base = device->reg_base + (0x80 * (idx + 1));
 109         INIT_DELAYED_WORK(&chan->work, work_fn);
 110         spin_lock_init(&chan->cleanup_lock);
 111         chan->common.device = dma;
 112         list_add_tail(&chan->common.device_node, &dma->channels);
 113         device->idx[idx] = chan;
 114         tasklet_init(&chan->cleanup_task, tasklet, tasklet_data);
 115         tasklet_disable(&chan->cleanup_task);
 116 }
 117
 118 static void ioat1_reset_part2(struct work_struct *work);
 119
 120 /**
 121  * ioat1_dma_enumerate_channels - find and initialize the device's channels
 122  * @device: the device to be enumerated
 123  */
 124 static int ioat1_enumerate_channels(struct ioatdma_device *device)
 125 {
 126         u8 xfercap_scale;
 127         u32 xfercap;
 128         int i;
 129         struct ioat_dma_chan *ioat;
 130         struct device *dev = &device->pdev->dev;
 131         struct dma_device *dma = &device->common;
 132
 133         INIT_LIST_HEAD(&dma->channels);
 134         dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
 135         xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
 136         xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
 137         dev_dbg(dev, "%s: xfercap = %d\n", __func__, xfercap);
 138
 139 #ifdef  CONFIG_I7300_IDLE_IOAT_CHANNEL
 140         if (i7300_idle_platform_probe(NULL, NULL, 1) == 0)
 141                 dma->chancnt--;
 142 #endif
 143         for (i = 0; i < dma->chancnt; i++) {
 144                 ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL);
 145                 if (!ioat)
 146                         break;
 147
 148                 ioat_init_channel(device, &ioat->base, i,
 149                                   ioat1_reset_part2,
 150                                   ioat1_cleanup_tasklet,
 151                                   (unsigned long) ioat);
 152                 ioat->xfercap = xfercap;
 153                 spin_lock_init(&ioat->desc_lock);
 154                 INIT_LIST_HEAD(&ioat->free_desc);
 155                 INIT_LIST_HEAD(&ioat->used_desc);
 156         }
 157         dma->chancnt = i;
 158         return i;
 159 }
 160
 161 /**
 162  * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended
 163  *                                 descriptors to hw
 164  * @chan: DMA channel handle
 165  */
 166 static inline void
 167 __ioat1_dma_memcpy_issue_pending(struct ioat_dma_chan *ioat)
 168 {
 169         void __iomem *reg_base = ioat->base.reg_base;
 170
 171         dev_dbg(to_dev(&ioat->base), "%s: pending: %d\n",
 172                 __func__, ioat->pending);
 173         ioat->pending = 0;
 174         writeb(IOAT_CHANCMD_APPEND, reg_base + IOAT1_CHANCMD_OFFSET);
 175 }
 176
 177 static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan)
 178 {
 179         struct ioat_dma_chan *ioat = to_ioat_chan(chan);
 180
 181         if (ioat->pending > 0) {
 182                 spin_lock_bh(&ioat->desc_lock);
 183                 __ioat1_dma_memcpy_issue_pending(ioat);
 184                 spin_unlock_bh(&ioat->desc_lock);
 185         }
 186 }
 187
 188 /**
 189  * ioat1_reset_part2 - reinit the channel after a reset
 190  */
 191 static void ioat1_reset_part2(struct work_struct *work)
 192 {
 193         struct ioat_chan_common *chan;
 194         struct ioat_dma_chan *ioat;
 195         struct ioat_desc_sw *desc;
 196         int dmacount;
 197         bool start_null = false;
 198
 199         chan = container_of(work, struct ioat_chan_common, work.work);
 200         ioat = container_of(chan, struct ioat_dma_chan, base);
 201         spin_lock_bh(&chan->cleanup_lock);
 202         spin_lock_bh(&ioat->desc_lock);
 203
 204         chan->completion_virt->low = 0;
 205         chan->completion_virt->high = 0;
 206         ioat->pending = 0;
 207
 208         /* count the descriptors waiting */
 209         dmacount = 0;
 210         if (ioat->used_desc.prev) {
 211                 desc = to_ioat_desc(ioat->used_desc.prev);
 212                 do {
 213                         dmacount++;
 214                         desc = to_ioat_desc(desc->node.next);
 215                 } while (&desc->node != ioat->used_desc.next);
 216         }
 217
 218         if (dmacount) {
 219                 /*
 220                  * write the new starting descriptor address
 221                  * this puts channel engine into ARMED state
 222                  */
 223                 desc = to_ioat_desc(ioat->used_desc.prev);
 224                 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
 225                        chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
 226                 writel(((u64) desc->txd.phys) >> 32,
 227                        chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
 228
 229                 writeb(IOAT_CHANCMD_START, chan->reg_base
 230                         + IOAT_CHANCMD_OFFSET(chan->device->version));
 231         } else
 232                 start_null = true;
 233         spin_unlock_bh(&ioat->desc_lock);
 234         spin_unlock_bh(&chan->cleanup_lock);
 235
 236         dev_err(to_dev(chan),
 237                 "chan%d reset - %d descs waiting, %d total desc\n",
 238                 chan_num(chan), dmacount, ioat->desccount);
 239
 240         if (start_null)
 241                 ioat1_dma_start_null_desc(ioat);
 242 }
 243
 244 /**
 245  * ioat1_reset_channel - restart a channel
 246  * @ioat: IOAT DMA channel handle
 247  */
 248 static void ioat1_reset_channel(struct ioat_dma_chan *ioat)
 249 {
 250         struct ioat_chan_common *chan = &ioat->base;
 251         void __iomem *reg_base = chan->reg_base;
 252         u32 chansts, chanerr;
 253
 254         if (!ioat->used_desc.prev)
 255                 return;
 256
 257         dev_dbg(to_dev(chan), "%s\n", __func__);
 258         chanerr = readl(reg_base + IOAT_CHANERR_OFFSET);
 259         chansts = (chan->completion_virt->low
 260                                         & IOAT_CHANSTS_DMA_TRANSFER_STATUS);
 261         if (chanerr) {
 262                 dev_err(to_dev(chan),
 263                         "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
 264                         chan_num(chan), chansts, chanerr);
 265                 writel(chanerr, reg_base + IOAT_CHANERR_OFFSET);
 266         }
 267
 268         /*
 269          * whack it upside the head with a reset
 270          * and wait for things to settle out.
 271          * force the pending count to a really big negative
 272          * to make sure no one forces an issue_pending
 273          * while we're waiting.
 274          */
 275
 276         spin_lock_bh(&ioat->desc_lock);
 277         ioat->pending = INT_MIN;
 278         writeb(IOAT_CHANCMD_RESET,
 279                reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
 280         spin_unlock_bh(&ioat->desc_lock);
 281
 282         /* schedule the 2nd half instead of sleeping a long time */
 283         schedule_delayed_work(&chan->work, RESET_DELAY);
 284 }
 285
 286 /**
 287  * ioat1_chan_watchdog - watch for stuck channels
 288  */
 289 static void ioat1_chan_watchdog(struct work_struct *work)
 290 {
 291         struct ioatdma_device *device =
 292                 container_of(work, struct ioatdma_device, work.work);
 293         struct ioat_dma_chan *ioat;
 294         struct ioat_chan_common *chan;
 295         int i;
 296
 297         union {
 298                 u64 full;
 299                 struct {
 300                         u32 low;
 301                         u32 high;
 302                 };
 303         } completion_hw;
 304         unsigned long compl_desc_addr_hw;
 305
 306         for (i = 0; i < device->common.chancnt; i++) {
 307                 chan = ioat_chan_by_index(device, i);
 308                 ioat = container_of(chan, struct ioat_dma_chan, base);
 309
 310                 if (/* have we started processing anything yet */
 311                     chan->last_completion
 312                     /* have we completed any since last watchdog cycle? */
 313                     && (chan->last_completion == chan->watchdog_completion)
 314                     /* has TCP stuck on one cookie since last watchdog? */
 315                     && (chan->watchdog_tcp_cookie == chan->watchdog_last_tcp_cookie)
 316                     && (chan->watchdog_tcp_cookie != chan->completed_cookie)
 317                     /* is there something in the chain to be processed? */
 318                     /* CB1 chain always has at least the last one processed */
 319                     && (ioat->used_desc.prev != ioat->used_desc.next)
 320                     && ioat->pending == 0) {
 321
 322                         /*
 323                          * check CHANSTS register for completed
 324                          * descriptor address.
 325                          * if it is different than completion writeback,
 326                          * it is not zero
 327                          * and it has changed since the last watchdog
 328                          *     we can assume that channel
 329                          *     is still working correctly
 330                          *     and the problem is in completion writeback.
 331                          *     update completion writeback
 332                          *     with actual CHANSTS value
 333                          * else
 334                          *     try resetting the channel
 335                          */
 336
 337                         completion_hw.low = readl(chan->reg_base +
 338                                 IOAT_CHANSTS_OFFSET_LOW(chan->device->version));
 339                         completion_hw.high = readl(chan->reg_base +
 340                                 IOAT_CHANSTS_OFFSET_HIGH(chan->device->version));
 341 #if (BITS_PER_LONG == 64)
 342                         compl_desc_addr_hw =
 343                                 completion_hw.full
 344                                 & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
 345 #else
 346                         compl_desc_addr_hw =
 347                                 completion_hw.low & IOAT_LOW_COMPLETION_MASK;
 348 #endif
 349
 350                         if ((compl_desc_addr_hw != 0)
 351                            && (compl_desc_addr_hw != chan->watchdog_completion)
 352                            && (compl_desc_addr_hw != chan->last_compl_desc_addr_hw)) {
 353                                 chan->last_compl_desc_addr_hw = compl_desc_addr_hw;
 354                                 chan->completion_virt->low = completion_hw.low;
 355                                 chan->completion_virt->high = completion_hw.high;
 356                         } else {
 357                                 ioat1_reset_channel(ioat);
 358                                 chan->watchdog_completion = 0;
 359                                 chan->last_compl_desc_addr_hw = 0;
 360                         }
 361                 } else {
 362                         chan->last_compl_desc_addr_hw = 0;
 363                         chan->watchdog_completion = chan->last_completion;
 364                 }
 365
 366                 chan->watchdog_last_tcp_cookie = chan->watchdog_tcp_cookie;
 367         }
 368
 369         schedule_delayed_work(&device->work, WATCHDOG_DELAY);
 370 }
 371
 372 static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
 373 {
 374         struct dma_chan *c = tx->chan;
 375         struct ioat_dma_chan *ioat = to_ioat_chan(c);
 376         struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
 377         struct ioat_desc_sw *first;
 378         struct ioat_desc_sw *chain_tail;
 379         dma_cookie_t cookie;
 380
 381         spin_lock_bh(&ioat->desc_lock);
 382         /* cookie incr and addition to used_list must be atomic */
 383         cookie = c->cookie;
 384         cookie++;
 385         if (cookie < 0)
 386                 cookie = 1;
 387         c->cookie = cookie;
 388         tx->cookie = cookie;
 389         dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie);
 390
 391         /* write address into NextDescriptor field of last desc in chain */
 392         first = to_ioat_desc(tx->tx_list.next);
 393         chain_tail = to_ioat_desc(ioat->used_desc.prev);
 394         /* make descriptor updates globally visible before chaining */
 395         wmb();
 396         chain_tail->hw->next = first->txd.phys;
 397         list_splice_tail_init(&tx->tx_list, &ioat->used_desc);
 398         dump_desc_dbg(ioat, chain_tail);
 399         dump_desc_dbg(ioat, first);
 400
 401         ioat->pending += desc->tx_cnt;
 402         if (ioat->pending >= ioat_pending_level)
 403                 __ioat1_dma_memcpy_issue_pending(ioat);
 404         spin_unlock_bh(&ioat->desc_lock);
 405
 406         return cookie;
 407 }
 408
 409 /**
 410  * ioat_dma_alloc_descriptor - allocate and return a sw and hw descriptor pair
 411  * @ioat: the channel supplying the memory pool for the descriptors
 412  * @flags: allocation flags
 413  */
 414 static struct ioat_desc_sw *
 415 ioat_dma_alloc_descriptor(struct ioat_dma_chan *ioat, gfp_t flags)
 416 {
 417         struct ioat_dma_descriptor *desc;
 418         struct ioat_desc_sw *desc_sw;
 419         struct ioatdma_device *ioatdma_device;
 420         dma_addr_t phys;
 421
 422         ioatdma_device = ioat->base.device;
 423         desc = pci_pool_alloc(ioatdma_device->dma_pool, flags, &phys);
 424         if (unlikely(!desc))
 425                 return NULL;
 426
 427         desc_sw = kzalloc(sizeof(*desc_sw), flags);
 428         if (unlikely(!desc_sw)) {
 429                 pci_pool_free(ioatdma_device->dma_pool, desc, phys);
 430                 return NULL;
 431         }
 432
 433         memset(desc, 0, sizeof(*desc));
 434
 435         dma_async_tx_descriptor_init(&desc_sw->txd, &ioat->base.common);
 436         desc_sw->txd.tx_submit = ioat1_tx_submit;
 437         desc_sw->hw = desc;
 438         desc_sw->txd.phys = phys;
 439         set_desc_id(desc_sw, -1);
 440
 441         return desc_sw;
 442 }
 443
 444 static int ioat_initial_desc_count = 256;
 445 module_param(ioat_initial_desc_count, int, 0644);
 446 MODULE_PARM_DESC(ioat_initial_desc_count,
 447                  "ioat1: initial descriptors per channel (default: 256)");
 448 /**
 449  * ioat1_dma_alloc_chan_resources - returns the number of allocated descriptors
 450  * @chan: the channel to be filled out
 451  */
 452 static int ioat1_dma_alloc_chan_resources(struct dma_chan *c)
 453 {
 454         struct ioat_dma_chan *ioat = to_ioat_chan(c);
 455         struct ioat_chan_common *chan = &ioat->base;
 456         struct ioat_desc_sw *desc;
 457         u16 chanctrl;
 458         u32 chanerr;
 459         int i;
 460         LIST_HEAD(tmp_list);
 461
 462         /* have we already been set up? */
 463         if (!list_empty(&ioat->free_desc))
 464                 return ioat->desccount;
 465
 466         /* Setup register to interrupt and write completion status on error */
 467         chanctrl = IOAT_CHANCTRL_ERR_INT_EN |
 468                 IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
 469                 IOAT_CHANCTRL_ERR_COMPLETION_EN;
 470         writew(chanctrl, chan->reg_base + IOAT_CHANCTRL_OFFSET);
 471
 472         chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
 473         if (chanerr) {
 474                 dev_err(to_dev(chan), "CHANERR = %x, clearing\n", chanerr);
 475                 writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
 476         }
 477
 478         /* Allocate descriptors */
 479         for (i = 0; i < ioat_initial_desc_count; i++) {
 480                 desc = ioat_dma_alloc_descriptor(ioat, GFP_KERNEL);
 481                 if (!desc) {
 482                         dev_err(to_dev(chan), "Only %d initial descriptors\n", i);
 483                         break;
 484                 }
 485                 set_desc_id(desc, i);
 486                 list_add_tail(&desc->node, &tmp_list);
 487         }
 488         spin_lock_bh(&ioat->desc_lock);
 489         ioat->desccount = i;
 490         list_splice(&tmp_list, &ioat->free_desc);
 491         spin_unlock_bh(&ioat->desc_lock);
 492
 493         /* allocate a completion writeback area */
 494         /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
 495         chan->completion_virt = pci_pool_alloc(chan->device->completion_pool,
 496                                                GFP_KERNEL,
 497                                                &chan->completion_addr);
 498         memset(chan->completion_virt, 0,
 499                sizeof(*chan->completion_virt));
 500         writel(((u64) chan->completion_addr) & 0x00000000FFFFFFFF,
 501                chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
 502         writel(((u64) chan->completion_addr) >> 32,
 503                chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
 504
 505         tasklet_enable(&chan->cleanup_task);
 506         ioat1_dma_start_null_desc(ioat);  /* give chain to dma device */
 507         dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n",
 508                 __func__, ioat->desccount);
 509         return ioat->desccount;
 510 }
 511
 512 /**
 513  * ioat1_dma_free_chan_resources - release all the descriptors
 514  * @chan: the channel to be cleaned
 515  */
 516 static void ioat1_dma_free_chan_resources(struct dma_chan *c)
 517 {
 518         struct ioat_dma_chan *ioat = to_ioat_chan(c);
 519         struct ioat_chan_common *chan = &ioat->base;
 520         struct ioatdma_device *ioatdma_device = chan->device;
 521         struct ioat_desc_sw *desc, *_desc;
 522         int in_use_descs = 0;
 523
 524         /* Before freeing channel resources first check
 525          * if they have been previously allocated for this channel.
 526          */
 527         if (ioat->desccount == 0)
 528                 return;
 529
 530         tasklet_disable(&chan->cleanup_task);
 531         ioat1_cleanup(ioat);
 532
 533         /* Delay 100ms after reset to allow internal DMA logic to quiesce
 534          * before removing DMA descriptor resources.
 535          */
 536         writeb(IOAT_CHANCMD_RESET,
 537                chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
 538         mdelay(100);
 539
 540         spin_lock_bh(&ioat->desc_lock);
 541         list_for_each_entry_safe(desc, _desc, &ioat->used_desc, node) {
 542                 dev_dbg(to_dev(chan), "%s: freeing %d from used list\n",
 543                         __func__, desc_id(desc));
 544                 dump_desc_dbg(ioat, desc);
 545                 in_use_descs++;
 546                 list_del(&desc->node);
 547                 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
 548                               desc->txd.phys);
 549                 kfree(desc);
 550         }
 551         list_for_each_entry_safe(desc, _desc,
 552                                  &ioat->free_desc, node) {
 553                 list_del(&desc->node);
 554                 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
 555                               desc->txd.phys);
 556                 kfree(desc);
 557         }
 558         spin_unlock_bh(&ioat->desc_lock);
 559
 560         pci_pool_free(ioatdma_device->completion_pool,
 561                       chan->completion_virt,
 562                       chan->completion_addr);
 563
 564         /* one is ok since we left it on there on purpose */
 565         if (in_use_descs > 1)
 566                 dev_err(to_dev(chan), "Freeing %d in use descriptors!\n",
 567                         in_use_descs - 1);
 568
 569         chan->last_completion = chan->completion_addr = 0;
 570         chan->watchdog_completion = 0;
 571         chan->last_compl_desc_addr_hw = 0;
 572         chan->watchdog_tcp_cookie = chan->watchdog_last_tcp_cookie = 0;
 573         ioat->pending = 0;
 574         ioat->desccount = 0;
 575 }
 576
 577 /**
 578  * ioat1_dma_get_next_descriptor - return the next available descriptor
 579  * @ioat: IOAT DMA channel handle
 580  *
 581  * Gets the next descriptor from the chain, and must be called with the
 582  * channel's desc_lock held.  Allocates more descriptors if the channel
 583  * has run out.
 584  */
 585 static struct ioat_desc_sw *
 586 ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat)
 587 {
 588         struct ioat_desc_sw *new;
 589
 590         if (!list_empty(&ioat->free_desc)) {
 591                 new = to_ioat_desc(ioat->free_desc.next);
 592                 list_del(&new->node);
 593         } else {
 594                 /* try to get another desc */
 595                 new = ioat_dma_alloc_descriptor(ioat, GFP_ATOMIC);
 596                 if (!new) {
 597                         dev_err(to_dev(&ioat->base), "alloc failed\n");
 598                         return NULL;
 599                 }
 600         }
 601         dev_dbg(to_dev(&ioat->base), "%s: allocated: %d\n",
 602                 __func__, desc_id(new));
 603         prefetch(new->hw);
 604         return new;
 605 }
 606
 607 static struct dma_async_tx_descriptor *
 608 ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
 609                       dma_addr_t dma_src, size_t len, unsigned long flags)
 610 {
 611         struct ioat_dma_chan *ioat = to_ioat_chan(c);
 612         struct ioat_desc_sw *desc;
 613         size_t copy;
 614         LIST_HEAD(chain);
 615         dma_addr_t src = dma_src;
 616         dma_addr_t dest = dma_dest;
 617         size_t total_len = len;
 618         struct ioat_dma_descriptor *hw = NULL;
 619         int tx_cnt = 0;
 620
 621         spin_lock_bh(&ioat->desc_lock);
 622         desc = ioat1_dma_get_next_descriptor(ioat);
 623         do {
 624                 if (!desc)
 625                         break;
 626
 627                 tx_cnt++;
 628                 copy = min_t(size_t, len, ioat->xfercap);
 629
 630                 hw = desc->hw;
 631                 hw->size = copy;
 632                 hw->ctl = 0;
 633                 hw->src_addr = src;
 634                 hw->dst_addr = dest;
 635
 636                 list_add_tail(&desc->node, &chain);
 637
 638                 len -= copy;
 639                 dest += copy;
 640                 src += copy;
 641                 if (len) {
 642                         struct ioat_desc_sw *next;
 643
 644                         async_tx_ack(&desc->txd);
 645                         next = ioat1_dma_get_next_descriptor(ioat);
 646                         hw->next = next ? next->txd.phys : 0;
 647                         dump_desc_dbg(ioat, desc);
 648                         desc = next;
 649                 } else
 650                         hw->next = 0;
 651         } while (len);
 652
 653         if (!desc) {
 654                 struct ioat_chan_common *chan = &ioat->base;
 655
 656                 dev_err(to_dev(chan),
 657                         "chan%d - get_next_desc failed\n", chan_num(chan));
 658                 list_splice(&chain, &ioat->free_desc);
 659                 spin_unlock_bh(&ioat->desc_lock);
 660                 return NULL;
 661         }
 662         spin_unlock_bh(&ioat->desc_lock);
 663
 664         desc->txd.flags = flags;
 665         desc->tx_cnt = tx_cnt;
 666         desc->len = total_len;
 667         list_splice(&chain, &desc->txd.tx_list);
 668         hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
 669         hw->ctl_f.compl_write = 1;
 670         dump_desc_dbg(ioat, desc);
 671
 672         return &desc->txd;
 673 }
 674
 675 static void ioat1_cleanup_tasklet(unsigned long data)
 676 {
 677         struct ioat_dma_chan *chan = (void *)data;
 678         ioat1_cleanup(chan);
 679         writew(IOAT_CHANCTRL_INT_DISABLE,
 680                chan->base.reg_base + IOAT_CHANCTRL_OFFSET);
 681 }
 682
 683 static void ioat_unmap(struct pci_dev *pdev, dma_addr_t addr, size_t len,
 684                        int direction, enum dma_ctrl_flags flags, bool dst)
 685 {
 686         if ((dst && (flags & DMA_COMPL_DEST_UNMAP_SINGLE)) ||
 687             (!dst && (flags & DMA_COMPL_SRC_UNMAP_SINGLE)))
 688                 pci_unmap_single(pdev, addr, len, direction);
 689         else
 690                 pci_unmap_page(pdev, addr, len, direction);
 691 }
 692
 693
 694 void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
 695                     size_t len, struct ioat_dma_descriptor *hw)
 696 {
 697         struct pci_dev *pdev = chan->device->pdev;
 698         size_t offset = len - hw->size;
 699
 700         if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP))
 701                 ioat_unmap(pdev, hw->dst_addr - offset, len,
 702                            PCI_DMA_FROMDEVICE, flags, 1);
 703
 704         if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP))
 705                 ioat_unmap(pdev, hw->src_addr - offset, len,
 706                            PCI_DMA_TODEVICE, flags, 0);
 707 }
 708
 709 unsigned long ioat_get_current_completion(struct ioat_chan_common *chan)
 710 {
 711         unsigned long phys_complete;
 712
 713         /* The completion writeback can happen at any time,
 714            so reads by the driver need to be atomic operations
 715            The descriptor physical addresses are limited to 32-bits
 716            when the CPU can only do a 32-bit mov */
 717
 718 #if (BITS_PER_LONG == 64)
 719         phys_complete =
 720                 chan->completion_virt->full
 721                 & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
 722 #else
 723         phys_complete = chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK;
 724 #endif
 725
 726         dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__,
 727                 (unsigned long long) phys_complete);
 728
 729         if ((chan->completion_virt->full
 730                 & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
 731                                 IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
 732                 dev_err(to_dev(chan), "Channel halted, chanerr = %x\n",
 733                         readl(chan->reg_base + IOAT_CHANERR_OFFSET));
 734
 735                 /* TODO do something to salvage the situation */
 736         }
 737
 738         return phys_complete;
 739 }
 740
 741 /**
 742  * ioat1_cleanup - cleanup up finished descriptors
 743  * @chan: ioat channel to be cleaned up
 744  */
 745 static void ioat1_cleanup(struct ioat_dma_chan *ioat)
 746 {
 747         struct ioat_chan_common *chan = &ioat->base;
 748         unsigned long phys_complete;
 749         struct ioat_desc_sw *desc, *_desc;
 750         dma_cookie_t cookie = 0;
 751         struct dma_async_tx_descriptor *tx;
 752
 753         prefetch(chan->completion_virt);
 754
 755         if (!spin_trylock_bh(&chan->cleanup_lock))
 756                 return;
 757
 758         phys_complete = ioat_get_current_completion(chan);
 759         if (phys_complete == chan->last_completion) {
 760                 spin_unlock_bh(&chan->cleanup_lock);
 761                 /*
 762                  * perhaps we're stuck so hard that the watchdog can't go off?
 763                  * try to catch it after 2 seconds
 764                  */
 765                 if (time_after(jiffies,
 766                                chan->last_completion_time + HZ*WATCHDOG_DELAY)) {
 767                         ioat1_chan_watchdog(&(chan->device->work.work));
 768                         chan->last_completion_time = jiffies;
 769                 }
 770                 return;
 771         }
 772         chan->last_completion_time = jiffies;
 773
 774         cookie = 0;
 775         if (!spin_trylock_bh(&ioat->desc_lock)) {
 776                 spin_unlock_bh(&chan->cleanup_lock);
 777                 return;
 778         }
 779
 780         dev_dbg(to_dev(chan), "%s: phys_complete: %lx\n",
 781                  __func__, phys_complete);
 782         list_for_each_entry_safe(desc, _desc, &ioat->used_desc, node) {
 783                 tx = &desc->txd;
 784                 /*
 785                  * Incoming DMA requests may use multiple descriptors,
 786                  * due to exceeding xfercap, perhaps. If so, only the
 787                  * last one will have a cookie, and require unmapping.
 788                  */
 789                 dump_desc_dbg(ioat, desc);
 790                 if (tx->cookie) {
 791                         cookie = tx->cookie;
 792                         ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
 793                         if (tx->callback) {
 794                                 tx->callback(tx->callback_param);
 795                                 tx->callback = NULL;
 796                         }
 797                 }
 798
 799                 if (tx->phys != phys_complete) {
 800                         /*
 801                          * a completed entry, but not the last, so clean
 802                          * up if the client is done with the descriptor
 803                          */
 804                         if (async_tx_test_ack(tx))
 805                                 list_move_tail(&desc->node, &ioat->free_desc);
 806                         else
 807                                 tx->cookie = 0;
 808                 } else {
 809                         /*
 810                          * last used desc. Do not remove, so we can
 811                          * append from it, but don't look at it next
 812                          * time, either
 813                          */
 814                         tx->cookie = 0;
 815
 816                         /* TODO check status bits? */
 817                         break;
 818                 }
 819         }
 820
 821         spin_unlock_bh(&ioat->desc_lock);
 822
 823         chan->last_completion = phys_complete;
 824         if (cookie != 0)
 825                 chan->completed_cookie = cookie;
 826
 827         spin_unlock_bh(&chan->cleanup_lock);
 828 }
 829
 830 static enum dma_status
 831 ioat1_dma_is_complete(struct dma_chan *c, dma_cookie_t cookie,
 832                       dma_cookie_t *done, dma_cookie_t *used)
 833 {
 834         struct ioat_dma_chan *ioat = to_ioat_chan(c);
 835
 836         if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS)
 837                 return DMA_SUCCESS;
 838
 839         ioat1_cleanup(ioat);
 840
 841         return ioat_is_complete(c, cookie, done, used);
 842 }
 843
 844 static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat)
 845 {
 846         struct ioat_chan_common *chan = &ioat->base;
 847         struct ioat_desc_sw *desc;
 848         struct ioat_dma_descriptor *hw;
 849
 850         spin_lock_bh(&ioat->desc_lock);
 851
 852         desc = ioat1_dma_get_next_descriptor(ioat);
 853
 854         if (!desc) {
 855                 dev_err(to_dev(chan),
 856                         "Unable to start null desc - get next desc failed\n");
 857                 spin_unlock_bh(&ioat->desc_lock);
 858                 return;
 859         }
 860
 861         hw = desc->hw;
 862         hw->ctl = 0;
 863         hw->ctl_f.null = 1;
 864         hw->ctl_f.int_en = 1;
 865         hw->ctl_f.compl_write = 1;
 866         /* set size to non-zero value (channel returns error when size is 0) */
 867         hw->size = NULL_DESC_BUFFER_SIZE;
 868         hw->src_addr = 0;
 869         hw->dst_addr = 0;
 870         async_tx_ack(&desc->txd);
 871         hw->next = 0;
 872         list_add_tail(&desc->node, &ioat->used_desc);
 873         dump_desc_dbg(ioat, desc);
 874
 875         writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
 876                chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
 877         writel(((u64) desc->txd.phys) >> 32,
 878                chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
 879
 880         writeb(IOAT_CHANCMD_START, chan->reg_base
 881                 + IOAT_CHANCMD_OFFSET(chan->device->version));
 882         spin_unlock_bh(&ioat->desc_lock);
 883 }
 884
 885 /*
 886  * Perform a IOAT transaction to verify the HW works.
 887  */
 888 #define IOAT_TEST_SIZE 2000
 889
 890 static void ioat_dma_test_callback(void *dma_async_param)
 891 {
 892         struct completion *cmp = dma_async_param;
 893
 894         complete(cmp);
 895 }
 896
 897 /**
 898  * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works.
 899  * @device: device to be tested
 900  */
 901 static int ioat_dma_self_test(struct ioatdma_device *device)
 902 {
 903         int i;
 904         u8 *src;
 905         u8 *dest;
 906         struct dma_device *dma = &device->common;
 907         struct device *dev = &device->pdev->dev;
 908         struct dma_chan *dma_chan;
 909         struct dma_async_tx_descriptor *tx;
 910         dma_addr_t dma_dest, dma_src;
 911         dma_cookie_t cookie;
 912         int err = 0;
 913         struct completion cmp;
 914         unsigned long tmo;
 915         unsigned long flags;
 916
 917         src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
 918         if (!src)
 919                 return -ENOMEM;
 920         dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
 921         if (!dest) {
 922                 kfree(src);
 923                 return -ENOMEM;
 924         }
 925
 926         /* Fill in src buffer */
 927         for (i = 0; i < IOAT_TEST_SIZE; i++)
 928                 src[i] = (u8)i;
 929
 930         /* Start copy, using first DMA channel */
 931         dma_chan = container_of(dma->channels.next, struct dma_chan,
 932                                 device_node);
 933         if (dma->device_alloc_chan_resources(dma_chan) < 1) {
 934                 dev_err(dev, "selftest cannot allocate chan resource\n");
 935                 err = -ENODEV;
 936                 goto out;
 937         }
 938
 939         dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE);
 940         dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
 941         flags = DMA_COMPL_SRC_UNMAP_SINGLE | DMA_COMPL_DEST_UNMAP_SINGLE |
 942                 DMA_PREP_INTERRUPT;
 943         tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src,
 944                                                    IOAT_TEST_SIZE, flags);
 945         if (!tx) {
 946                 dev_err(dev, "Self-test prep failed, disabling\n");
 947                 err = -ENODEV;
 948                 goto free_resources;
 949         }
 950
 951         async_tx_ack(tx);
 952         init_completion(&cmp);
 953         tx->callback = ioat_dma_test_callback;
 954         tx->callback_param = &cmp;
 955         cookie = tx->tx_submit(tx);
 956         if (cookie < 0) {
 957                 dev_err(dev, "Self-test setup failed, disabling\n");
 958                 err = -ENODEV;
 959                 goto free_resources;
 960         }
 961         dma->device_issue_pending(dma_chan);
 962
 963         tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
 964
 965         if (tmo == 0 ||
 966             dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL)
 967                                         != DMA_SUCCESS) {
 968                 dev_err(dev, "Self-test copy timed out, disabling\n");
 969                 err = -ENODEV;
 970                 goto free_resources;
 971         }
 972         if (memcmp(src, dest, IOAT_TEST_SIZE)) {
 973                 dev_err(dev, "Self-test copy failed compare, disabling\n");
 974                 err = -ENODEV;
 975                 goto free_resources;
 976         }
 977
 978 free_resources:
 979         dma->device_free_chan_resources(dma_chan);
 980 out:
 981         kfree(src);
 982         kfree(dest);
 983         return err;
 984 }
 985
 986 static char ioat_interrupt_style[32] = "msix";
 987 module_param_string(ioat_interrupt_style, ioat_interrupt_style,
 988                     sizeof(ioat_interrupt_style), 0644);
 989 MODULE_PARM_DESC(ioat_interrupt_style,
 990                  "set ioat interrupt style: msix (default), "
 991                  "msix-single-vector, msi, intx)");
 992
 993 /**
 994  * ioat_dma_setup_interrupts - setup interrupt handler
 995  * @device: ioat device
 996  */
 997 static int ioat_dma_setup_interrupts(struct ioatdma_device *device)
 998 {
 999         struct ioat_chan_common *chan;
1000         struct pci_dev *pdev = device->pdev;
1001         struct device *dev = &pdev->dev;
1002         struct msix_entry *msix;
1003         int i, j, msixcnt;
1004         int err = -EINVAL;
1005         u8 intrctrl = 0;
1006
1007         if (!strcmp(ioat_interrupt_style, "msix"))
1008                 goto msix;
1009         if (!strcmp(ioat_interrupt_style, "msix-single-vector"))
1010                 goto msix_single_vector;
1011         if (!strcmp(ioat_interrupt_style, "msi"))
1012                 goto msi;
1013         if (!strcmp(ioat_interrupt_style, "intx"))
1014                 goto intx;
1015         dev_err(dev, "invalid ioat_interrupt_style %s\n", ioat_interrupt_style);
1016         goto err_no_irq;
1017
1018 msix:
1019         /* The number of MSI-X vectors should equal the number of channels */
1020         msixcnt = device->common.chancnt;
1021         for (i = 0; i < msixcnt; i++)
1022                 device->msix_entries[i].entry = i;
1023
1024         err = pci_enable_msix(pdev, device->msix_entries, msixcnt);
1025         if (err < 0)
1026                 goto msi;
1027         if (err > 0)
1028                 goto msix_single_vector;
1029
1030         for (i = 0; i < msixcnt; i++) {
1031                 msix = &device->msix_entries[i];
1032                 chan = ioat_chan_by_index(device, i);
1033                 err = devm_request_irq(dev, msix->vector,
1034                                        ioat_dma_do_interrupt_msix, 0,
1035                                        "ioat-msix", chan);
1036                 if (err) {
1037                         for (j = 0; j < i; j++) {
1038                                 msix = &device->msix_entries[j];
1039                                 chan = ioat_chan_by_index(device, j);
1040                                 devm_free_irq(dev, msix->vector, chan);
1041                         }
1042                         goto msix_single_vector;
1043                 }
1044         }
1045         intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL;
1046         goto done;
1047
1048 msix_single_vector:
1049         msix = &device->msix_entries[0];
1050         msix->entry = 0;
1051         err = pci_enable_msix(pdev, device->msix_entries, 1);
1052         if (err)
1053                 goto msi;
1054
1055         err = devm_request_irq(dev, msix->vector, ioat_dma_do_interrupt, 0,
1056                                "ioat-msix", device);
1057         if (err) {
1058                 pci_disable_msix(pdev);
1059                 goto msi;
1060         }
1061         goto done;
1062
1063 msi:
1064         err = pci_enable_msi(pdev);
1065         if (err)
1066                 goto intx;
1067
1068         err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 0,
1069                                "ioat-msi", device);
1070         if (err) {
1071                 pci_disable_msi(pdev);
1072                 goto intx;
1073         }
1074         goto done;
1075
1076 intx:
1077         err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt,
1078                                IRQF_SHARED, "ioat-intx", device);
1079         if (err)
1080                 goto err_no_irq;
1081
1082 done:
1083         if (device->intr_quirk)
1084                 device->intr_quirk(device);
1085         intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN;
1086         writeb(intrctrl, device->reg_base + IOAT_INTRCTRL_OFFSET);
1087         return 0;
1088
1089 err_no_irq:
1090         /* Disable all interrupt generation */
1091         writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
1092         dev_err(dev, "no usable interrupts\n");
1093         return err;
1094 }
1095
1096 static void ioat_disable_interrupts(struct ioatdma_device *device)
1097 {
1098         /* Disable all interrupt generation */
1099         writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
1100 }
1101
1102 int ioat_probe(struct ioatdma_device *device)
1103 {
1104         int err = -ENODEV;
1105         struct dma_device *dma = &device->common;
1106         struct pci_dev *pdev = device->pdev;
1107         struct device *dev = &pdev->dev;
1108
1109         /* DMA coherent memory pool for DMA descriptor allocations */
1110         device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
1111                                            sizeof(struct ioat_dma_descriptor),
1112                                            64, 0);
1113         if (!device->dma_pool) {
1114                 err = -ENOMEM;
1115                 goto err_dma_pool;
1116         }
1117
1118         device->completion_pool = pci_pool_create("completion_pool", pdev,
1119                                                   sizeof(u64), SMP_CACHE_BYTES,
1120                                                   SMP_CACHE_BYTES);
1121
1122         if (!device->completion_pool) {
1123                 err = -ENOMEM;
1124                 goto err_completion_pool;
1125         }
1126
1127         device->enumerate_channels(device);
1128
1129         dma_cap_set(DMA_MEMCPY, dma->cap_mask);
1130         dma->dev = &pdev->dev;
1131
1132         dev_err(dev, "Intel(R) I/OAT DMA Engine found,"
1133                 " %d channels, device version 0x%02x, driver version %s\n",
1134                 dma->chancnt, device->version, IOAT_DMA_VERSION);
1135
1136         if (!dma->chancnt) {
1137                 dev_err(dev, "Intel(R) I/OAT DMA Engine problem found: "
1138                         "zero channels detected\n");
1139                 goto err_setup_interrupts;
1140         }
1141
1142         err = ioat_dma_setup_interrupts(device);
1143         if (err)
1144                 goto err_setup_interrupts;
1145
1146         err = ioat_dma_self_test(device);
1147         if (err)
1148                 goto err_self_test;
1149
1150         return 0;
1151
1152 err_self_test:
1153         ioat_disable_interrupts(device);
1154 err_setup_interrupts:
1155         pci_pool_destroy(device->completion_pool);
1156 err_completion_pool:
1157         pci_pool_destroy(device->dma_pool);
1158 err_dma_pool:
1159         return err;
1160 }
1161
1162 int ioat_register(struct ioatdma_device *device)
1163 {
1164         int err = dma_async_device_register(&device->common);
1165
1166         if (err) {
1167                 ioat_disable_interrupts(device);
1168                 pci_pool_destroy(device->completion_pool);
1169                 pci_pool_destroy(device->dma_pool);
1170         }
1171
1172         return err;
1173 }
1174
1175 /* ioat1_intr_quirk - fix up dma ctrl register to enable / disable msi */
1176 static void ioat1_intr_quirk(struct ioatdma_device *device)
1177 {
1178         struct pci_dev *pdev = device->pdev;
1179         u32 dmactrl;
1180
1181         pci_read_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, &dmactrl);
1182         if (pdev->msi_enabled)
1183                 dmactrl |= IOAT_PCI_DMACTRL_MSI_EN;
1184         else
1185                 dmactrl &= ~IOAT_PCI_DMACTRL_MSI_EN;
1186         pci_write_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, dmactrl);
1187 }
1188
1189 int ioat1_dma_probe(struct ioatdma_device *device, int dca)
1190 {
1191         struct pci_dev *pdev = device->pdev;
1192         struct dma_device *dma;
1193         int err;
1194
1195         device->intr_quirk = ioat1_intr_quirk;
1196         device->enumerate_channels = ioat1_enumerate_channels;
1197         dma = &device->common;
1198         dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy;
1199         dma->device_issue_pending = ioat1_dma_memcpy_issue_pending;
1200         dma->device_alloc_chan_resources = ioat1_dma_alloc_chan_resources;
1201         dma->device_free_chan_resources = ioat1_dma_free_chan_resources;
1202         dma->device_is_tx_complete = ioat1_dma_is_complete;
1203
1204         err = ioat_probe(device);
1205         if (err)
1206                 return err;
1207         ioat_set_tcp_copy_break(4096);
1208         err = ioat_register(device);
1209         if (err)
1210                 return err;
1211         if (dca)
1212                 device->dca = ioat_dca_init(pdev, device->reg_base);
1213
1214         INIT_DELAYED_WORK(&device->work, ioat1_chan_watchdog);
1215         schedule_delayed_work(&device->work, WATCHDOG_DELAY);
1216
1217         return err;
1218 }
1219
1220 void ioat_dma_remove(struct ioatdma_device *device)
1221 {
1222         struct dma_device *dma = &device->common;
1223
1224         if (device->version != IOAT_VER_3_0)
1225                 cancel_delayed_work(&device->work);
1226
1227         ioat_disable_interrupts(device);
1228
1229         dma_async_device_unregister(dma);
1230
1231         pci_pool_destroy(device->dma_pool);
1232         pci_pool_destroy(device->completion_pool);
1233
1234         INIT_LIST_HEAD(&dma->channels);
1235 }