]>
Commit | Line | Data |
---|---|---|
0bbd5f4e CL |
1 | /* |
2 | * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify it | |
5 | * under the terms of the GNU General Public License as published by the Free | |
6 | * Software Foundation; either version 2 of the License, or (at your option) | |
7 | * any later version. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, but WITHOUT | |
10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
12 | * more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License along with | |
15 | * this program; if not, write to the Free Software Foundation, Inc., 59 | |
16 | * Temple Place - Suite 330, Boston, MA 02111-1307, USA. | |
17 | * | |
18 | * The full GNU General Public License is included in this distribution in the | |
19 | * file called COPYING. | |
20 | */ | |
21 | ||
22 | /* | |
23 | * This driver supports an Intel I/OAT DMA engine, which does asynchronous | |
24 | * copy operations. | |
25 | */ | |
26 | ||
27 | #include <linux/init.h> | |
28 | #include <linux/module.h> | |
29 | #include <linux/pci.h> | |
30 | #include <linux/interrupt.h> | |
31 | #include <linux/dmaengine.h> | |
32 | #include <linux/delay.h> | |
6b00c92c | 33 | #include <linux/dma-mapping.h> |
0bbd5f4e CL |
34 | #include "ioatdma.h" |
35 | #include "ioatdma_io.h" | |
36 | #include "ioatdma_registers.h" | |
37 | #include "ioatdma_hw.h" | |
38 | ||
39 | #define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common) | |
40 | #define to_ioat_device(dev) container_of(dev, struct ioat_device, common) | |
41 | #define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node) | |
42 | ||
43 | /* internal functions */ | |
44 | static int __devinit ioat_probe(struct pci_dev *pdev, const struct pci_device_id *ent); | |
45 | static void __devexit ioat_remove(struct pci_dev *pdev); | |
46 | ||
47 | static int enumerate_dma_channels(struct ioat_device *device) | |
48 | { | |
49 | u8 xfercap_scale; | |
50 | u32 xfercap; | |
51 | int i; | |
52 | struct ioat_dma_chan *ioat_chan; | |
53 | ||
54 | device->common.chancnt = ioatdma_read8(device, IOAT_CHANCNT_OFFSET); | |
55 | xfercap_scale = ioatdma_read8(device, IOAT_XFERCAP_OFFSET); | |
56 | xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale)); | |
57 | ||
58 | for (i = 0; i < device->common.chancnt; i++) { | |
59 | ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL); | |
60 | if (!ioat_chan) { | |
61 | device->common.chancnt = i; | |
62 | break; | |
63 | } | |
64 | ||
65 | ioat_chan->device = device; | |
66 | ioat_chan->reg_base = device->reg_base + (0x80 * (i + 1)); | |
67 | ioat_chan->xfercap = xfercap; | |
68 | spin_lock_init(&ioat_chan->cleanup_lock); | |
69 | spin_lock_init(&ioat_chan->desc_lock); | |
70 | INIT_LIST_HEAD(&ioat_chan->free_desc); | |
71 | INIT_LIST_HEAD(&ioat_chan->used_desc); | |
72 | /* This should be made common somewhere in dmaengine.c */ | |
73 | ioat_chan->common.device = &device->common; | |
74 | ioat_chan->common.client = NULL; | |
75 | list_add_tail(&ioat_chan->common.device_node, | |
76 | &device->common.channels); | |
77 | } | |
78 | return device->common.chancnt; | |
79 | } | |
80 | ||
81 | static struct ioat_desc_sw *ioat_dma_alloc_descriptor( | |
82 | struct ioat_dma_chan *ioat_chan, | |
83 | int flags) | |
84 | { | |
85 | struct ioat_dma_descriptor *desc; | |
86 | struct ioat_desc_sw *desc_sw; | |
87 | struct ioat_device *ioat_device; | |
88 | dma_addr_t phys; | |
89 | ||
90 | ioat_device = to_ioat_device(ioat_chan->common.device); | |
91 | desc = pci_pool_alloc(ioat_device->dma_pool, flags, &phys); | |
92 | if (unlikely(!desc)) | |
93 | return NULL; | |
94 | ||
95 | desc_sw = kzalloc(sizeof(*desc_sw), flags); | |
96 | if (unlikely(!desc_sw)) { | |
97 | pci_pool_free(ioat_device->dma_pool, desc, phys); | |
98 | return NULL; | |
99 | } | |
100 | ||
101 | memset(desc, 0, sizeof(*desc)); | |
102 | desc_sw->hw = desc; | |
103 | desc_sw->phys = phys; | |
104 | ||
105 | return desc_sw; | |
106 | } | |
107 | ||
108 | #define INITIAL_IOAT_DESC_COUNT 128 | |
109 | ||
110 | static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan); | |
111 | ||
112 | /* returns the actual number of allocated descriptors */ | |
113 | static int ioat_dma_alloc_chan_resources(struct dma_chan *chan) | |
114 | { | |
115 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
116 | struct ioat_desc_sw *desc = NULL; | |
117 | u16 chanctrl; | |
118 | u32 chanerr; | |
119 | int i; | |
120 | LIST_HEAD(tmp_list); | |
121 | ||
122 | /* | |
123 | * In-use bit automatically set by reading chanctrl | |
124 | * If 0, we got it, if 1, someone else did | |
125 | */ | |
126 | chanctrl = ioatdma_chan_read16(ioat_chan, IOAT_CHANCTRL_OFFSET); | |
127 | if (chanctrl & IOAT_CHANCTRL_CHANNEL_IN_USE) | |
128 | return -EBUSY; | |
129 | ||
130 | /* Setup register to interrupt and write completion status on error */ | |
131 | chanctrl = IOAT_CHANCTRL_CHANNEL_IN_USE | | |
132 | IOAT_CHANCTRL_ERR_INT_EN | | |
133 | IOAT_CHANCTRL_ANY_ERR_ABORT_EN | | |
134 | IOAT_CHANCTRL_ERR_COMPLETION_EN; | |
135 | ioatdma_chan_write16(ioat_chan, IOAT_CHANCTRL_OFFSET, chanctrl); | |
136 | ||
137 | chanerr = ioatdma_chan_read32(ioat_chan, IOAT_CHANERR_OFFSET); | |
138 | if (chanerr) { | |
139 | printk("IOAT: CHANERR = %x, clearing\n", chanerr); | |
140 | ioatdma_chan_write32(ioat_chan, IOAT_CHANERR_OFFSET, chanerr); | |
141 | } | |
142 | ||
143 | /* Allocate descriptors */ | |
144 | for (i = 0; i < INITIAL_IOAT_DESC_COUNT; i++) { | |
145 | desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL); | |
146 | if (!desc) { | |
147 | printk(KERN_ERR "IOAT: Only %d initial descriptors\n", i); | |
148 | break; | |
149 | } | |
150 | list_add_tail(&desc->node, &tmp_list); | |
151 | } | |
152 | spin_lock_bh(&ioat_chan->desc_lock); | |
153 | list_splice(&tmp_list, &ioat_chan->free_desc); | |
154 | spin_unlock_bh(&ioat_chan->desc_lock); | |
155 | ||
156 | /* allocate a completion writeback area */ | |
157 | /* doing 2 32bit writes to mmio since 1 64b write doesn't work */ | |
158 | ioat_chan->completion_virt = | |
159 | pci_pool_alloc(ioat_chan->device->completion_pool, | |
160 | GFP_KERNEL, | |
161 | &ioat_chan->completion_addr); | |
162 | memset(ioat_chan->completion_virt, 0, | |
163 | sizeof(*ioat_chan->completion_virt)); | |
164 | ioatdma_chan_write32(ioat_chan, IOAT_CHANCMP_OFFSET_LOW, | |
165 | ((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF); | |
166 | ioatdma_chan_write32(ioat_chan, IOAT_CHANCMP_OFFSET_HIGH, | |
167 | ((u64) ioat_chan->completion_addr) >> 32); | |
168 | ||
169 | ioat_start_null_desc(ioat_chan); | |
170 | return i; | |
171 | } | |
172 | ||
173 | static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan); | |
174 | ||
175 | static void ioat_dma_free_chan_resources(struct dma_chan *chan) | |
176 | { | |
177 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
178 | struct ioat_device *ioat_device = to_ioat_device(chan->device); | |
179 | struct ioat_desc_sw *desc, *_desc; | |
180 | u16 chanctrl; | |
181 | int in_use_descs = 0; | |
182 | ||
183 | ioat_dma_memcpy_cleanup(ioat_chan); | |
184 | ||
185 | ioatdma_chan_write8(ioat_chan, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_RESET); | |
186 | ||
187 | spin_lock_bh(&ioat_chan->desc_lock); | |
188 | list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) { | |
189 | in_use_descs++; | |
190 | list_del(&desc->node); | |
191 | pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys); | |
192 | kfree(desc); | |
193 | } | |
194 | list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) { | |
195 | list_del(&desc->node); | |
196 | pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys); | |
197 | kfree(desc); | |
198 | } | |
199 | spin_unlock_bh(&ioat_chan->desc_lock); | |
200 | ||
201 | pci_pool_free(ioat_device->completion_pool, | |
202 | ioat_chan->completion_virt, | |
203 | ioat_chan->completion_addr); | |
204 | ||
205 | /* one is ok since we left it on there on purpose */ | |
206 | if (in_use_descs > 1) | |
207 | printk(KERN_ERR "IOAT: Freeing %d in use descriptors!\n", | |
208 | in_use_descs - 1); | |
209 | ||
210 | ioat_chan->last_completion = ioat_chan->completion_addr = 0; | |
211 | ||
212 | /* Tell hw the chan is free */ | |
213 | chanctrl = ioatdma_chan_read16(ioat_chan, IOAT_CHANCTRL_OFFSET); | |
214 | chanctrl &= ~IOAT_CHANCTRL_CHANNEL_IN_USE; | |
215 | ioatdma_chan_write16(ioat_chan, IOAT_CHANCTRL_OFFSET, chanctrl); | |
216 | } | |
217 | ||
218 | /** | |
219 | * do_ioat_dma_memcpy - actual function that initiates a IOAT DMA transaction | |
6508871e | 220 | * @ioat_chan: IOAT DMA channel handle |
0bbd5f4e CL |
221 | * @dest: DMA destination address |
222 | * @src: DMA source address | |
223 | * @len: transaction length in bytes | |
224 | */ | |
225 | ||
226 | static dma_cookie_t do_ioat_dma_memcpy(struct ioat_dma_chan *ioat_chan, | |
227 | dma_addr_t dest, | |
228 | dma_addr_t src, | |
229 | size_t len) | |
230 | { | |
231 | struct ioat_desc_sw *first; | |
232 | struct ioat_desc_sw *prev; | |
233 | struct ioat_desc_sw *new; | |
234 | dma_cookie_t cookie; | |
235 | LIST_HEAD(new_chain); | |
236 | u32 copy; | |
237 | size_t orig_len; | |
238 | dma_addr_t orig_src, orig_dst; | |
239 | unsigned int desc_count = 0; | |
240 | unsigned int append = 0; | |
241 | ||
242 | if (!ioat_chan || !dest || !src) | |
243 | return -EFAULT; | |
244 | ||
245 | if (!len) | |
246 | return ioat_chan->common.cookie; | |
247 | ||
248 | orig_len = len; | |
249 | orig_src = src; | |
250 | orig_dst = dest; | |
251 | ||
252 | first = NULL; | |
253 | prev = NULL; | |
254 | ||
255 | spin_lock_bh(&ioat_chan->desc_lock); | |
256 | ||
257 | while (len) { | |
258 | if (!list_empty(&ioat_chan->free_desc)) { | |
259 | new = to_ioat_desc(ioat_chan->free_desc.next); | |
260 | list_del(&new->node); | |
261 | } else { | |
262 | /* try to get another desc */ | |
263 | new = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC); | |
264 | /* will this ever happen? */ | |
265 | /* TODO add upper limit on these */ | |
266 | BUG_ON(!new); | |
267 | } | |
268 | ||
269 | copy = min((u32) len, ioat_chan->xfercap); | |
270 | ||
271 | new->hw->size = copy; | |
272 | new->hw->ctl = 0; | |
273 | new->hw->src_addr = src; | |
274 | new->hw->dst_addr = dest; | |
275 | new->cookie = 0; | |
276 | ||
277 | /* chain together the physical address list for the HW */ | |
278 | if (!first) | |
279 | first = new; | |
280 | else | |
281 | prev->hw->next = (u64) new->phys; | |
282 | ||
283 | prev = new; | |
284 | ||
285 | len -= copy; | |
286 | dest += copy; | |
287 | src += copy; | |
288 | ||
289 | list_add_tail(&new->node, &new_chain); | |
290 | desc_count++; | |
291 | } | |
292 | new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS; | |
293 | new->hw->next = 0; | |
294 | ||
295 | /* cookie incr and addition to used_list must be atomic */ | |
296 | ||
297 | cookie = ioat_chan->common.cookie; | |
298 | cookie++; | |
299 | if (cookie < 0) | |
300 | cookie = 1; | |
301 | ioat_chan->common.cookie = new->cookie = cookie; | |
302 | ||
303 | pci_unmap_addr_set(new, src, orig_src); | |
304 | pci_unmap_addr_set(new, dst, orig_dst); | |
305 | pci_unmap_len_set(new, src_len, orig_len); | |
306 | pci_unmap_len_set(new, dst_len, orig_len); | |
307 | ||
308 | /* write address into NextDescriptor field of last desc in chain */ | |
309 | to_ioat_desc(ioat_chan->used_desc.prev)->hw->next = first->phys; | |
310 | list_splice_init(&new_chain, ioat_chan->used_desc.prev); | |
311 | ||
312 | ioat_chan->pending += desc_count; | |
313 | if (ioat_chan->pending >= 20) { | |
314 | append = 1; | |
315 | ioat_chan->pending = 0; | |
316 | } | |
317 | ||
318 | spin_unlock_bh(&ioat_chan->desc_lock); | |
319 | ||
320 | if (append) | |
321 | ioatdma_chan_write8(ioat_chan, | |
322 | IOAT_CHANCMD_OFFSET, | |
323 | IOAT_CHANCMD_APPEND); | |
324 | return cookie; | |
325 | } | |
326 | ||
327 | /** | |
328 | * ioat_dma_memcpy_buf_to_buf - wrapper that takes src & dest bufs | |
329 | * @chan: IOAT DMA channel handle | |
330 | * @dest: DMA destination address | |
331 | * @src: DMA source address | |
332 | * @len: transaction length in bytes | |
333 | */ | |
334 | ||
335 | static dma_cookie_t ioat_dma_memcpy_buf_to_buf(struct dma_chan *chan, | |
336 | void *dest, | |
337 | void *src, | |
338 | size_t len) | |
339 | { | |
340 | dma_addr_t dest_addr; | |
341 | dma_addr_t src_addr; | |
342 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
343 | ||
344 | dest_addr = pci_map_single(ioat_chan->device->pdev, | |
345 | dest, len, PCI_DMA_FROMDEVICE); | |
346 | src_addr = pci_map_single(ioat_chan->device->pdev, | |
347 | src, len, PCI_DMA_TODEVICE); | |
348 | ||
349 | return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len); | |
350 | } | |
351 | ||
352 | /** | |
353 | * ioat_dma_memcpy_buf_to_pg - wrapper, copying from a buf to a page | |
354 | * @chan: IOAT DMA channel handle | |
355 | * @page: pointer to the page to copy to | |
356 | * @offset: offset into that page | |
357 | * @src: DMA source address | |
358 | * @len: transaction length in bytes | |
359 | */ | |
360 | ||
361 | static dma_cookie_t ioat_dma_memcpy_buf_to_pg(struct dma_chan *chan, | |
362 | struct page *page, | |
363 | unsigned int offset, | |
364 | void *src, | |
365 | size_t len) | |
366 | { | |
367 | dma_addr_t dest_addr; | |
368 | dma_addr_t src_addr; | |
369 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
370 | ||
371 | dest_addr = pci_map_page(ioat_chan->device->pdev, | |
372 | page, offset, len, PCI_DMA_FROMDEVICE); | |
373 | src_addr = pci_map_single(ioat_chan->device->pdev, | |
374 | src, len, PCI_DMA_TODEVICE); | |
375 | ||
376 | return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len); | |
377 | } | |
378 | ||
379 | /** | |
380 | * ioat_dma_memcpy_pg_to_pg - wrapper, copying between two pages | |
381 | * @chan: IOAT DMA channel handle | |
382 | * @dest_pg: pointer to the page to copy to | |
383 | * @dest_off: offset into that page | |
384 | * @src_pg: pointer to the page to copy from | |
385 | * @src_off: offset into that page | |
6508871e | 386 | * @len: transaction length in bytes. This is guaranteed not to make a copy |
0bbd5f4e CL |
387 | * across a page boundary. |
388 | */ | |
389 | ||
390 | static dma_cookie_t ioat_dma_memcpy_pg_to_pg(struct dma_chan *chan, | |
391 | struct page *dest_pg, | |
392 | unsigned int dest_off, | |
393 | struct page *src_pg, | |
394 | unsigned int src_off, | |
395 | size_t len) | |
396 | { | |
397 | dma_addr_t dest_addr; | |
398 | dma_addr_t src_addr; | |
399 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
400 | ||
401 | dest_addr = pci_map_page(ioat_chan->device->pdev, | |
402 | dest_pg, dest_off, len, PCI_DMA_FROMDEVICE); | |
403 | src_addr = pci_map_page(ioat_chan->device->pdev, | |
404 | src_pg, src_off, len, PCI_DMA_TODEVICE); | |
405 | ||
406 | return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len); | |
407 | } | |
408 | ||
409 | /** | |
6508871e | 410 | * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended descriptors to hw |
0bbd5f4e CL |
411 | * @chan: DMA channel handle |
412 | */ | |
413 | ||
414 | static void ioat_dma_memcpy_issue_pending(struct dma_chan *chan) | |
415 | { | |
416 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
417 | ||
418 | if (ioat_chan->pending != 0) { | |
419 | ioat_chan->pending = 0; | |
420 | ioatdma_chan_write8(ioat_chan, | |
421 | IOAT_CHANCMD_OFFSET, | |
422 | IOAT_CHANCMD_APPEND); | |
423 | } | |
424 | } | |
425 | ||
426 | static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan) | |
427 | { | |
428 | unsigned long phys_complete; | |
429 | struct ioat_desc_sw *desc, *_desc; | |
430 | dma_cookie_t cookie = 0; | |
431 | ||
432 | prefetch(chan->completion_virt); | |
433 | ||
434 | if (!spin_trylock(&chan->cleanup_lock)) | |
435 | return; | |
436 | ||
437 | /* The completion writeback can happen at any time, | |
438 | so reads by the driver need to be atomic operations | |
439 | The descriptor physical addresses are limited to 32-bits | |
440 | when the CPU can only do a 32-bit mov */ | |
441 | ||
442 | #if (BITS_PER_LONG == 64) | |
443 | phys_complete = | |
444 | chan->completion_virt->full & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR; | |
445 | #else | |
446 | phys_complete = chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK; | |
447 | #endif | |
448 | ||
449 | if ((chan->completion_virt->full & IOAT_CHANSTS_DMA_TRANSFER_STATUS) == | |
450 | IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) { | |
451 | printk("IOAT: Channel halted, chanerr = %x\n", | |
452 | ioatdma_chan_read32(chan, IOAT_CHANERR_OFFSET)); | |
453 | ||
454 | /* TODO do something to salvage the situation */ | |
455 | } | |
456 | ||
457 | if (phys_complete == chan->last_completion) { | |
458 | spin_unlock(&chan->cleanup_lock); | |
459 | return; | |
460 | } | |
461 | ||
462 | spin_lock_bh(&chan->desc_lock); | |
463 | list_for_each_entry_safe(desc, _desc, &chan->used_desc, node) { | |
464 | ||
465 | /* | |
466 | * Incoming DMA requests may use multiple descriptors, due to | |
467 | * exceeding xfercap, perhaps. If so, only the last one will | |
468 | * have a cookie, and require unmapping. | |
469 | */ | |
470 | if (desc->cookie) { | |
471 | cookie = desc->cookie; | |
472 | ||
473 | /* yes we are unmapping both _page and _single alloc'd | |
474 | regions with unmap_page. Is this *really* that bad? | |
475 | */ | |
476 | pci_unmap_page(chan->device->pdev, | |
477 | pci_unmap_addr(desc, dst), | |
478 | pci_unmap_len(desc, dst_len), | |
479 | PCI_DMA_FROMDEVICE); | |
480 | pci_unmap_page(chan->device->pdev, | |
481 | pci_unmap_addr(desc, src), | |
482 | pci_unmap_len(desc, src_len), | |
483 | PCI_DMA_TODEVICE); | |
484 | } | |
485 | ||
486 | if (desc->phys != phys_complete) { | |
487 | /* a completed entry, but not the last, so cleanup */ | |
488 | list_del(&desc->node); | |
489 | list_add_tail(&desc->node, &chan->free_desc); | |
490 | } else { | |
491 | /* last used desc. Do not remove, so we can append from | |
492 | it, but don't look at it next time, either */ | |
493 | desc->cookie = 0; | |
494 | ||
495 | /* TODO check status bits? */ | |
496 | break; | |
497 | } | |
498 | } | |
499 | ||
500 | spin_unlock_bh(&chan->desc_lock); | |
501 | ||
502 | chan->last_completion = phys_complete; | |
503 | if (cookie != 0) | |
504 | chan->completed_cookie = cookie; | |
505 | ||
506 | spin_unlock(&chan->cleanup_lock); | |
507 | } | |
508 | ||
509 | /** | |
510 | * ioat_dma_is_complete - poll the status of a IOAT DMA transaction | |
511 | * @chan: IOAT DMA channel handle | |
512 | * @cookie: DMA transaction identifier | |
6508871e RD |
513 | * @done: if not %NULL, updated with last completed transaction |
514 | * @used: if not %NULL, updated with last used transaction | |
0bbd5f4e CL |
515 | */ |
516 | ||
517 | static enum dma_status ioat_dma_is_complete(struct dma_chan *chan, | |
518 | dma_cookie_t cookie, | |
519 | dma_cookie_t *done, | |
520 | dma_cookie_t *used) | |
521 | { | |
522 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
523 | dma_cookie_t last_used; | |
524 | dma_cookie_t last_complete; | |
525 | enum dma_status ret; | |
526 | ||
527 | last_used = chan->cookie; | |
528 | last_complete = ioat_chan->completed_cookie; | |
529 | ||
530 | if (done) | |
531 | *done= last_complete; | |
532 | if (used) | |
533 | *used = last_used; | |
534 | ||
535 | ret = dma_async_is_complete(cookie, last_complete, last_used); | |
536 | if (ret == DMA_SUCCESS) | |
537 | return ret; | |
538 | ||
539 | ioat_dma_memcpy_cleanup(ioat_chan); | |
540 | ||
541 | last_used = chan->cookie; | |
542 | last_complete = ioat_chan->completed_cookie; | |
543 | ||
544 | if (done) | |
545 | *done= last_complete; | |
546 | if (used) | |
547 | *used = last_used; | |
548 | ||
549 | return dma_async_is_complete(cookie, last_complete, last_used); | |
550 | } | |
551 | ||
552 | /* PCI API */ | |
553 | ||
554 | static struct pci_device_id ioat_pci_tbl[] = { | |
555 | { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT) }, | |
556 | { 0, } | |
557 | }; | |
558 | ||
559 | static struct pci_driver ioat_pci_drv = { | |
560 | .name = "ioatdma", | |
561 | .id_table = ioat_pci_tbl, | |
562 | .probe = ioat_probe, | |
563 | .remove = __devexit_p(ioat_remove), | |
564 | }; | |
565 | ||
7d12e780 | 566 | static irqreturn_t ioat_do_interrupt(int irq, void *data) |
0bbd5f4e CL |
567 | { |
568 | struct ioat_device *instance = data; | |
569 | unsigned long attnstatus; | |
570 | u8 intrctrl; | |
571 | ||
572 | intrctrl = ioatdma_read8(instance, IOAT_INTRCTRL_OFFSET); | |
573 | ||
574 | if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN)) | |
575 | return IRQ_NONE; | |
576 | ||
577 | if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) { | |
578 | ioatdma_write8(instance, IOAT_INTRCTRL_OFFSET, intrctrl); | |
579 | return IRQ_NONE; | |
580 | } | |
581 | ||
582 | attnstatus = ioatdma_read32(instance, IOAT_ATTNSTATUS_OFFSET); | |
583 | ||
584 | printk(KERN_ERR "ioatdma error: interrupt! status %lx\n", attnstatus); | |
585 | ||
586 | ioatdma_write8(instance, IOAT_INTRCTRL_OFFSET, intrctrl); | |
587 | return IRQ_HANDLED; | |
588 | } | |
589 | ||
590 | static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan) | |
591 | { | |
592 | struct ioat_desc_sw *desc; | |
593 | ||
594 | spin_lock_bh(&ioat_chan->desc_lock); | |
595 | ||
596 | if (!list_empty(&ioat_chan->free_desc)) { | |
597 | desc = to_ioat_desc(ioat_chan->free_desc.next); | |
598 | list_del(&desc->node); | |
599 | } else { | |
600 | /* try to get another desc */ | |
601 | spin_unlock_bh(&ioat_chan->desc_lock); | |
602 | desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL); | |
603 | spin_lock_bh(&ioat_chan->desc_lock); | |
604 | /* will this ever happen? */ | |
605 | BUG_ON(!desc); | |
606 | } | |
607 | ||
608 | desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL; | |
609 | desc->hw->next = 0; | |
610 | ||
611 | list_add_tail(&desc->node, &ioat_chan->used_desc); | |
612 | spin_unlock_bh(&ioat_chan->desc_lock); | |
613 | ||
614 | #if (BITS_PER_LONG == 64) | |
615 | ioatdma_chan_write64(ioat_chan, IOAT_CHAINADDR_OFFSET, desc->phys); | |
616 | #else | |
617 | ioatdma_chan_write32(ioat_chan, | |
618 | IOAT_CHAINADDR_OFFSET_LOW, | |
619 | (u32) desc->phys); | |
620 | ioatdma_chan_write32(ioat_chan, IOAT_CHAINADDR_OFFSET_HIGH, 0); | |
621 | #endif | |
622 | ioatdma_chan_write8(ioat_chan, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_START); | |
623 | } | |
624 | ||
625 | /* | |
626 | * Perform a IOAT transaction to verify the HW works. | |
627 | */ | |
628 | #define IOAT_TEST_SIZE 2000 | |
629 | ||
630 | static int ioat_self_test(struct ioat_device *device) | |
631 | { | |
632 | int i; | |
633 | u8 *src; | |
634 | u8 *dest; | |
635 | struct dma_chan *dma_chan; | |
636 | dma_cookie_t cookie; | |
637 | int err = 0; | |
638 | ||
639 | src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, SLAB_KERNEL); | |
640 | if (!src) | |
641 | return -ENOMEM; | |
642 | dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, SLAB_KERNEL); | |
643 | if (!dest) { | |
644 | kfree(src); | |
645 | return -ENOMEM; | |
646 | } | |
647 | ||
648 | /* Fill in src buffer */ | |
649 | for (i = 0; i < IOAT_TEST_SIZE; i++) | |
650 | src[i] = (u8)i; | |
651 | ||
652 | /* Start copy, using first DMA channel */ | |
653 | dma_chan = container_of(device->common.channels.next, | |
654 | struct dma_chan, | |
655 | device_node); | |
656 | if (ioat_dma_alloc_chan_resources(dma_chan) < 1) { | |
657 | err = -ENODEV; | |
658 | goto out; | |
659 | } | |
660 | ||
661 | cookie = ioat_dma_memcpy_buf_to_buf(dma_chan, dest, src, IOAT_TEST_SIZE); | |
662 | ioat_dma_memcpy_issue_pending(dma_chan); | |
663 | msleep(1); | |
664 | ||
665 | if (ioat_dma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) { | |
666 | printk(KERN_ERR "ioatdma: Self-test copy timed out, disabling\n"); | |
667 | err = -ENODEV; | |
668 | goto free_resources; | |
669 | } | |
670 | if (memcmp(src, dest, IOAT_TEST_SIZE)) { | |
671 | printk(KERN_ERR "ioatdma: Self-test copy failed compare, disabling\n"); | |
672 | err = -ENODEV; | |
673 | goto free_resources; | |
674 | } | |
675 | ||
676 | free_resources: | |
677 | ioat_dma_free_chan_resources(dma_chan); | |
678 | out: | |
679 | kfree(src); | |
680 | kfree(dest); | |
681 | return err; | |
682 | } | |
683 | ||
684 | static int __devinit ioat_probe(struct pci_dev *pdev, | |
685 | const struct pci_device_id *ent) | |
686 | { | |
687 | int err; | |
688 | unsigned long mmio_start, mmio_len; | |
689 | void *reg_base; | |
690 | struct ioat_device *device; | |
691 | ||
692 | err = pci_enable_device(pdev); | |
693 | if (err) | |
694 | goto err_enable_device; | |
695 | ||
696 | err = pci_set_dma_mask(pdev, DMA_64BIT_MASK); | |
697 | if (err) | |
698 | err = pci_set_dma_mask(pdev, DMA_32BIT_MASK); | |
699 | if (err) | |
700 | goto err_set_dma_mask; | |
701 | ||
702 | err = pci_request_regions(pdev, ioat_pci_drv.name); | |
703 | if (err) | |
704 | goto err_request_regions; | |
705 | ||
706 | mmio_start = pci_resource_start(pdev, 0); | |
707 | mmio_len = pci_resource_len(pdev, 0); | |
708 | ||
709 | reg_base = ioremap(mmio_start, mmio_len); | |
710 | if (!reg_base) { | |
711 | err = -ENOMEM; | |
712 | goto err_ioremap; | |
713 | } | |
714 | ||
715 | device = kzalloc(sizeof(*device), GFP_KERNEL); | |
716 | if (!device) { | |
717 | err = -ENOMEM; | |
718 | goto err_kzalloc; | |
719 | } | |
720 | ||
721 | /* DMA coherent memory pool for DMA descriptor allocations */ | |
722 | device->dma_pool = pci_pool_create("dma_desc_pool", pdev, | |
723 | sizeof(struct ioat_dma_descriptor), 64, 0); | |
724 | if (!device->dma_pool) { | |
725 | err = -ENOMEM; | |
726 | goto err_dma_pool; | |
727 | } | |
728 | ||
729 | device->completion_pool = pci_pool_create("completion_pool", pdev, sizeof(u64), SMP_CACHE_BYTES, SMP_CACHE_BYTES); | |
730 | if (!device->completion_pool) { | |
731 | err = -ENOMEM; | |
732 | goto err_completion_pool; | |
733 | } | |
734 | ||
735 | device->pdev = pdev; | |
736 | pci_set_drvdata(pdev, device); | |
737 | #ifdef CONFIG_PCI_MSI | |
738 | if (pci_enable_msi(pdev) == 0) { | |
739 | device->msi = 1; | |
740 | } else { | |
741 | device->msi = 0; | |
742 | } | |
743 | #endif | |
dace1453 | 744 | err = request_irq(pdev->irq, &ioat_do_interrupt, IRQF_SHARED, "ioat", |
0bbd5f4e CL |
745 | device); |
746 | if (err) | |
747 | goto err_irq; | |
748 | ||
749 | device->reg_base = reg_base; | |
750 | ||
751 | ioatdma_write8(device, IOAT_INTRCTRL_OFFSET, IOAT_INTRCTRL_MASTER_INT_EN); | |
752 | pci_set_master(pdev); | |
753 | ||
754 | INIT_LIST_HEAD(&device->common.channels); | |
755 | enumerate_dma_channels(device); | |
756 | ||
757 | device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources; | |
758 | device->common.device_free_chan_resources = ioat_dma_free_chan_resources; | |
759 | device->common.device_memcpy_buf_to_buf = ioat_dma_memcpy_buf_to_buf; | |
760 | device->common.device_memcpy_buf_to_pg = ioat_dma_memcpy_buf_to_pg; | |
761 | device->common.device_memcpy_pg_to_pg = ioat_dma_memcpy_pg_to_pg; | |
762 | device->common.device_memcpy_complete = ioat_dma_is_complete; | |
763 | device->common.device_memcpy_issue_pending = ioat_dma_memcpy_issue_pending; | |
764 | printk(KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n", | |
765 | device->common.chancnt); | |
766 | ||
767 | err = ioat_self_test(device); | |
768 | if (err) | |
769 | goto err_self_test; | |
770 | ||
771 | dma_async_device_register(&device->common); | |
772 | ||
773 | return 0; | |
774 | ||
775 | err_self_test: | |
776 | err_irq: | |
777 | pci_pool_destroy(device->completion_pool); | |
778 | err_completion_pool: | |
779 | pci_pool_destroy(device->dma_pool); | |
780 | err_dma_pool: | |
781 | kfree(device); | |
782 | err_kzalloc: | |
783 | iounmap(reg_base); | |
784 | err_ioremap: | |
785 | pci_release_regions(pdev); | |
786 | err_request_regions: | |
787 | err_set_dma_mask: | |
788 | pci_disable_device(pdev); | |
789 | err_enable_device: | |
790 | return err; | |
791 | } | |
792 | ||
793 | static void __devexit ioat_remove(struct pci_dev *pdev) | |
794 | { | |
795 | struct ioat_device *device; | |
796 | struct dma_chan *chan, *_chan; | |
797 | struct ioat_dma_chan *ioat_chan; | |
798 | ||
799 | device = pci_get_drvdata(pdev); | |
800 | dma_async_device_unregister(&device->common); | |
801 | ||
802 | free_irq(device->pdev->irq, device); | |
803 | #ifdef CONFIG_PCI_MSI | |
804 | if (device->msi) | |
805 | pci_disable_msi(device->pdev); | |
806 | #endif | |
807 | pci_pool_destroy(device->dma_pool); | |
808 | pci_pool_destroy(device->completion_pool); | |
809 | iounmap(device->reg_base); | |
810 | pci_release_regions(pdev); | |
811 | pci_disable_device(pdev); | |
812 | list_for_each_entry_safe(chan, _chan, &device->common.channels, device_node) { | |
813 | ioat_chan = to_ioat_chan(chan); | |
814 | list_del(&chan->device_node); | |
815 | kfree(ioat_chan); | |
816 | } | |
817 | kfree(device); | |
818 | } | |
819 | ||
820 | /* MODULE API */ | |
821 | MODULE_VERSION("1.7"); | |
822 | MODULE_LICENSE("GPL"); | |
823 | MODULE_AUTHOR("Intel Corporation"); | |
824 | ||
825 | static int __init ioat_init_module(void) | |
826 | { | |
827 | /* it's currently unsafe to unload this module */ | |
828 | /* if forced, worst case is that rmmod hangs */ | |
8070b2b1 | 829 | __unsafe(THIS_MODULE); |
0bbd5f4e | 830 | |
b8263158 | 831 | return pci_register_driver(&ioat_pci_drv); |
0bbd5f4e CL |
832 | } |
833 | ||
834 | module_init(ioat_init_module); | |
835 | ||
836 | static void __exit ioat_exit_module(void) | |
837 | { | |
838 | pci_unregister_driver(&ioat_pci_drv); | |
839 | } | |
840 | ||
841 | module_exit(ioat_exit_module); |