]>
Commit | Line | Data |
---|---|---|
bf40a686 DW |
1 | /* |
2 | * This file is provided under a dual BSD/GPLv2 license. When using or | |
3 | * redistributing this file, you may do so under either license. | |
4 | * | |
5 | * GPL LICENSE SUMMARY | |
6 | * | |
7 | * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved. | |
8 | * | |
9 | * This program is free software; you can redistribute it and/or modify it | |
10 | * under the terms and conditions of the GNU General Public License, | |
11 | * version 2, as published by the Free Software Foundation. | |
12 | * | |
13 | * This program is distributed in the hope that it will be useful, but WITHOUT | |
14 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
15 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
16 | * more details. | |
17 | * | |
18 | * You should have received a copy of the GNU General Public License along with | |
19 | * this program; if not, write to the Free Software Foundation, Inc., | |
20 | * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. | |
21 | * | |
22 | * The full GNU General Public License is included in this distribution in | |
23 | * the file called "COPYING". | |
24 | * | |
25 | * BSD LICENSE | |
26 | * | |
27 | * Copyright(c) 2004-2009 Intel Corporation. All rights reserved. | |
28 | * | |
29 | * Redistribution and use in source and binary forms, with or without | |
30 | * modification, are permitted provided that the following conditions are met: | |
31 | * | |
32 | * * Redistributions of source code must retain the above copyright | |
33 | * notice, this list of conditions and the following disclaimer. | |
34 | * * Redistributions in binary form must reproduce the above copyright | |
35 | * notice, this list of conditions and the following disclaimer in | |
36 | * the documentation and/or other materials provided with the | |
37 | * distribution. | |
38 | * * Neither the name of Intel Corporation nor the names of its | |
39 | * contributors may be used to endorse or promote products derived | |
40 | * from this software without specific prior written permission. | |
41 | * | |
42 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | |
43 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |
44 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |
45 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE | |
46 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
47 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
48 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | |
49 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | |
50 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | |
51 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | |
52 | * POSSIBILITY OF SUCH DAMAGE. | |
53 | */ | |
54 | ||
55 | /* | |
56 | * Support routines for v3+ hardware | |
57 | */ | |
58 | ||
59 | #include <linux/pci.h> | |
60 | #include <linux/dmaengine.h> | |
61 | #include <linux/dma-mapping.h> | |
62 | #include "registers.h" | |
63 | #include "hw.h" | |
64 | #include "dma.h" | |
65 | #include "dma_v2.h" | |
66 | ||
b094ad3b DW |
67 | /* ioat hardware assumes at least two sources for raid operations */ |
68 | #define src_cnt_to_sw(x) ((x) + 2) | |
69 | #define src_cnt_to_hw(x) ((x) - 2) | |
70 | ||
71 | /* provide a lookup table for setting the source address in the base or | |
72 | * extended descriptor of an xor descriptor | |
73 | */ | |
74 | static const u8 xor_idx_to_desc __read_mostly = 0xd0; | |
75 | static const u8 xor_idx_to_field[] __read_mostly = { 1, 4, 5, 6, 7, 0, 1, 2 }; | |
76 | ||
77 | static dma_addr_t xor_get_src(struct ioat_raw_descriptor *descs[2], int idx) | |
78 | { | |
79 | struct ioat_raw_descriptor *raw = descs[xor_idx_to_desc >> idx & 1]; | |
80 | ||
81 | return raw->field[xor_idx_to_field[idx]]; | |
82 | } | |
83 | ||
84 | static void xor_set_src(struct ioat_raw_descriptor *descs[2], | |
85 | dma_addr_t addr, u32 offset, int idx) | |
86 | { | |
87 | struct ioat_raw_descriptor *raw = descs[xor_idx_to_desc >> idx & 1]; | |
88 | ||
89 | raw->field[xor_idx_to_field[idx]] = addr + offset; | |
90 | } | |
91 | ||
bf40a686 | 92 | static void ioat3_dma_unmap(struct ioat2_dma_chan *ioat, |
b094ad3b | 93 | struct ioat_ring_ent *desc, int idx) |
bf40a686 DW |
94 | { |
95 | struct ioat_chan_common *chan = &ioat->base; | |
96 | struct pci_dev *pdev = chan->device->pdev; | |
97 | size_t len = desc->len; | |
98 | size_t offset = len - desc->hw->size; | |
99 | struct dma_async_tx_descriptor *tx = &desc->txd; | |
100 | enum dma_ctrl_flags flags = tx->flags; | |
101 | ||
102 | switch (desc->hw->ctl_f.op) { | |
103 | case IOAT_OP_COPY: | |
104 | ioat_dma_unmap(chan, flags, len, desc->hw); | |
105 | break; | |
106 | case IOAT_OP_FILL: { | |
107 | struct ioat_fill_descriptor *hw = desc->fill; | |
108 | ||
109 | if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) | |
110 | ioat_unmap(pdev, hw->dst_addr - offset, len, | |
111 | PCI_DMA_FROMDEVICE, flags, 1); | |
112 | break; | |
113 | } | |
b094ad3b DW |
114 | case IOAT_OP_XOR_VAL: |
115 | case IOAT_OP_XOR: { | |
116 | struct ioat_xor_descriptor *xor = desc->xor; | |
117 | struct ioat_ring_ent *ext; | |
118 | struct ioat_xor_ext_descriptor *xor_ex = NULL; | |
119 | int src_cnt = src_cnt_to_sw(xor->ctl_f.src_cnt); | |
120 | struct ioat_raw_descriptor *descs[2]; | |
121 | int i; | |
122 | ||
123 | if (src_cnt > 5) { | |
124 | ext = ioat2_get_ring_ent(ioat, idx + 1); | |
125 | xor_ex = ext->xor_ex; | |
126 | } | |
127 | ||
128 | if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) { | |
129 | descs[0] = (struct ioat_raw_descriptor *) xor; | |
130 | descs[1] = (struct ioat_raw_descriptor *) xor_ex; | |
131 | for (i = 0; i < src_cnt; i++) { | |
132 | dma_addr_t src = xor_get_src(descs, i); | |
133 | ||
134 | ioat_unmap(pdev, src - offset, len, | |
135 | PCI_DMA_TODEVICE, flags, 0); | |
136 | } | |
137 | ||
138 | /* dest is a source in xor validate operations */ | |
139 | if (xor->ctl_f.op == IOAT_OP_XOR_VAL) { | |
140 | ioat_unmap(pdev, xor->dst_addr - offset, len, | |
141 | PCI_DMA_TODEVICE, flags, 1); | |
142 | break; | |
143 | } | |
144 | } | |
145 | ||
146 | if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) | |
147 | ioat_unmap(pdev, xor->dst_addr - offset, len, | |
148 | PCI_DMA_FROMDEVICE, flags, 1); | |
149 | break; | |
150 | } | |
bf40a686 DW |
151 | default: |
152 | dev_err(&pdev->dev, "%s: unknown op type: %#x\n", | |
153 | __func__, desc->hw->ctl_f.op); | |
154 | } | |
155 | } | |
156 | ||
b094ad3b DW |
157 | static bool desc_has_ext(struct ioat_ring_ent *desc) |
158 | { | |
159 | struct ioat_dma_descriptor *hw = desc->hw; | |
160 | ||
161 | if (hw->ctl_f.op == IOAT_OP_XOR || | |
162 | hw->ctl_f.op == IOAT_OP_XOR_VAL) { | |
163 | struct ioat_xor_descriptor *xor = desc->xor; | |
bf40a686 | 164 | |
b094ad3b DW |
165 | if (src_cnt_to_sw(xor->ctl_f.src_cnt) > 5) |
166 | return true; | |
167 | } | |
168 | ||
169 | return false; | |
170 | } | |
171 | ||
172 | /** | |
173 | * __cleanup - reclaim used descriptors | |
174 | * @ioat: channel (ring) to clean | |
175 | * | |
176 | * The difference from the dma_v2.c __cleanup() is that this routine | |
177 | * handles extended descriptors and dma-unmapping raid operations. | |
178 | */ | |
bf40a686 DW |
179 | static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete) |
180 | { | |
181 | struct ioat_chan_common *chan = &ioat->base; | |
182 | struct ioat_ring_ent *desc; | |
183 | bool seen_current = false; | |
184 | u16 active; | |
185 | int i; | |
186 | ||
187 | dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n", | |
188 | __func__, ioat->head, ioat->tail, ioat->issued); | |
189 | ||
190 | active = ioat2_ring_active(ioat); | |
191 | for (i = 0; i < active && !seen_current; i++) { | |
192 | struct dma_async_tx_descriptor *tx; | |
193 | ||
194 | prefetch(ioat2_get_ring_ent(ioat, ioat->tail + i + 1)); | |
195 | desc = ioat2_get_ring_ent(ioat, ioat->tail + i); | |
196 | dump_desc_dbg(ioat, desc); | |
197 | tx = &desc->txd; | |
198 | if (tx->cookie) { | |
199 | chan->completed_cookie = tx->cookie; | |
b094ad3b | 200 | ioat3_dma_unmap(ioat, desc, ioat->tail + i); |
bf40a686 DW |
201 | tx->cookie = 0; |
202 | if (tx->callback) { | |
203 | tx->callback(tx->callback_param); | |
204 | tx->callback = NULL; | |
205 | } | |
206 | } | |
207 | ||
208 | if (tx->phys == phys_complete) | |
209 | seen_current = true; | |
b094ad3b DW |
210 | |
211 | /* skip extended descriptors */ | |
212 | if (desc_has_ext(desc)) { | |
213 | BUG_ON(i + 1 >= active); | |
214 | i++; | |
215 | } | |
bf40a686 DW |
216 | } |
217 | ioat->tail += i; | |
218 | BUG_ON(!seen_current); /* no active descs have written a completion? */ | |
219 | chan->last_completion = phys_complete; | |
220 | if (ioat->head == ioat->tail) { | |
221 | dev_dbg(to_dev(chan), "%s: cancel completion timeout\n", | |
222 | __func__); | |
223 | clear_bit(IOAT_COMPLETION_PENDING, &chan->state); | |
224 | mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); | |
225 | } | |
226 | } | |
227 | ||
228 | static void ioat3_cleanup(struct ioat2_dma_chan *ioat) | |
229 | { | |
230 | struct ioat_chan_common *chan = &ioat->base; | |
231 | unsigned long phys_complete; | |
232 | ||
233 | prefetch(chan->completion); | |
234 | ||
235 | if (!spin_trylock_bh(&chan->cleanup_lock)) | |
236 | return; | |
237 | ||
238 | if (!ioat_cleanup_preamble(chan, &phys_complete)) { | |
239 | spin_unlock_bh(&chan->cleanup_lock); | |
240 | return; | |
241 | } | |
242 | ||
243 | if (!spin_trylock_bh(&ioat->ring_lock)) { | |
244 | spin_unlock_bh(&chan->cleanup_lock); | |
245 | return; | |
246 | } | |
247 | ||
248 | __cleanup(ioat, phys_complete); | |
249 | ||
250 | spin_unlock_bh(&ioat->ring_lock); | |
251 | spin_unlock_bh(&chan->cleanup_lock); | |
252 | } | |
253 | ||
254 | static void ioat3_cleanup_tasklet(unsigned long data) | |
255 | { | |
256 | struct ioat2_dma_chan *ioat = (void *) data; | |
257 | ||
258 | ioat3_cleanup(ioat); | |
e61dacae DW |
259 | writew(IOAT_CHANCTRL_RUN | IOAT3_CHANCTRL_COMPL_DCA_EN, |
260 | ioat->base.reg_base + IOAT_CHANCTRL_OFFSET); | |
bf40a686 DW |
261 | } |
262 | ||
263 | static void ioat3_restart_channel(struct ioat2_dma_chan *ioat) | |
264 | { | |
265 | struct ioat_chan_common *chan = &ioat->base; | |
266 | unsigned long phys_complete; | |
267 | u32 status; | |
268 | ||
269 | status = ioat_chansts(chan); | |
270 | if (is_ioat_active(status) || is_ioat_idle(status)) | |
271 | ioat_suspend(chan); | |
272 | while (is_ioat_active(status) || is_ioat_idle(status)) { | |
273 | status = ioat_chansts(chan); | |
274 | cpu_relax(); | |
275 | } | |
276 | ||
277 | if (ioat_cleanup_preamble(chan, &phys_complete)) | |
278 | __cleanup(ioat, phys_complete); | |
279 | ||
280 | __ioat2_restart_chan(ioat); | |
281 | } | |
282 | ||
283 | static void ioat3_timer_event(unsigned long data) | |
284 | { | |
285 | struct ioat2_dma_chan *ioat = (void *) data; | |
286 | struct ioat_chan_common *chan = &ioat->base; | |
287 | ||
288 | spin_lock_bh(&chan->cleanup_lock); | |
289 | if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) { | |
290 | unsigned long phys_complete; | |
291 | u64 status; | |
292 | ||
293 | spin_lock_bh(&ioat->ring_lock); | |
294 | status = ioat_chansts(chan); | |
295 | ||
296 | /* when halted due to errors check for channel | |
297 | * programming errors before advancing the completion state | |
298 | */ | |
299 | if (is_ioat_halted(status)) { | |
300 | u32 chanerr; | |
301 | ||
302 | chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); | |
303 | BUG_ON(is_ioat_bug(chanerr)); | |
304 | } | |
305 | ||
306 | /* if we haven't made progress and we have already | |
307 | * acknowledged a pending completion once, then be more | |
308 | * forceful with a restart | |
309 | */ | |
310 | if (ioat_cleanup_preamble(chan, &phys_complete)) | |
311 | __cleanup(ioat, phys_complete); | |
312 | else if (test_bit(IOAT_COMPLETION_ACK, &chan->state)) | |
313 | ioat3_restart_channel(ioat); | |
314 | else { | |
315 | set_bit(IOAT_COMPLETION_ACK, &chan->state); | |
316 | mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); | |
317 | } | |
318 | spin_unlock_bh(&ioat->ring_lock); | |
319 | } else { | |
320 | u16 active; | |
321 | ||
322 | /* if the ring is idle, empty, and oversized try to step | |
323 | * down the size | |
324 | */ | |
325 | spin_lock_bh(&ioat->ring_lock); | |
326 | active = ioat2_ring_active(ioat); | |
327 | if (active == 0 && ioat->alloc_order > ioat_get_alloc_order()) | |
328 | reshape_ring(ioat, ioat->alloc_order-1); | |
329 | spin_unlock_bh(&ioat->ring_lock); | |
330 | ||
331 | /* keep shrinking until we get back to our minimum | |
332 | * default size | |
333 | */ | |
334 | if (ioat->alloc_order > ioat_get_alloc_order()) | |
335 | mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); | |
336 | } | |
337 | spin_unlock_bh(&chan->cleanup_lock); | |
338 | } | |
339 | ||
340 | static enum dma_status | |
341 | ioat3_is_complete(struct dma_chan *c, dma_cookie_t cookie, | |
342 | dma_cookie_t *done, dma_cookie_t *used) | |
343 | { | |
344 | struct ioat2_dma_chan *ioat = to_ioat2_chan(c); | |
345 | ||
346 | if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS) | |
347 | return DMA_SUCCESS; | |
348 | ||
349 | ioat3_cleanup(ioat); | |
350 | ||
351 | return ioat_is_complete(c, cookie, done, used); | |
352 | } | |
353 | ||
354 | static struct dma_async_tx_descriptor * | |
355 | ioat3_prep_memset_lock(struct dma_chan *c, dma_addr_t dest, int value, | |
356 | size_t len, unsigned long flags) | |
357 | { | |
358 | struct ioat2_dma_chan *ioat = to_ioat2_chan(c); | |
359 | struct ioat_ring_ent *desc; | |
360 | size_t total_len = len; | |
361 | struct ioat_fill_descriptor *fill; | |
362 | int num_descs; | |
363 | u64 src_data = (0x0101010101010101ULL) * (value & 0xff); | |
364 | u16 idx; | |
365 | int i; | |
366 | ||
367 | num_descs = ioat2_xferlen_to_descs(ioat, len); | |
368 | if (likely(num_descs) && | |
369 | ioat2_alloc_and_lock(&idx, ioat, num_descs) == 0) | |
370 | /* pass */; | |
371 | else | |
372 | return NULL; | |
373 | for (i = 0; i < num_descs; i++) { | |
374 | size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log); | |
375 | ||
376 | desc = ioat2_get_ring_ent(ioat, idx + i); | |
377 | fill = desc->fill; | |
378 | ||
379 | fill->size = xfer_size; | |
380 | fill->src_data = src_data; | |
381 | fill->dst_addr = dest; | |
382 | fill->ctl = 0; | |
383 | fill->ctl_f.op = IOAT_OP_FILL; | |
384 | ||
385 | len -= xfer_size; | |
386 | dest += xfer_size; | |
387 | dump_desc_dbg(ioat, desc); | |
388 | } | |
389 | ||
390 | desc->txd.flags = flags; | |
391 | desc->len = total_len; | |
392 | fill->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); | |
393 | fill->ctl_f.fence = !!(flags & DMA_PREP_FENCE); | |
394 | fill->ctl_f.compl_write = 1; | |
395 | dump_desc_dbg(ioat, desc); | |
396 | ||
397 | /* we leave the channel locked to ensure in order submission */ | |
398 | return &desc->txd; | |
399 | } | |
400 | ||
b094ad3b DW |
401 | static struct dma_async_tx_descriptor * |
402 | __ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result, | |
403 | dma_addr_t dest, dma_addr_t *src, unsigned int src_cnt, | |
404 | size_t len, unsigned long flags) | |
405 | { | |
406 | struct ioat2_dma_chan *ioat = to_ioat2_chan(c); | |
407 | struct ioat_ring_ent *compl_desc; | |
408 | struct ioat_ring_ent *desc; | |
409 | struct ioat_ring_ent *ext; | |
410 | size_t total_len = len; | |
411 | struct ioat_xor_descriptor *xor; | |
412 | struct ioat_xor_ext_descriptor *xor_ex = NULL; | |
413 | struct ioat_dma_descriptor *hw; | |
414 | u32 offset = 0; | |
415 | int num_descs; | |
416 | int with_ext; | |
417 | int i; | |
418 | u16 idx; | |
419 | u8 op = result ? IOAT_OP_XOR_VAL : IOAT_OP_XOR; | |
420 | ||
421 | BUG_ON(src_cnt < 2); | |
422 | ||
423 | num_descs = ioat2_xferlen_to_descs(ioat, len); | |
424 | /* we need 2x the number of descriptors to cover greater than 5 | |
425 | * sources | |
426 | */ | |
427 | if (src_cnt > 5) { | |
428 | with_ext = 1; | |
429 | num_descs *= 2; | |
430 | } else | |
431 | with_ext = 0; | |
432 | ||
433 | /* completion writes from the raid engine may pass completion | |
434 | * writes from the legacy engine, so we need one extra null | |
435 | * (legacy) descriptor to ensure all completion writes arrive in | |
436 | * order. | |
437 | */ | |
438 | if (likely(num_descs) && | |
439 | ioat2_alloc_and_lock(&idx, ioat, num_descs+1) == 0) | |
440 | /* pass */; | |
441 | else | |
442 | return NULL; | |
443 | for (i = 0; i < num_descs; i += 1 + with_ext) { | |
444 | struct ioat_raw_descriptor *descs[2]; | |
445 | size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log); | |
446 | int s; | |
447 | ||
448 | desc = ioat2_get_ring_ent(ioat, idx + i); | |
449 | xor = desc->xor; | |
450 | ||
451 | /* save a branch by unconditionally retrieving the | |
452 | * extended descriptor xor_set_src() knows to not write | |
453 | * to it in the single descriptor case | |
454 | */ | |
455 | ext = ioat2_get_ring_ent(ioat, idx + i + 1); | |
456 | xor_ex = ext->xor_ex; | |
457 | ||
458 | descs[0] = (struct ioat_raw_descriptor *) xor; | |
459 | descs[1] = (struct ioat_raw_descriptor *) xor_ex; | |
460 | for (s = 0; s < src_cnt; s++) | |
461 | xor_set_src(descs, src[s], offset, s); | |
462 | xor->size = xfer_size; | |
463 | xor->dst_addr = dest + offset; | |
464 | xor->ctl = 0; | |
465 | xor->ctl_f.op = op; | |
466 | xor->ctl_f.src_cnt = src_cnt_to_hw(src_cnt); | |
467 | ||
468 | len -= xfer_size; | |
469 | offset += xfer_size; | |
470 | dump_desc_dbg(ioat, desc); | |
471 | } | |
472 | ||
473 | /* last xor descriptor carries the unmap parameters and fence bit */ | |
474 | desc->txd.flags = flags; | |
475 | desc->len = total_len; | |
476 | if (result) | |
477 | desc->result = result; | |
478 | xor->ctl_f.fence = !!(flags & DMA_PREP_FENCE); | |
479 | ||
480 | /* completion descriptor carries interrupt bit */ | |
481 | compl_desc = ioat2_get_ring_ent(ioat, idx + i); | |
482 | compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT; | |
483 | hw = compl_desc->hw; | |
484 | hw->ctl = 0; | |
485 | hw->ctl_f.null = 1; | |
486 | hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); | |
487 | hw->ctl_f.compl_write = 1; | |
488 | hw->size = NULL_DESC_BUFFER_SIZE; | |
489 | dump_desc_dbg(ioat, compl_desc); | |
490 | ||
491 | /* we leave the channel locked to ensure in order submission */ | |
492 | return &desc->txd; | |
493 | } | |
494 | ||
495 | static struct dma_async_tx_descriptor * | |
496 | ioat3_prep_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, | |
497 | unsigned int src_cnt, size_t len, unsigned long flags) | |
498 | { | |
499 | return __ioat3_prep_xor_lock(chan, NULL, dest, src, src_cnt, len, flags); | |
500 | } | |
501 | ||
502 | struct dma_async_tx_descriptor * | |
503 | ioat3_prep_xor_val(struct dma_chan *chan, dma_addr_t *src, | |
504 | unsigned int src_cnt, size_t len, | |
505 | enum sum_check_flags *result, unsigned long flags) | |
506 | { | |
507 | /* the cleanup routine only sets bits on validate failure, it | |
508 | * does not clear bits on validate success... so clear it here | |
509 | */ | |
510 | *result = 0; | |
511 | ||
512 | return __ioat3_prep_xor_lock(chan, result, src[0], &src[1], | |
513 | src_cnt - 1, len, flags); | |
514 | } | |
515 | ||
bf40a686 DW |
516 | int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca) |
517 | { | |
518 | struct pci_dev *pdev = device->pdev; | |
519 | struct dma_device *dma; | |
520 | struct dma_chan *c; | |
521 | struct ioat_chan_common *chan; | |
522 | int err; | |
523 | u16 dev_id; | |
524 | u32 cap; | |
525 | ||
526 | device->enumerate_channels = ioat2_enumerate_channels; | |
527 | device->cleanup_tasklet = ioat3_cleanup_tasklet; | |
528 | device->timer_fn = ioat3_timer_event; | |
529 | dma = &device->common; | |
530 | dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock; | |
531 | dma->device_issue_pending = ioat2_issue_pending; | |
532 | dma->device_alloc_chan_resources = ioat2_alloc_chan_resources; | |
533 | dma->device_free_chan_resources = ioat2_free_chan_resources; | |
534 | dma->device_is_tx_complete = ioat3_is_complete; | |
535 | cap = readl(device->reg_base + IOAT_DMA_CAP_OFFSET); | |
536 | if (cap & IOAT_CAP_FILL_BLOCK) { | |
537 | dma_cap_set(DMA_MEMSET, dma->cap_mask); | |
538 | dma->device_prep_dma_memset = ioat3_prep_memset_lock; | |
539 | } | |
b094ad3b DW |
540 | if (cap & IOAT_CAP_XOR) { |
541 | dma->max_xor = 8; | |
542 | dma->xor_align = 2; | |
543 | ||
544 | dma_cap_set(DMA_XOR, dma->cap_mask); | |
545 | dma->device_prep_dma_xor = ioat3_prep_xor; | |
546 | ||
547 | dma_cap_set(DMA_XOR_VAL, dma->cap_mask); | |
548 | dma->device_prep_dma_xor_val = ioat3_prep_xor_val; | |
549 | } | |
bf40a686 DW |
550 | |
551 | /* -= IOAT ver.3 workarounds =- */ | |
552 | /* Write CHANERRMSK_INT with 3E07h to mask out the errors | |
553 | * that can cause stability issues for IOAT ver.3 | |
554 | */ | |
555 | pci_write_config_dword(pdev, IOAT_PCI_CHANERRMASK_INT_OFFSET, 0x3e07); | |
556 | ||
557 | /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit | |
558 | * (workaround for spurious config parity error after restart) | |
559 | */ | |
560 | pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id); | |
561 | if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0) | |
562 | pci_write_config_dword(pdev, IOAT_PCI_DMAUNCERRSTS_OFFSET, 0x10); | |
563 | ||
564 | err = ioat_probe(device); | |
565 | if (err) | |
566 | return err; | |
567 | ioat_set_tcp_copy_break(262144); | |
568 | ||
569 | list_for_each_entry(c, &dma->channels, device_node) { | |
570 | chan = to_chan_common(c); | |
571 | writel(IOAT_DMA_DCA_ANY_CPU, | |
572 | chan->reg_base + IOAT_DCACTRL_OFFSET); | |
573 | } | |
574 | ||
575 | err = ioat_register(device); | |
576 | if (err) | |
577 | return err; | |
5669e31c DW |
578 | |
579 | ioat_kobject_add(device, &ioat2_ktype); | |
580 | ||
bf40a686 DW |
581 | if (dca) |
582 | device->dca = ioat3_dca_init(pdev, device->reg_base); | |
583 | ||
584 | return 0; | |
585 | } |