]>
Commit | Line | Data |
---|---|---|
7a291083 JBT |
1 | /* |
2 | * linux/drivers/net/ehea/ehea_qmr.c | |
3 | * | |
4 | * eHEA ethernet device driver for IBM eServer System p | |
5 | * | |
6 | * (C) Copyright IBM Corp. 2006 | |
7 | * | |
8 | * Authors: | |
9 | * Christoph Raisch <raisch@de.ibm.com> | |
10 | * Jan-Bernd Themann <themann@de.ibm.com> | |
11 | * Thomas Klein <tklein@de.ibm.com> | |
12 | * | |
13 | * | |
14 | * This program is free software; you can redistribute it and/or modify | |
15 | * it under the terms of the GNU General Public License as published by | |
16 | * the Free Software Foundation; either version 2, or (at your option) | |
17 | * any later version. | |
18 | * | |
19 | * This program is distributed in the hope that it will be useful, | |
20 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
21 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
22 | * GNU General Public License for more details. | |
23 | * | |
24 | * You should have received a copy of the GNU General Public License | |
25 | * along with this program; if not, write to the Free Software | |
26 | * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. | |
27 | */ | |
28 | ||
d7fe0f24 | 29 | #include <linux/mm.h> |
5a0e3ad6 | 30 | #include <linux/slab.h> |
7a291083 JBT |
31 | #include "ehea.h" |
32 | #include "ehea_phyp.h" | |
33 | #include "ehea_qmr.h" | |
34 | ||
48cfb14f | 35 | struct ehea_bmap *ehea_bmap = NULL; |
44c82152 | 36 | |
44c82152 TK |
37 | |
38 | ||
7a291083 JBT |
39 | static void *hw_qpageit_get_inc(struct hw_queue *queue) |
40 | { | |
41 | void *retvalue = hw_qeit_get(queue); | |
42 | ||
43 | queue->current_q_offset += queue->pagesize; | |
44 | if (queue->current_q_offset > queue->queue_length) { | |
45 | queue->current_q_offset -= queue->pagesize; | |
46 | retvalue = NULL; | |
47 | } else if (((u64) retvalue) & (EHEA_PAGESIZE-1)) { | |
48 | ehea_error("not on pageboundary"); | |
49 | retvalue = NULL; | |
50 | } | |
51 | return retvalue; | |
52 | } | |
53 | ||
54 | static int hw_queue_ctor(struct hw_queue *queue, const u32 nr_of_pages, | |
55 | const u32 pagesize, const u32 qe_size) | |
56 | { | |
57 | int pages_per_kpage = PAGE_SIZE / pagesize; | |
58 | int i, k; | |
59 | ||
60 | if ((pagesize > PAGE_SIZE) || (!pages_per_kpage)) { | |
61 | ehea_error("pagesize conflict! kernel pagesize=%d, " | |
62 | "ehea pagesize=%d", (int)PAGE_SIZE, (int)pagesize); | |
63 | return -EINVAL; | |
64 | } | |
65 | ||
66 | queue->queue_length = nr_of_pages * pagesize; | |
f67c6275 | 67 | queue->queue_pages = kmalloc(nr_of_pages * sizeof(void *), GFP_KERNEL); |
7a291083 JBT |
68 | if (!queue->queue_pages) { |
69 | ehea_error("no mem for queue_pages"); | |
70 | return -ENOMEM; | |
71 | } | |
72 | ||
73 | /* | |
74 | * allocate pages for queue: | |
75 | * outer loop allocates whole kernel pages (page aligned) and | |
76 | * inner loop divides a kernel page into smaller hea queue pages | |
77 | */ | |
78 | i = 0; | |
79 | while (i < nr_of_pages) { | |
f67c6275 | 80 | u8 *kpage = (u8 *)get_zeroed_page(GFP_KERNEL); |
7a291083 JBT |
81 | if (!kpage) |
82 | goto out_nomem; | |
83 | for (k = 0; k < pages_per_kpage && i < nr_of_pages; k++) { | |
f67c6275 | 84 | (queue->queue_pages)[i] = (struct ehea_page *)kpage; |
7a291083 JBT |
85 | kpage += pagesize; |
86 | i++; | |
87 | } | |
88 | } | |
89 | ||
90 | queue->current_q_offset = 0; | |
91 | queue->qe_size = qe_size; | |
92 | queue->pagesize = pagesize; | |
93 | queue->toggle_state = 1; | |
94 | ||
95 | return 0; | |
96 | out_nomem: | |
97 | for (i = 0; i < nr_of_pages; i += pages_per_kpage) { | |
98 | if (!(queue->queue_pages)[i]) | |
99 | break; | |
100 | free_page((unsigned long)(queue->queue_pages)[i]); | |
101 | } | |
102 | return -ENOMEM; | |
103 | } | |
104 | ||
105 | static void hw_queue_dtor(struct hw_queue *queue) | |
106 | { | |
107 | int pages_per_kpage = PAGE_SIZE / queue->pagesize; | |
108 | int i, nr_pages; | |
109 | ||
110 | if (!queue || !queue->queue_pages) | |
111 | return; | |
112 | ||
113 | nr_pages = queue->queue_length / queue->pagesize; | |
114 | ||
115 | for (i = 0; i < nr_pages; i += pages_per_kpage) | |
116 | free_page((unsigned long)(queue->queue_pages)[i]); | |
117 | ||
118 | kfree(queue->queue_pages); | |
119 | } | |
120 | ||
121 | struct ehea_cq *ehea_create_cq(struct ehea_adapter *adapter, | |
122 | int nr_of_cqe, u64 eq_handle, u32 cq_token) | |
123 | { | |
124 | struct ehea_cq *cq; | |
125 | struct h_epa epa; | |
126 | u64 *cq_handle_ref, hret, rpage; | |
127 | u32 act_nr_of_entries, act_pages, counter; | |
128 | int ret; | |
129 | void *vpage; | |
130 | ||
131 | cq = kzalloc(sizeof(*cq), GFP_KERNEL); | |
132 | if (!cq) { | |
133 | ehea_error("no mem for cq"); | |
134 | goto out_nomem; | |
135 | } | |
136 | ||
137 | cq->attr.max_nr_of_cqes = nr_of_cqe; | |
138 | cq->attr.cq_token = cq_token; | |
139 | cq->attr.eq_handle = eq_handle; | |
140 | ||
141 | cq->adapter = adapter; | |
142 | ||
143 | cq_handle_ref = &cq->fw_handle; | |
144 | act_nr_of_entries = 0; | |
145 | act_pages = 0; | |
146 | ||
147 | hret = ehea_h_alloc_resource_cq(adapter->handle, &cq->attr, | |
148 | &cq->fw_handle, &cq->epas); | |
149 | if (hret != H_SUCCESS) { | |
150 | ehea_error("alloc_resource_cq failed"); | |
151 | goto out_freemem; | |
152 | } | |
153 | ||
154 | ret = hw_queue_ctor(&cq->hw_queue, cq->attr.nr_pages, | |
155 | EHEA_PAGESIZE, sizeof(struct ehea_cqe)); | |
156 | if (ret) | |
157 | goto out_freeres; | |
158 | ||
159 | for (counter = 0; counter < cq->attr.nr_pages; counter++) { | |
160 | vpage = hw_qpageit_get_inc(&cq->hw_queue); | |
161 | if (!vpage) { | |
162 | ehea_error("hw_qpageit_get_inc failed"); | |
163 | goto out_kill_hwq; | |
164 | } | |
165 | ||
166 | rpage = virt_to_abs(vpage); | |
167 | hret = ehea_h_register_rpage(adapter->handle, | |
168 | 0, EHEA_CQ_REGISTER_ORIG, | |
169 | cq->fw_handle, rpage, 1); | |
170 | if (hret < H_SUCCESS) { | |
171 | ehea_error("register_rpage_cq failed ehea_cq=%p " | |
a1c5a893 | 172 | "hret=%llx counter=%i act_pages=%i", |
7a291083 JBT |
173 | cq, hret, counter, cq->attr.nr_pages); |
174 | goto out_kill_hwq; | |
175 | } | |
176 | ||
177 | if (counter == (cq->attr.nr_pages - 1)) { | |
178 | vpage = hw_qpageit_get_inc(&cq->hw_queue); | |
179 | ||
180 | if ((hret != H_SUCCESS) || (vpage)) { | |
181 | ehea_error("registration of pages not " | |
a1c5a893 | 182 | "complete hret=%llx\n", hret); |
7a291083 JBT |
183 | goto out_kill_hwq; |
184 | } | |
185 | } else { | |
662f44af | 186 | if (hret != H_PAGE_REGISTERED) { |
7a291083 | 187 | ehea_error("CQ: registration of page failed " |
a1c5a893 | 188 | "hret=%llx\n", hret); |
7a291083 JBT |
189 | goto out_kill_hwq; |
190 | } | |
191 | } | |
192 | } | |
193 | ||
194 | hw_qeit_reset(&cq->hw_queue); | |
195 | epa = cq->epas.kernel; | |
196 | ehea_reset_cq_ep(cq); | |
197 | ehea_reset_cq_n1(cq); | |
198 | ||
199 | return cq; | |
200 | ||
201 | out_kill_hwq: | |
202 | hw_queue_dtor(&cq->hw_queue); | |
203 | ||
204 | out_freeres: | |
e542aa6b | 205 | ehea_h_free_resource(adapter->handle, cq->fw_handle, FORCE_FREE); |
7a291083 JBT |
206 | |
207 | out_freemem: | |
208 | kfree(cq); | |
209 | ||
210 | out_nomem: | |
211 | return NULL; | |
212 | } | |
213 | ||
e542aa6b | 214 | u64 ehea_destroy_cq_res(struct ehea_cq *cq, u64 force) |
7a291083 | 215 | { |
e542aa6b JBT |
216 | u64 hret; |
217 | u64 adapter_handle = cq->adapter->handle; | |
218 | ||
d1d25aab | 219 | /* deregister all previous registered pages */ |
e542aa6b JBT |
220 | hret = ehea_h_free_resource(adapter_handle, cq->fw_handle, force); |
221 | if (hret != H_SUCCESS) | |
222 | return hret; | |
223 | ||
224 | hw_queue_dtor(&cq->hw_queue); | |
225 | kfree(cq); | |
226 | ||
227 | return hret; | |
228 | } | |
7a291083 | 229 | |
e542aa6b JBT |
230 | int ehea_destroy_cq(struct ehea_cq *cq) |
231 | { | |
232 | u64 hret; | |
7a291083 JBT |
233 | if (!cq) |
234 | return 0; | |
235 | ||
28721c89 | 236 | hcp_epas_dtor(&cq->epas); |
f67c6275 DM |
237 | hret = ehea_destroy_cq_res(cq, NORMAL_FREE); |
238 | if (hret == H_R_STATE) { | |
e542aa6b JBT |
239 | ehea_error_data(cq->adapter, cq->fw_handle); |
240 | hret = ehea_destroy_cq_res(cq, FORCE_FREE); | |
241 | } | |
1b5135d9 | 242 | |
7a291083 JBT |
243 | if (hret != H_SUCCESS) { |
244 | ehea_error("destroy CQ failed"); | |
245 | return -EIO; | |
246 | } | |
247 | ||
7a291083 JBT |
248 | return 0; |
249 | } | |
250 | ||
251 | struct ehea_eq *ehea_create_eq(struct ehea_adapter *adapter, | |
252 | const enum ehea_eq_type type, | |
253 | const u32 max_nr_of_eqes, const u8 eqe_gen) | |
254 | { | |
255 | int ret, i; | |
256 | u64 hret, rpage; | |
257 | void *vpage; | |
258 | struct ehea_eq *eq; | |
259 | ||
260 | eq = kzalloc(sizeof(*eq), GFP_KERNEL); | |
261 | if (!eq) { | |
262 | ehea_error("no mem for eq"); | |
263 | return NULL; | |
264 | } | |
265 | ||
266 | eq->adapter = adapter; | |
267 | eq->attr.type = type; | |
268 | eq->attr.max_nr_of_eqes = max_nr_of_eqes; | |
269 | eq->attr.eqe_gen = eqe_gen; | |
270 | spin_lock_init(&eq->spinlock); | |
271 | ||
272 | hret = ehea_h_alloc_resource_eq(adapter->handle, | |
273 | &eq->attr, &eq->fw_handle); | |
274 | if (hret != H_SUCCESS) { | |
275 | ehea_error("alloc_resource_eq failed"); | |
276 | goto out_freemem; | |
277 | } | |
278 | ||
279 | ret = hw_queue_ctor(&eq->hw_queue, eq->attr.nr_pages, | |
280 | EHEA_PAGESIZE, sizeof(struct ehea_eqe)); | |
281 | if (ret) { | |
282 | ehea_error("can't allocate eq pages"); | |
283 | goto out_freeres; | |
284 | } | |
285 | ||
286 | for (i = 0; i < eq->attr.nr_pages; i++) { | |
287 | vpage = hw_qpageit_get_inc(&eq->hw_queue); | |
288 | if (!vpage) { | |
289 | ehea_error("hw_qpageit_get_inc failed"); | |
290 | hret = H_RESOURCE; | |
291 | goto out_kill_hwq; | |
292 | } | |
293 | ||
294 | rpage = virt_to_abs(vpage); | |
295 | ||
296 | hret = ehea_h_register_rpage(adapter->handle, 0, | |
297 | EHEA_EQ_REGISTER_ORIG, | |
298 | eq->fw_handle, rpage, 1); | |
299 | ||
300 | if (i == (eq->attr.nr_pages - 1)) { | |
301 | /* last page */ | |
302 | vpage = hw_qpageit_get_inc(&eq->hw_queue); | |
f67c6275 | 303 | if ((hret != H_SUCCESS) || (vpage)) |
7a291083 | 304 | goto out_kill_hwq; |
f67c6275 | 305 | |
7a291083 | 306 | } else { |
662f44af | 307 | if (hret != H_PAGE_REGISTERED) |
7a291083 | 308 | goto out_kill_hwq; |
f67c6275 | 309 | |
7a291083 JBT |
310 | } |
311 | } | |
312 | ||
313 | hw_qeit_reset(&eq->hw_queue); | |
314 | return eq; | |
315 | ||
316 | out_kill_hwq: | |
317 | hw_queue_dtor(&eq->hw_queue); | |
318 | ||
319 | out_freeres: | |
e542aa6b | 320 | ehea_h_free_resource(adapter->handle, eq->fw_handle, FORCE_FREE); |
7a291083 JBT |
321 | |
322 | out_freemem: | |
323 | kfree(eq); | |
324 | return NULL; | |
325 | } | |
326 | ||
327 | struct ehea_eqe *ehea_poll_eq(struct ehea_eq *eq) | |
328 | { | |
329 | struct ehea_eqe *eqe; | |
330 | unsigned long flags; | |
331 | ||
332 | spin_lock_irqsave(&eq->spinlock, flags); | |
f67c6275 | 333 | eqe = (struct ehea_eqe *)hw_eqit_eq_get_inc_valid(&eq->hw_queue); |
7a291083 JBT |
334 | spin_unlock_irqrestore(&eq->spinlock, flags); |
335 | ||
336 | return eqe; | |
337 | } | |
338 | ||
e542aa6b | 339 | u64 ehea_destroy_eq_res(struct ehea_eq *eq, u64 force) |
7a291083 JBT |
340 | { |
341 | u64 hret; | |
342 | unsigned long flags; | |
343 | ||
7a291083 JBT |
344 | spin_lock_irqsave(&eq->spinlock, flags); |
345 | ||
e542aa6b | 346 | hret = ehea_h_free_resource(eq->adapter->handle, eq->fw_handle, force); |
7a291083 JBT |
347 | spin_unlock_irqrestore(&eq->spinlock, flags); |
348 | ||
e542aa6b JBT |
349 | if (hret != H_SUCCESS) |
350 | return hret; | |
7a291083 JBT |
351 | |
352 | hw_queue_dtor(&eq->hw_queue); | |
353 | kfree(eq); | |
354 | ||
e542aa6b JBT |
355 | return hret; |
356 | } | |
357 | ||
358 | int ehea_destroy_eq(struct ehea_eq *eq) | |
359 | { | |
360 | u64 hret; | |
361 | if (!eq) | |
362 | return 0; | |
363 | ||
28721c89 JBT |
364 | hcp_epas_dtor(&eq->epas); |
365 | ||
f67c6275 DM |
366 | hret = ehea_destroy_eq_res(eq, NORMAL_FREE); |
367 | if (hret == H_R_STATE) { | |
e542aa6b JBT |
368 | ehea_error_data(eq->adapter, eq->fw_handle); |
369 | hret = ehea_destroy_eq_res(eq, FORCE_FREE); | |
370 | } | |
371 | ||
372 | if (hret != H_SUCCESS) { | |
373 | ehea_error("destroy EQ failed"); | |
374 | return -EIO; | |
d1d25aab | 375 | } |
e542aa6b | 376 | |
7a291083 JBT |
377 | return 0; |
378 | } | |
379 | ||
380 | /** | |
381 | * allocates memory for a queue and registers pages in phyp | |
382 | */ | |
383 | int ehea_qp_alloc_register(struct ehea_qp *qp, struct hw_queue *hw_queue, | |
384 | int nr_pages, int wqe_size, int act_nr_sges, | |
385 | struct ehea_adapter *adapter, int h_call_q_selector) | |
386 | { | |
387 | u64 hret, rpage; | |
388 | int ret, cnt; | |
389 | void *vpage; | |
390 | ||
391 | ret = hw_queue_ctor(hw_queue, nr_pages, EHEA_PAGESIZE, wqe_size); | |
392 | if (ret) | |
393 | return ret; | |
394 | ||
395 | for (cnt = 0; cnt < nr_pages; cnt++) { | |
396 | vpage = hw_qpageit_get_inc(hw_queue); | |
397 | if (!vpage) { | |
398 | ehea_error("hw_qpageit_get_inc failed"); | |
399 | goto out_kill_hwq; | |
400 | } | |
401 | rpage = virt_to_abs(vpage); | |
402 | hret = ehea_h_register_rpage(adapter->handle, | |
403 | 0, h_call_q_selector, | |
404 | qp->fw_handle, rpage, 1); | |
405 | if (hret < H_SUCCESS) { | |
406 | ehea_error("register_rpage_qp failed"); | |
407 | goto out_kill_hwq; | |
408 | } | |
409 | } | |
410 | hw_qeit_reset(hw_queue); | |
411 | return 0; | |
412 | ||
413 | out_kill_hwq: | |
414 | hw_queue_dtor(hw_queue); | |
415 | return -EIO; | |
416 | } | |
417 | ||
418 | static inline u32 map_wqe_size(u8 wqe_enc_size) | |
419 | { | |
420 | return 128 << wqe_enc_size; | |
421 | } | |
422 | ||
423 | struct ehea_qp *ehea_create_qp(struct ehea_adapter *adapter, | |
424 | u32 pd, struct ehea_qp_init_attr *init_attr) | |
425 | { | |
426 | int ret; | |
427 | u64 hret; | |
428 | struct ehea_qp *qp; | |
429 | u32 wqe_size_in_bytes_sq, wqe_size_in_bytes_rq1; | |
430 | u32 wqe_size_in_bytes_rq2, wqe_size_in_bytes_rq3; | |
431 | ||
432 | ||
433 | qp = kzalloc(sizeof(*qp), GFP_KERNEL); | |
434 | if (!qp) { | |
435 | ehea_error("no mem for qp"); | |
436 | return NULL; | |
437 | } | |
438 | ||
439 | qp->adapter = adapter; | |
440 | ||
441 | hret = ehea_h_alloc_resource_qp(adapter->handle, init_attr, pd, | |
442 | &qp->fw_handle, &qp->epas); | |
443 | if (hret != H_SUCCESS) { | |
444 | ehea_error("ehea_h_alloc_resource_qp failed"); | |
445 | goto out_freemem; | |
446 | } | |
447 | ||
448 | wqe_size_in_bytes_sq = map_wqe_size(init_attr->act_wqe_size_enc_sq); | |
449 | wqe_size_in_bytes_rq1 = map_wqe_size(init_attr->act_wqe_size_enc_rq1); | |
450 | wqe_size_in_bytes_rq2 = map_wqe_size(init_attr->act_wqe_size_enc_rq2); | |
451 | wqe_size_in_bytes_rq3 = map_wqe_size(init_attr->act_wqe_size_enc_rq3); | |
452 | ||
453 | ret = ehea_qp_alloc_register(qp, &qp->hw_squeue, init_attr->nr_sq_pages, | |
454 | wqe_size_in_bytes_sq, | |
455 | init_attr->act_wqe_size_enc_sq, adapter, | |
456 | 0); | |
457 | if (ret) { | |
458 | ehea_error("can't register for sq ret=%x", ret); | |
459 | goto out_freeres; | |
460 | } | |
461 | ||
462 | ret = ehea_qp_alloc_register(qp, &qp->hw_rqueue1, | |
463 | init_attr->nr_rq1_pages, | |
464 | wqe_size_in_bytes_rq1, | |
465 | init_attr->act_wqe_size_enc_rq1, | |
466 | adapter, 1); | |
467 | if (ret) { | |
468 | ehea_error("can't register for rq1 ret=%x", ret); | |
469 | goto out_kill_hwsq; | |
470 | } | |
471 | ||
472 | if (init_attr->rq_count > 1) { | |
473 | ret = ehea_qp_alloc_register(qp, &qp->hw_rqueue2, | |
474 | init_attr->nr_rq2_pages, | |
475 | wqe_size_in_bytes_rq2, | |
476 | init_attr->act_wqe_size_enc_rq2, | |
477 | adapter, 2); | |
478 | if (ret) { | |
479 | ehea_error("can't register for rq2 ret=%x", ret); | |
480 | goto out_kill_hwr1q; | |
481 | } | |
482 | } | |
483 | ||
484 | if (init_attr->rq_count > 2) { | |
485 | ret = ehea_qp_alloc_register(qp, &qp->hw_rqueue3, | |
486 | init_attr->nr_rq3_pages, | |
487 | wqe_size_in_bytes_rq3, | |
488 | init_attr->act_wqe_size_enc_rq3, | |
489 | adapter, 3); | |
490 | if (ret) { | |
491 | ehea_error("can't register for rq3 ret=%x", ret); | |
492 | goto out_kill_hwr2q; | |
493 | } | |
494 | } | |
495 | ||
496 | qp->init_attr = *init_attr; | |
497 | ||
498 | return qp; | |
499 | ||
500 | out_kill_hwr2q: | |
501 | hw_queue_dtor(&qp->hw_rqueue2); | |
502 | ||
503 | out_kill_hwr1q: | |
504 | hw_queue_dtor(&qp->hw_rqueue1); | |
505 | ||
506 | out_kill_hwsq: | |
507 | hw_queue_dtor(&qp->hw_squeue); | |
508 | ||
509 | out_freeres: | |
510 | ehea_h_disable_and_get_hea(adapter->handle, qp->fw_handle); | |
e542aa6b | 511 | ehea_h_free_resource(adapter->handle, qp->fw_handle, FORCE_FREE); |
7a291083 JBT |
512 | |
513 | out_freemem: | |
514 | kfree(qp); | |
515 | return NULL; | |
516 | } | |
517 | ||
e542aa6b | 518 | u64 ehea_destroy_qp_res(struct ehea_qp *qp, u64 force) |
7a291083 | 519 | { |
d1d25aab JBT |
520 | u64 hret; |
521 | struct ehea_qp_init_attr *qp_attr = &qp->init_attr; | |
7a291083 | 522 | |
7a291083 | 523 | |
d1d25aab JBT |
524 | ehea_h_disable_and_get_hea(qp->adapter->handle, qp->fw_handle); |
525 | hret = ehea_h_free_resource(qp->adapter->handle, qp->fw_handle, force); | |
526 | if (hret != H_SUCCESS) | |
527 | return hret; | |
7a291083 | 528 | |
d1d25aab JBT |
529 | hw_queue_dtor(&qp->hw_squeue); |
530 | hw_queue_dtor(&qp->hw_rqueue1); | |
7a291083 | 531 | |
d1d25aab JBT |
532 | if (qp_attr->rq_count > 1) |
533 | hw_queue_dtor(&qp->hw_rqueue2); | |
534 | if (qp_attr->rq_count > 2) | |
535 | hw_queue_dtor(&qp->hw_rqueue3); | |
536 | kfree(qp); | |
7a291083 | 537 | |
d1d25aab | 538 | return hret; |
7a291083 JBT |
539 | } |
540 | ||
e542aa6b JBT |
541 | int ehea_destroy_qp(struct ehea_qp *qp) |
542 | { | |
d1d25aab JBT |
543 | u64 hret; |
544 | if (!qp) | |
545 | return 0; | |
e542aa6b | 546 | |
28721c89 JBT |
547 | hcp_epas_dtor(&qp->epas); |
548 | ||
f67c6275 DM |
549 | hret = ehea_destroy_qp_res(qp, NORMAL_FREE); |
550 | if (hret == H_R_STATE) { | |
d1d25aab JBT |
551 | ehea_error_data(qp->adapter, qp->fw_handle); |
552 | hret = ehea_destroy_qp_res(qp, FORCE_FREE); | |
553 | } | |
e542aa6b | 554 | |
d1d25aab JBT |
555 | if (hret != H_SUCCESS) { |
556 | ehea_error("destroy QP failed"); | |
557 | return -EIO; | |
558 | } | |
e542aa6b | 559 | |
d1d25aab | 560 | return 0; |
e542aa6b JBT |
561 | } |
562 | ||
48cfb14f | 563 | static inline int ehea_calc_index(unsigned long i, unsigned long s) |
44c82152 | 564 | { |
48cfb14f HH |
565 | return (i >> s) & EHEA_INDEX_MASK; |
566 | } | |
44c82152 | 567 | |
48cfb14f HH |
568 | static inline int ehea_init_top_bmap(struct ehea_top_bmap *ehea_top_bmap, |
569 | int dir) | |
570 | { | |
d4f12daf | 571 | if (!ehea_top_bmap->dir[dir]) { |
48cfb14f HH |
572 | ehea_top_bmap->dir[dir] = |
573 | kzalloc(sizeof(struct ehea_dir_bmap), GFP_KERNEL); | |
574 | if (!ehea_top_bmap->dir[dir]) | |
575 | return -ENOMEM; | |
576 | } | |
577 | return 0; | |
578 | } | |
44c82152 | 579 | |
48cfb14f HH |
580 | static inline int ehea_init_bmap(struct ehea_bmap *ehea_bmap, int top, int dir) |
581 | { | |
d4f12daf | 582 | if (!ehea_bmap->top[top]) { |
48cfb14f HH |
583 | ehea_bmap->top[top] = |
584 | kzalloc(sizeof(struct ehea_top_bmap), GFP_KERNEL); | |
585 | if (!ehea_bmap->top[top]) | |
586 | return -ENOMEM; | |
587 | } | |
588 | return ehea_init_top_bmap(ehea_bmap->top[top], dir); | |
589 | } | |
44c82152 | 590 | |
d4f12daf HH |
591 | static DEFINE_MUTEX(ehea_busmap_mutex); |
592 | static unsigned long ehea_mr_len; | |
44c82152 | 593 | |
d4f12daf HH |
594 | #define EHEA_BUSMAP_ADD_SECT 1 |
595 | #define EHEA_BUSMAP_REM_SECT 0 | |
44c82152 | 596 | |
d4f12daf HH |
597 | static void ehea_rebuild_busmap(void) |
598 | { | |
599 | u64 vaddr = EHEA_BUSMAP_START; | |
600 | int top, dir, idx; | |
48cfb14f | 601 | |
d4f12daf HH |
602 | for (top = 0; top < EHEA_MAP_ENTRIES; top++) { |
603 | struct ehea_top_bmap *ehea_top; | |
604 | int valid_dir_entries = 0; | |
48cfb14f | 605 | |
d4f12daf HH |
606 | if (!ehea_bmap->top[top]) |
607 | continue; | |
608 | ehea_top = ehea_bmap->top[top]; | |
609 | for (dir = 0; dir < EHEA_MAP_ENTRIES; dir++) { | |
610 | struct ehea_dir_bmap *ehea_dir; | |
611 | int valid_entries = 0; | |
612 | ||
613 | if (!ehea_top->dir[dir]) | |
614 | continue; | |
615 | valid_dir_entries++; | |
616 | ehea_dir = ehea_top->dir[dir]; | |
617 | for (idx = 0; idx < EHEA_MAP_ENTRIES; idx++) { | |
618 | if (!ehea_dir->ent[idx]) | |
619 | continue; | |
620 | valid_entries++; | |
621 | ehea_dir->ent[idx] = vaddr; | |
622 | vaddr += EHEA_SECTSIZE; | |
623 | } | |
624 | if (!valid_entries) { | |
625 | ehea_top->dir[dir] = NULL; | |
626 | kfree(ehea_dir); | |
627 | } | |
628 | } | |
629 | if (!valid_dir_entries) { | |
630 | ehea_bmap->top[top] = NULL; | |
631 | kfree(ehea_top); | |
632 | } | |
633 | } | |
634 | } | |
48cfb14f | 635 | |
3fd09c45 | 636 | static int ehea_update_busmap(unsigned long pfn, unsigned long nr_pages, int add) |
d4f12daf HH |
637 | { |
638 | unsigned long i, start_section, end_section; | |
44c82152 | 639 | |
3fd09c45 TK |
640 | if (!nr_pages) |
641 | return 0; | |
642 | ||
d4f12daf HH |
643 | if (!ehea_bmap) { |
644 | ehea_bmap = kzalloc(sizeof(struct ehea_bmap), GFP_KERNEL); | |
645 | if (!ehea_bmap) | |
646 | return -ENOMEM; | |
44c82152 TK |
647 | } |
648 | ||
d4f12daf | 649 | start_section = (pfn * PAGE_SIZE) / EHEA_SECTSIZE; |
3fd09c45 | 650 | end_section = start_section + ((nr_pages * PAGE_SIZE) / EHEA_SECTSIZE); |
d4f12daf HH |
651 | /* Mark entries as valid or invalid only; address is assigned later */ |
652 | for (i = start_section; i < end_section; i++) { | |
653 | u64 flag; | |
654 | int top = ehea_calc_index(i, EHEA_TOP_INDEX_SHIFT); | |
655 | int dir = ehea_calc_index(i, EHEA_DIR_INDEX_SHIFT); | |
656 | int idx = i & EHEA_INDEX_MASK; | |
c5916cf8 | 657 | |
d4f12daf HH |
658 | if (add) { |
659 | int ret = ehea_init_bmap(ehea_bmap, top, dir); | |
660 | if (ret) | |
661 | return ret; | |
662 | flag = 1; /* valid */ | |
663 | ehea_mr_len += EHEA_SECTSIZE; | |
664 | } else { | |
665 | if (!ehea_bmap->top[top]) | |
666 | continue; | |
667 | if (!ehea_bmap->top[top]->dir[dir]) | |
668 | continue; | |
669 | flag = 0; /* invalid */ | |
670 | ehea_mr_len -= EHEA_SECTSIZE; | |
671 | } | |
48cfb14f | 672 | |
d4f12daf HH |
673 | ehea_bmap->top[top]->dir[dir]->ent[idx] = flag; |
674 | } | |
675 | ehea_rebuild_busmap(); /* Assign contiguous addresses for mr */ | |
44c82152 TK |
676 | return 0; |
677 | } | |
678 | ||
d4f12daf HH |
679 | int ehea_add_sect_bmap(unsigned long pfn, unsigned long nr_pages) |
680 | { | |
681 | int ret; | |
48cfb14f | 682 | |
d4f12daf HH |
683 | mutex_lock(&ehea_busmap_mutex); |
684 | ret = ehea_update_busmap(pfn, nr_pages, EHEA_BUSMAP_ADD_SECT); | |
685 | mutex_unlock(&ehea_busmap_mutex); | |
686 | return ret; | |
687 | } | |
688 | ||
689 | int ehea_rem_sect_bmap(unsigned long pfn, unsigned long nr_pages) | |
690 | { | |
691 | int ret; | |
692 | ||
693 | mutex_lock(&ehea_busmap_mutex); | |
694 | ret = ehea_update_busmap(pfn, nr_pages, EHEA_BUSMAP_REM_SECT); | |
695 | mutex_unlock(&ehea_busmap_mutex); | |
696 | return ret; | |
697 | } | |
698 | ||
3fd09c45 TK |
699 | static int ehea_is_hugepage(unsigned long pfn) |
700 | { | |
701 | int page_order; | |
702 | ||
703 | if (pfn & EHEA_HUGEPAGE_PFN_MASK) | |
704 | return 0; | |
705 | ||
706 | page_order = compound_order(pfn_to_page(pfn)); | |
707 | if (page_order + PAGE_SHIFT != EHEA_HUGEPAGESHIFT) | |
708 | return 0; | |
709 | ||
710 | return 1; | |
711 | } | |
712 | ||
713 | static int ehea_create_busmap_callback(unsigned long initial_pfn, | |
714 | unsigned long total_nr_pages, void *arg) | |
d4f12daf | 715 | { |
3fd09c45 TK |
716 | int ret; |
717 | unsigned long pfn, start_pfn, end_pfn, nr_pages; | |
718 | ||
719 | if ((total_nr_pages * PAGE_SIZE) < EHEA_HUGEPAGE_SIZE) | |
720 | return ehea_update_busmap(initial_pfn, total_nr_pages, | |
721 | EHEA_BUSMAP_ADD_SECT); | |
722 | ||
723 | /* Given chunk is >= 16GB -> check for hugepages */ | |
724 | start_pfn = initial_pfn; | |
725 | end_pfn = initial_pfn + total_nr_pages; | |
726 | pfn = start_pfn; | |
727 | ||
728 | while (pfn < end_pfn) { | |
729 | if (ehea_is_hugepage(pfn)) { | |
730 | /* Add mem found in front of the hugepage */ | |
731 | nr_pages = pfn - start_pfn; | |
732 | ret = ehea_update_busmap(start_pfn, nr_pages, | |
733 | EHEA_BUSMAP_ADD_SECT); | |
734 | if (ret) | |
735 | return ret; | |
736 | ||
737 | /* Skip the hugepage */ | |
738 | pfn += (EHEA_HUGEPAGE_SIZE / PAGE_SIZE); | |
739 | start_pfn = pfn; | |
740 | } else | |
741 | pfn += (EHEA_SECTSIZE / PAGE_SIZE); | |
742 | } | |
743 | ||
744 | /* Add mem found behind the hugepage(s) */ | |
745 | nr_pages = pfn - start_pfn; | |
746 | return ehea_update_busmap(start_pfn, nr_pages, EHEA_BUSMAP_ADD_SECT); | |
d4f12daf | 747 | } |
48cfb14f HH |
748 | |
749 | int ehea_create_busmap(void) | |
750 | { | |
751 | int ret; | |
d4f12daf | 752 | |
48cfb14f HH |
753 | mutex_lock(&ehea_busmap_mutex); |
754 | ehea_mr_len = 0; | |
908eedc6 | 755 | ret = walk_system_ram_range(0, 1ULL << MAX_PHYSMEM_BITS, NULL, |
48cfb14f HH |
756 | ehea_create_busmap_callback); |
757 | mutex_unlock(&ehea_busmap_mutex); | |
758 | return ret; | |
759 | } | |
760 | ||
f67c6275 | 761 | void ehea_destroy_busmap(void) |
44c82152 | 762 | { |
48cfb14f HH |
763 | int top, dir; |
764 | mutex_lock(&ehea_busmap_mutex); | |
765 | if (!ehea_bmap) | |
766 | goto out_destroy; | |
767 | ||
768 | for (top = 0; top < EHEA_MAP_ENTRIES; top++) { | |
769 | if (!ehea_bmap->top[top]) | |
770 | continue; | |
771 | ||
772 | for (dir = 0; dir < EHEA_MAP_ENTRIES; dir++) { | |
773 | if (!ehea_bmap->top[top]->dir[dir]) | |
774 | continue; | |
775 | ||
776 | kfree(ehea_bmap->top[top]->dir[dir]); | |
777 | } | |
778 | ||
779 | kfree(ehea_bmap->top[top]); | |
780 | } | |
781 | ||
782 | kfree(ehea_bmap); | |
783 | ehea_bmap = NULL; | |
c5916cf8 | 784 | out_destroy: |
48cfb14f | 785 | mutex_unlock(&ehea_busmap_mutex); |
44c82152 TK |
786 | } |
787 | ||
788 | u64 ehea_map_vaddr(void *caddr) | |
789 | { | |
48cfb14f HH |
790 | int top, dir, idx; |
791 | unsigned long index, offset; | |
792 | ||
793 | if (!ehea_bmap) | |
794 | return EHEA_INVAL_ADDR; | |
795 | ||
796 | index = virt_to_abs(caddr) >> SECTION_SIZE_BITS; | |
797 | top = (index >> EHEA_TOP_INDEX_SHIFT) & EHEA_INDEX_MASK; | |
798 | if (!ehea_bmap->top[top]) | |
799 | return EHEA_INVAL_ADDR; | |
800 | ||
801 | dir = (index >> EHEA_DIR_INDEX_SHIFT) & EHEA_INDEX_MASK; | |
802 | if (!ehea_bmap->top[top]->dir[dir]) | |
803 | return EHEA_INVAL_ADDR; | |
804 | ||
805 | idx = index & EHEA_INDEX_MASK; | |
806 | if (!ehea_bmap->top[top]->dir[dir]->ent[idx]) | |
807 | return EHEA_INVAL_ADDR; | |
808 | ||
809 | offset = (unsigned long)caddr & (EHEA_SECTSIZE - 1); | |
810 | return ehea_bmap->top[top]->dir[dir]->ent[idx] | offset; | |
811 | } | |
812 | ||
813 | static inline void *ehea_calc_sectbase(int top, int dir, int idx) | |
814 | { | |
815 | unsigned long ret = idx; | |
816 | ret |= dir << EHEA_DIR_INDEX_SHIFT; | |
817 | ret |= top << EHEA_TOP_INDEX_SHIFT; | |
818 | return abs_to_virt(ret << SECTION_SIZE_BITS); | |
819 | } | |
820 | ||
821 | static u64 ehea_reg_mr_section(int top, int dir, int idx, u64 *pt, | |
822 | struct ehea_adapter *adapter, | |
823 | struct ehea_mr *mr) | |
824 | { | |
825 | void *pg; | |
826 | u64 j, m, hret; | |
827 | unsigned long k = 0; | |
828 | u64 pt_abs = virt_to_abs(pt); | |
829 | ||
830 | void *sectbase = ehea_calc_sectbase(top, dir, idx); | |
831 | ||
832 | for (j = 0; j < (EHEA_PAGES_PER_SECTION / EHEA_MAX_RPAGE); j++) { | |
833 | ||
834 | for (m = 0; m < EHEA_MAX_RPAGE; m++) { | |
835 | pg = sectbase + ((k++) * EHEA_PAGESIZE); | |
836 | pt[m] = virt_to_abs(pg); | |
837 | } | |
838 | hret = ehea_h_register_rpage_mr(adapter->handle, mr->handle, 0, | |
839 | 0, pt_abs, EHEA_MAX_RPAGE); | |
840 | ||
8e95a202 JP |
841 | if ((hret != H_SUCCESS) && |
842 | (hret != H_PAGE_REGISTERED)) { | |
48cfb14f HH |
843 | ehea_h_free_resource(adapter->handle, mr->handle, |
844 | FORCE_FREE); | |
845 | ehea_error("register_rpage_mr failed"); | |
846 | return hret; | |
847 | } | |
848 | } | |
849 | return hret; | |
850 | } | |
851 | ||
852 | static u64 ehea_reg_mr_sections(int top, int dir, u64 *pt, | |
853 | struct ehea_adapter *adapter, | |
854 | struct ehea_mr *mr) | |
855 | { | |
856 | u64 hret = H_SUCCESS; | |
857 | int idx; | |
858 | ||
859 | for (idx = 0; idx < EHEA_MAP_ENTRIES; idx++) { | |
860 | if (!ehea_bmap->top[top]->dir[dir]->ent[idx]) | |
861 | continue; | |
c5916cf8 | 862 | |
48cfb14f HH |
863 | hret = ehea_reg_mr_section(top, dir, idx, pt, adapter, mr); |
864 | if ((hret != H_SUCCESS) && (hret != H_PAGE_REGISTERED)) | |
c5916cf8 | 865 | return hret; |
48cfb14f HH |
866 | } |
867 | return hret; | |
868 | } | |
869 | ||
870 | static u64 ehea_reg_mr_dir_sections(int top, u64 *pt, | |
871 | struct ehea_adapter *adapter, | |
872 | struct ehea_mr *mr) | |
873 | { | |
874 | u64 hret = H_SUCCESS; | |
875 | int dir; | |
876 | ||
877 | for (dir = 0; dir < EHEA_MAP_ENTRIES; dir++) { | |
878 | if (!ehea_bmap->top[top]->dir[dir]) | |
879 | continue; | |
880 | ||
881 | hret = ehea_reg_mr_sections(top, dir, pt, adapter, mr); | |
882 | if ((hret != H_SUCCESS) && (hret != H_PAGE_REGISTERED)) | |
c5916cf8 | 883 | return hret; |
48cfb14f HH |
884 | } |
885 | return hret; | |
44c82152 TK |
886 | } |
887 | ||
e542aa6b | 888 | int ehea_reg_kernel_mr(struct ehea_adapter *adapter, struct ehea_mr *mr) |
7a291083 | 889 | { |
44c82152 | 890 | int ret; |
7a291083 | 891 | u64 *pt; |
48cfb14f | 892 | u64 hret; |
44c82152 | 893 | u32 acc_ctrl = EHEA_MR_ACC_CTRL; |
7a291083 | 894 | |
48cfb14f | 895 | unsigned long top; |
7a291083 | 896 | |
d76e56b4 | 897 | pt = (void *)get_zeroed_page(GFP_KERNEL); |
7a291083 JBT |
898 | if (!pt) { |
899 | ehea_error("no mem"); | |
900 | ret = -ENOMEM; | |
901 | goto out; | |
902 | } | |
7a291083 | 903 | |
48cfb14f HH |
904 | hret = ehea_h_alloc_resource_mr(adapter->handle, EHEA_BUSMAP_START, |
905 | ehea_mr_len, acc_ctrl, adapter->pd, | |
e542aa6b | 906 | &mr->handle, &mr->lkey); |
48cfb14f | 907 | |
7a291083 JBT |
908 | if (hret != H_SUCCESS) { |
909 | ehea_error("alloc_resource_mr failed"); | |
910 | ret = -EIO; | |
911 | goto out; | |
912 | } | |
913 | ||
48cfb14f HH |
914 | if (!ehea_bmap) { |
915 | ehea_h_free_resource(adapter->handle, mr->handle, FORCE_FREE); | |
916 | ehea_error("no busmap available"); | |
917 | ret = -EIO; | |
918 | goto out; | |
919 | } | |
920 | ||
921 | for (top = 0; top < EHEA_MAP_ENTRIES; top++) { | |
922 | if (!ehea_bmap->top[top]) | |
923 | continue; | |
924 | ||
925 | hret = ehea_reg_mr_dir_sections(top, pt, adapter, mr); | |
926 | if((hret != H_PAGE_REGISTERED) && (hret != H_SUCCESS)) | |
927 | break; | |
928 | } | |
7a291083 JBT |
929 | |
930 | if (hret != H_SUCCESS) { | |
44c82152 TK |
931 | ehea_h_free_resource(adapter->handle, mr->handle, FORCE_FREE); |
932 | ehea_error("registering mr failed"); | |
7a291083 JBT |
933 | ret = -EIO; |
934 | goto out; | |
935 | } | |
e542aa6b | 936 | |
44c82152 | 937 | mr->vaddr = EHEA_BUSMAP_START; |
e542aa6b | 938 | mr->adapter = adapter; |
7a291083 JBT |
939 | ret = 0; |
940 | out: | |
d76e56b4 | 941 | free_page((unsigned long)pt); |
7a291083 JBT |
942 | return ret; |
943 | } | |
944 | ||
e542aa6b JBT |
945 | int ehea_rem_mr(struct ehea_mr *mr) |
946 | { | |
947 | u64 hret; | |
948 | ||
949 | if (!mr || !mr->adapter) | |
950 | return -EINVAL; | |
951 | ||
952 | hret = ehea_h_free_resource(mr->adapter->handle, mr->handle, | |
953 | FORCE_FREE); | |
954 | if (hret != H_SUCCESS) { | |
955 | ehea_error("destroy MR failed"); | |
956 | return -EIO; | |
957 | } | |
958 | ||
959 | return 0; | |
960 | } | |
961 | ||
962 | int ehea_gen_smr(struct ehea_adapter *adapter, struct ehea_mr *old_mr, | |
963 | struct ehea_mr *shared_mr) | |
964 | { | |
965 | u64 hret; | |
966 | ||
967 | hret = ehea_h_register_smr(adapter->handle, old_mr->handle, | |
968 | old_mr->vaddr, EHEA_MR_ACC_CTRL, | |
969 | adapter->pd, shared_mr); | |
970 | if (hret != H_SUCCESS) | |
971 | return -EIO; | |
972 | ||
973 | shared_mr->adapter = adapter; | |
974 | ||
975 | return 0; | |
976 | } | |
977 | ||
d2db9eea JBT |
978 | void print_error_data(u64 *data) |
979 | { | |
980 | int length; | |
981 | u64 type = EHEA_BMASK_GET(ERROR_DATA_TYPE, data[2]); | |
982 | u64 resource = data[1]; | |
983 | ||
984 | length = EHEA_BMASK_GET(ERROR_DATA_LENGTH, data[0]); | |
985 | ||
986 | if (length > EHEA_PAGESIZE) | |
987 | length = EHEA_PAGESIZE; | |
988 | ||
989 | if (type == 0x8) /* Queue Pair */ | |
a1c5a893 SR |
990 | ehea_error("QP (resource=%llX) state: AER=0x%llX, AERR=0x%llX, " |
991 | "port=%llX", resource, data[6], data[12], data[22]); | |
d2db9eea | 992 | |
e542aa6b | 993 | if (type == 0x4) /* Completion Queue */ |
a1c5a893 | 994 | ehea_error("CQ (resource=%llX) state: AER=0x%llX", resource, |
e542aa6b JBT |
995 | data[6]); |
996 | ||
997 | if (type == 0x3) /* Event Queue */ | |
a1c5a893 | 998 | ehea_error("EQ (resource=%llX) state: AER=0x%llX", resource, |
e542aa6b JBT |
999 | data[6]); |
1000 | ||
d2db9eea JBT |
1001 | ehea_dump(data, length, "error data"); |
1002 | } | |
1003 | ||
1004 | void ehea_error_data(struct ehea_adapter *adapter, u64 res_handle) | |
1005 | { | |
1006 | unsigned long ret; | |
1007 | u64 *rblock; | |
1008 | ||
3faf2693 | 1009 | rblock = (void *)get_zeroed_page(GFP_KERNEL); |
d2db9eea JBT |
1010 | if (!rblock) { |
1011 | ehea_error("Cannot allocate rblock memory."); | |
1012 | return; | |
1013 | } | |
7a291083 | 1014 | |
d2db9eea JBT |
1015 | ret = ehea_h_error_data(adapter->handle, |
1016 | res_handle, | |
1017 | rblock); | |
1018 | ||
1019 | if (ret == H_R_STATE) | |
a1c5a893 | 1020 | ehea_error("No error data is available: %llX.", res_handle); |
d2db9eea JBT |
1021 | else if (ret == H_SUCCESS) |
1022 | print_error_data(rblock); | |
1023 | else | |
a1c5a893 | 1024 | ehea_error("Error data could not be fetched: %llX", res_handle); |
d2db9eea | 1025 | |
3faf2693 | 1026 | free_page((unsigned long)rblock); |
d2db9eea | 1027 | } |