]>
Commit | Line | Data |
---|---|---|
b038ced7 SW |
1 | /* |
2 | * Copyright (c) 2006 Chelsio, Inc. All rights reserved. | |
b038ced7 SW |
3 | * |
4 | * This software is available to you under a choice of one of two | |
5 | * licenses. You may choose to be licensed under the terms of the GNU | |
6 | * General Public License (GPL) Version 2, available from the file | |
7 | * COPYING in the main directory of this source tree, or the | |
8 | * OpenIB.org BSD license below: | |
9 | * | |
10 | * Redistribution and use in source and binary forms, with or | |
11 | * without modification, are permitted provided that the following | |
12 | * conditions are met: | |
13 | * | |
14 | * - Redistributions of source code must retain the above | |
15 | * copyright notice, this list of conditions and the following | |
16 | * disclaimer. | |
17 | * | |
18 | * - Redistributions in binary form must reproduce the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer in the documentation and/or other materials | |
21 | * provided with the distribution. | |
22 | * | |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
30 | * SOFTWARE. | |
31 | */ | |
32 | #include "iwch_provider.h" | |
33 | #include "iwch.h" | |
34 | #include "iwch_cm.h" | |
35 | #include "cxio_hal.h" | |
36 | ||
37 | #define NO_SUPPORT -1 | |
38 | ||
2b540355 AB |
39 | static int iwch_build_rdma_send(union t3_wr *wqe, struct ib_send_wr *wr, |
40 | u8 * flit_cnt) | |
b038ced7 SW |
41 | { |
42 | int i; | |
43 | u32 plen; | |
44 | ||
45 | switch (wr->opcode) { | |
46 | case IB_WR_SEND: | |
47 | case IB_WR_SEND_WITH_IMM: | |
48 | if (wr->send_flags & IB_SEND_SOLICITED) | |
49 | wqe->send.rdmaop = T3_SEND_WITH_SE; | |
50 | else | |
51 | wqe->send.rdmaop = T3_SEND; | |
52 | wqe->send.rem_stag = 0; | |
53 | break; | |
54 | #if 0 /* Not currently supported */ | |
55 | case TYPE_SEND_INVALIDATE: | |
56 | case TYPE_SEND_INVALIDATE_IMMEDIATE: | |
57 | wqe->send.rdmaop = T3_SEND_WITH_INV; | |
58 | wqe->send.rem_stag = cpu_to_be32(wr->wr.rdma.rkey); | |
59 | break; | |
60 | case TYPE_SEND_SE_INVALIDATE: | |
61 | wqe->send.rdmaop = T3_SEND_WITH_SE_INV; | |
62 | wqe->send.rem_stag = cpu_to_be32(wr->wr.rdma.rkey); | |
63 | break; | |
64 | #endif | |
65 | default: | |
66 | break; | |
67 | } | |
68 | if (wr->num_sge > T3_MAX_SGE) | |
69 | return -EINVAL; | |
70 | wqe->send.reserved[0] = 0; | |
71 | wqe->send.reserved[1] = 0; | |
72 | wqe->send.reserved[2] = 0; | |
73 | if (wr->opcode == IB_WR_SEND_WITH_IMM) { | |
74 | plen = 4; | |
0f39cf3d | 75 | wqe->send.sgl[0].stag = wr->ex.imm_data; |
b038ced7 SW |
76 | wqe->send.sgl[0].len = __constant_cpu_to_be32(0); |
77 | wqe->send.num_sgle = __constant_cpu_to_be32(0); | |
78 | *flit_cnt = 5; | |
79 | } else { | |
80 | plen = 0; | |
81 | for (i = 0; i < wr->num_sge; i++) { | |
82 | if ((plen + wr->sg_list[i].length) < plen) { | |
83 | return -EMSGSIZE; | |
84 | } | |
85 | plen += wr->sg_list[i].length; | |
86 | wqe->send.sgl[i].stag = | |
87 | cpu_to_be32(wr->sg_list[i].lkey); | |
88 | wqe->send.sgl[i].len = | |
89 | cpu_to_be32(wr->sg_list[i].length); | |
90 | wqe->send.sgl[i].to = cpu_to_be64(wr->sg_list[i].addr); | |
91 | } | |
92 | wqe->send.num_sgle = cpu_to_be32(wr->num_sge); | |
93 | *flit_cnt = 4 + ((wr->num_sge) << 1); | |
94 | } | |
95 | wqe->send.plen = cpu_to_be32(plen); | |
96 | return 0; | |
97 | } | |
98 | ||
2b540355 AB |
99 | static int iwch_build_rdma_write(union t3_wr *wqe, struct ib_send_wr *wr, |
100 | u8 *flit_cnt) | |
b038ced7 SW |
101 | { |
102 | int i; | |
103 | u32 plen; | |
104 | if (wr->num_sge > T3_MAX_SGE) | |
105 | return -EINVAL; | |
106 | wqe->write.rdmaop = T3_RDMA_WRITE; | |
107 | wqe->write.reserved[0] = 0; | |
108 | wqe->write.reserved[1] = 0; | |
109 | wqe->write.reserved[2] = 0; | |
110 | wqe->write.stag_sink = cpu_to_be32(wr->wr.rdma.rkey); | |
111 | wqe->write.to_sink = cpu_to_be64(wr->wr.rdma.remote_addr); | |
112 | ||
113 | if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) { | |
114 | plen = 4; | |
0f39cf3d | 115 | wqe->write.sgl[0].stag = wr->ex.imm_data; |
b038ced7 SW |
116 | wqe->write.sgl[0].len = __constant_cpu_to_be32(0); |
117 | wqe->write.num_sgle = __constant_cpu_to_be32(0); | |
118 | *flit_cnt = 6; | |
119 | } else { | |
120 | plen = 0; | |
121 | for (i = 0; i < wr->num_sge; i++) { | |
122 | if ((plen + wr->sg_list[i].length) < plen) { | |
123 | return -EMSGSIZE; | |
124 | } | |
125 | plen += wr->sg_list[i].length; | |
126 | wqe->write.sgl[i].stag = | |
127 | cpu_to_be32(wr->sg_list[i].lkey); | |
128 | wqe->write.sgl[i].len = | |
129 | cpu_to_be32(wr->sg_list[i].length); | |
130 | wqe->write.sgl[i].to = | |
131 | cpu_to_be64(wr->sg_list[i].addr); | |
132 | } | |
133 | wqe->write.num_sgle = cpu_to_be32(wr->num_sge); | |
134 | *flit_cnt = 5 + ((wr->num_sge) << 1); | |
135 | } | |
136 | wqe->write.plen = cpu_to_be32(plen); | |
137 | return 0; | |
138 | } | |
139 | ||
2b540355 AB |
140 | static int iwch_build_rdma_read(union t3_wr *wqe, struct ib_send_wr *wr, |
141 | u8 *flit_cnt) | |
b038ced7 SW |
142 | { |
143 | if (wr->num_sge > 1) | |
144 | return -EINVAL; | |
145 | wqe->read.rdmaop = T3_READ_REQ; | |
146 | wqe->read.reserved[0] = 0; | |
147 | wqe->read.reserved[1] = 0; | |
148 | wqe->read.reserved[2] = 0; | |
149 | wqe->read.rem_stag = cpu_to_be32(wr->wr.rdma.rkey); | |
150 | wqe->read.rem_to = cpu_to_be64(wr->wr.rdma.remote_addr); | |
151 | wqe->read.local_stag = cpu_to_be32(wr->sg_list[0].lkey); | |
152 | wqe->read.local_len = cpu_to_be32(wr->sg_list[0].length); | |
153 | wqe->read.local_to = cpu_to_be64(wr->sg_list[0].addr); | |
154 | *flit_cnt = sizeof(struct t3_rdma_read_wr) >> 3; | |
155 | return 0; | |
156 | } | |
157 | ||
158 | /* | |
159 | * TBD: this is going to be moved to firmware. Missing pdid/qpid check for now. | |
160 | */ | |
2b540355 AB |
161 | static int iwch_sgl2pbl_map(struct iwch_dev *rhp, struct ib_sge *sg_list, |
162 | u32 num_sgle, u32 * pbl_addr, u8 * page_size) | |
b038ced7 SW |
163 | { |
164 | int i; | |
165 | struct iwch_mr *mhp; | |
166 | u32 offset; | |
167 | for (i = 0; i < num_sgle; i++) { | |
168 | ||
169 | mhp = get_mhp(rhp, (sg_list[i].lkey) >> 8); | |
170 | if (!mhp) { | |
33718363 | 171 | PDBG("%s %d\n", __func__, __LINE__); |
b038ced7 SW |
172 | return -EIO; |
173 | } | |
174 | if (!mhp->attr.state) { | |
33718363 | 175 | PDBG("%s %d\n", __func__, __LINE__); |
b038ced7 SW |
176 | return -EIO; |
177 | } | |
178 | if (mhp->attr.zbva) { | |
33718363 | 179 | PDBG("%s %d\n", __func__, __LINE__); |
b038ced7 SW |
180 | return -EIO; |
181 | } | |
182 | ||
183 | if (sg_list[i].addr < mhp->attr.va_fbo) { | |
33718363 | 184 | PDBG("%s %d\n", __func__, __LINE__); |
b038ced7 SW |
185 | return -EINVAL; |
186 | } | |
187 | if (sg_list[i].addr + ((u64) sg_list[i].length) < | |
188 | sg_list[i].addr) { | |
33718363 | 189 | PDBG("%s %d\n", __func__, __LINE__); |
b038ced7 SW |
190 | return -EINVAL; |
191 | } | |
192 | if (sg_list[i].addr + ((u64) sg_list[i].length) > | |
193 | mhp->attr.va_fbo + ((u64) mhp->attr.len)) { | |
33718363 | 194 | PDBG("%s %d\n", __func__, __LINE__); |
b038ced7 SW |
195 | return -EINVAL; |
196 | } | |
197 | offset = sg_list[i].addr - mhp->attr.va_fbo; | |
198 | offset += ((u32) mhp->attr.va_fbo) % | |
199 | (1UL << (12 + mhp->attr.page_size)); | |
200 | pbl_addr[i] = ((mhp->attr.pbl_addr - | |
201 | rhp->rdev.rnic_info.pbl_base) >> 3) + | |
202 | (offset >> (12 + mhp->attr.page_size)); | |
203 | page_size[i] = mhp->attr.page_size; | |
204 | } | |
205 | return 0; | |
206 | } | |
207 | ||
2b540355 AB |
208 | static int iwch_build_rdma_recv(struct iwch_dev *rhp, union t3_wr *wqe, |
209 | struct ib_recv_wr *wr) | |
b038ced7 | 210 | { |
457fe7b8 | 211 | int i; |
b038ced7 SW |
212 | if (wr->num_sge > T3_MAX_SGE) |
213 | return -EINVAL; | |
b038ced7 SW |
214 | wqe->recv.num_sgle = cpu_to_be32(wr->num_sge); |
215 | for (i = 0; i < wr->num_sge; i++) { | |
216 | wqe->recv.sgl[i].stag = cpu_to_be32(wr->sg_list[i].lkey); | |
217 | wqe->recv.sgl[i].len = cpu_to_be32(wr->sg_list[i].length); | |
457fe7b8 | 218 | wqe->recv.sgl[i].to = cpu_to_be64(wr->sg_list[i].addr); |
b038ced7 SW |
219 | } |
220 | for (; i < T3_MAX_SGE; i++) { | |
221 | wqe->recv.sgl[i].stag = 0; | |
222 | wqe->recv.sgl[i].len = 0; | |
223 | wqe->recv.sgl[i].to = 0; | |
b038ced7 SW |
224 | } |
225 | return 0; | |
226 | } | |
227 | ||
228 | int iwch_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, | |
229 | struct ib_send_wr **bad_wr) | |
230 | { | |
231 | int err = 0; | |
232 | u8 t3_wr_flit_cnt; | |
233 | enum t3_wr_opcode t3_wr_opcode = 0; | |
234 | enum t3_wr_flags t3_wr_flags; | |
235 | struct iwch_qp *qhp; | |
236 | u32 idx; | |
237 | union t3_wr *wqe; | |
238 | u32 num_wrs; | |
239 | unsigned long flag; | |
240 | struct t3_swsq *sqp; | |
241 | ||
242 | qhp = to_iwch_qp(ibqp); | |
243 | spin_lock_irqsave(&qhp->lock, flag); | |
244 | if (qhp->attr.state > IWCH_QP_STATE_RTS) { | |
245 | spin_unlock_irqrestore(&qhp->lock, flag); | |
246 | return -EINVAL; | |
247 | } | |
248 | num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr, | |
249 | qhp->wq.sq_size_log2); | |
250 | if (num_wrs <= 0) { | |
251 | spin_unlock_irqrestore(&qhp->lock, flag); | |
252 | return -ENOMEM; | |
253 | } | |
254 | while (wr) { | |
255 | if (num_wrs == 0) { | |
256 | err = -ENOMEM; | |
257 | *bad_wr = wr; | |
258 | break; | |
259 | } | |
260 | idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); | |
261 | wqe = (union t3_wr *) (qhp->wq.queue + idx); | |
262 | t3_wr_flags = 0; | |
263 | if (wr->send_flags & IB_SEND_SOLICITED) | |
264 | t3_wr_flags |= T3_SOLICITED_EVENT_FLAG; | |
265 | if (wr->send_flags & IB_SEND_FENCE) | |
266 | t3_wr_flags |= T3_READ_FENCE_FLAG; | |
267 | if (wr->send_flags & IB_SEND_SIGNALED) | |
268 | t3_wr_flags |= T3_COMPLETION_FLAG; | |
269 | sqp = qhp->wq.sq + | |
270 | Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2); | |
271 | switch (wr->opcode) { | |
272 | case IB_WR_SEND: | |
273 | case IB_WR_SEND_WITH_IMM: | |
274 | t3_wr_opcode = T3_WR_SEND; | |
275 | err = iwch_build_rdma_send(wqe, wr, &t3_wr_flit_cnt); | |
276 | break; | |
277 | case IB_WR_RDMA_WRITE: | |
278 | case IB_WR_RDMA_WRITE_WITH_IMM: | |
279 | t3_wr_opcode = T3_WR_WRITE; | |
280 | err = iwch_build_rdma_write(wqe, wr, &t3_wr_flit_cnt); | |
281 | break; | |
282 | case IB_WR_RDMA_READ: | |
283 | t3_wr_opcode = T3_WR_READ; | |
284 | t3_wr_flags = 0; /* T3 reads are always signaled */ | |
285 | err = iwch_build_rdma_read(wqe, wr, &t3_wr_flit_cnt); | |
286 | if (err) | |
287 | break; | |
288 | sqp->read_len = wqe->read.local_len; | |
289 | if (!qhp->wq.oldest_read) | |
290 | qhp->wq.oldest_read = sqp; | |
291 | break; | |
292 | default: | |
33718363 | 293 | PDBG("%s post of type=%d TBD!\n", __func__, |
b038ced7 SW |
294 | wr->opcode); |
295 | err = -EINVAL; | |
296 | } | |
297 | if (err) { | |
298 | *bad_wr = wr; | |
299 | break; | |
300 | } | |
301 | wqe->send.wrid.id0.hi = qhp->wq.sq_wptr; | |
302 | sqp->wr_id = wr->wr_id; | |
303 | sqp->opcode = wr2opcode(t3_wr_opcode); | |
304 | sqp->sq_wptr = qhp->wq.sq_wptr; | |
305 | sqp->complete = 0; | |
306 | sqp->signaled = (wr->send_flags & IB_SEND_SIGNALED); | |
307 | ||
308 | build_fw_riwrh((void *) wqe, t3_wr_opcode, t3_wr_flags, | |
309 | Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), | |
310 | 0, t3_wr_flit_cnt); | |
311 | PDBG("%s cookie 0x%llx wq idx 0x%x swsq idx %ld opcode %d\n", | |
33718363 | 312 | __func__, (unsigned long long) wr->wr_id, idx, |
b038ced7 SW |
313 | Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2), |
314 | sqp->opcode); | |
315 | wr = wr->next; | |
316 | num_wrs--; | |
317 | ++(qhp->wq.wptr); | |
318 | ++(qhp->wq.sq_wptr); | |
319 | } | |
320 | spin_unlock_irqrestore(&qhp->lock, flag); | |
321 | ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); | |
322 | return err; | |
323 | } | |
324 | ||
325 | int iwch_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, | |
326 | struct ib_recv_wr **bad_wr) | |
327 | { | |
328 | int err = 0; | |
329 | struct iwch_qp *qhp; | |
330 | u32 idx; | |
331 | union t3_wr *wqe; | |
332 | u32 num_wrs; | |
333 | unsigned long flag; | |
334 | ||
335 | qhp = to_iwch_qp(ibqp); | |
336 | spin_lock_irqsave(&qhp->lock, flag); | |
337 | if (qhp->attr.state > IWCH_QP_STATE_RTS) { | |
338 | spin_unlock_irqrestore(&qhp->lock, flag); | |
339 | return -EINVAL; | |
340 | } | |
341 | num_wrs = Q_FREECNT(qhp->wq.rq_rptr, qhp->wq.rq_wptr, | |
342 | qhp->wq.rq_size_log2) - 1; | |
343 | if (!wr) { | |
344 | spin_unlock_irqrestore(&qhp->lock, flag); | |
345 | return -EINVAL; | |
346 | } | |
347 | while (wr) { | |
348 | idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); | |
349 | wqe = (union t3_wr *) (qhp->wq.queue + idx); | |
350 | if (num_wrs) | |
351 | err = iwch_build_rdma_recv(qhp->rhp, wqe, wr); | |
352 | else | |
353 | err = -ENOMEM; | |
354 | if (err) { | |
355 | *bad_wr = wr; | |
356 | break; | |
357 | } | |
358 | qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr, qhp->wq.rq_size_log2)] = | |
359 | wr->wr_id; | |
360 | build_fw_riwrh((void *) wqe, T3_WR_RCV, T3_COMPLETION_FLAG, | |
361 | Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), | |
362 | 0, sizeof(struct t3_receive_wr) >> 3); | |
363 | PDBG("%s cookie 0x%llx idx 0x%x rq_wptr 0x%x rw_rptr 0x%x " | |
33718363 | 364 | "wqe %p \n", __func__, (unsigned long long) wr->wr_id, |
b038ced7 SW |
365 | idx, qhp->wq.rq_wptr, qhp->wq.rq_rptr, wqe); |
366 | ++(qhp->wq.rq_wptr); | |
367 | ++(qhp->wq.wptr); | |
368 | wr = wr->next; | |
369 | num_wrs--; | |
370 | } | |
371 | spin_unlock_irqrestore(&qhp->lock, flag); | |
372 | ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); | |
373 | return err; | |
374 | } | |
375 | ||
376 | int iwch_bind_mw(struct ib_qp *qp, | |
377 | struct ib_mw *mw, | |
378 | struct ib_mw_bind *mw_bind) | |
379 | { | |
380 | struct iwch_dev *rhp; | |
381 | struct iwch_mw *mhp; | |
382 | struct iwch_qp *qhp; | |
383 | union t3_wr *wqe; | |
384 | u32 pbl_addr; | |
385 | u8 page_size; | |
386 | u32 num_wrs; | |
387 | unsigned long flag; | |
388 | struct ib_sge sgl; | |
389 | int err=0; | |
390 | enum t3_wr_flags t3_wr_flags; | |
391 | u32 idx; | |
392 | struct t3_swsq *sqp; | |
393 | ||
394 | qhp = to_iwch_qp(qp); | |
395 | mhp = to_iwch_mw(mw); | |
396 | rhp = qhp->rhp; | |
397 | ||
398 | spin_lock_irqsave(&qhp->lock, flag); | |
399 | if (qhp->attr.state > IWCH_QP_STATE_RTS) { | |
400 | spin_unlock_irqrestore(&qhp->lock, flag); | |
401 | return -EINVAL; | |
402 | } | |
403 | num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr, | |
404 | qhp->wq.sq_size_log2); | |
405 | if ((num_wrs) <= 0) { | |
406 | spin_unlock_irqrestore(&qhp->lock, flag); | |
407 | return -ENOMEM; | |
408 | } | |
409 | idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); | |
33718363 | 410 | PDBG("%s: idx 0x%0x, mw 0x%p, mw_bind 0x%p\n", __func__, idx, |
b038ced7 SW |
411 | mw, mw_bind); |
412 | wqe = (union t3_wr *) (qhp->wq.queue + idx); | |
413 | ||
414 | t3_wr_flags = 0; | |
415 | if (mw_bind->send_flags & IB_SEND_SIGNALED) | |
416 | t3_wr_flags = T3_COMPLETION_FLAG; | |
417 | ||
418 | sgl.addr = mw_bind->addr; | |
419 | sgl.lkey = mw_bind->mr->lkey; | |
420 | sgl.length = mw_bind->length; | |
421 | wqe->bind.reserved = 0; | |
422 | wqe->bind.type = T3_VA_BASED_TO; | |
423 | ||
424 | /* TBD: check perms */ | |
e64518f3 | 425 | wqe->bind.perms = iwch_ib_to_mwbind_access(mw_bind->mw_access_flags); |
b038ced7 SW |
426 | wqe->bind.mr_stag = cpu_to_be32(mw_bind->mr->lkey); |
427 | wqe->bind.mw_stag = cpu_to_be32(mw->rkey); | |
428 | wqe->bind.mw_len = cpu_to_be32(mw_bind->length); | |
429 | wqe->bind.mw_va = cpu_to_be64(mw_bind->addr); | |
430 | err = iwch_sgl2pbl_map(rhp, &sgl, 1, &pbl_addr, &page_size); | |
431 | if (err) { | |
432 | spin_unlock_irqrestore(&qhp->lock, flag); | |
433 | return err; | |
434 | } | |
435 | wqe->send.wrid.id0.hi = qhp->wq.sq_wptr; | |
436 | sqp = qhp->wq.sq + Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2); | |
437 | sqp->wr_id = mw_bind->wr_id; | |
438 | sqp->opcode = T3_BIND_MW; | |
439 | sqp->sq_wptr = qhp->wq.sq_wptr; | |
440 | sqp->complete = 0; | |
441 | sqp->signaled = (mw_bind->send_flags & IB_SEND_SIGNALED); | |
442 | wqe->bind.mr_pbl_addr = cpu_to_be32(pbl_addr); | |
443 | wqe->bind.mr_pagesz = page_size; | |
444 | wqe->flit[T3_SQ_COOKIE_FLIT] = mw_bind->wr_id; | |
445 | build_fw_riwrh((void *)wqe, T3_WR_BIND, t3_wr_flags, | |
446 | Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), 0, | |
447 | sizeof(struct t3_bind_mw_wr) >> 3); | |
448 | ++(qhp->wq.wptr); | |
449 | ++(qhp->wq.sq_wptr); | |
450 | spin_unlock_irqrestore(&qhp->lock, flag); | |
451 | ||
452 | ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); | |
453 | ||
454 | return err; | |
455 | } | |
456 | ||
4a97d47e SW |
457 | static inline void build_term_codes(struct respQ_msg_t *rsp_msg, |
458 | u8 *layer_type, u8 *ecode) | |
b038ced7 | 459 | { |
4a97d47e SW |
460 | int status = TPT_ERR_INTERNAL_ERR; |
461 | int tagged = 0; | |
462 | int opcode = -1; | |
463 | int rqtype = 0; | |
464 | int send_inv = 0; | |
465 | ||
466 | if (rsp_msg) { | |
467 | status = CQE_STATUS(rsp_msg->cqe); | |
468 | opcode = CQE_OPCODE(rsp_msg->cqe); | |
469 | rqtype = RQ_TYPE(rsp_msg->cqe); | |
470 | send_inv = (opcode == T3_SEND_WITH_INV) || | |
471 | (opcode == T3_SEND_WITH_SE_INV); | |
472 | tagged = (opcode == T3_RDMA_WRITE) || | |
473 | (rqtype && (opcode == T3_READ_RESP)); | |
474 | } | |
475 | ||
476 | switch (status) { | |
b038ced7 | 477 | case TPT_ERR_STAG: |
4a97d47e SW |
478 | if (send_inv) { |
479 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; | |
480 | *ecode = RDMAP_CANT_INV_STAG; | |
481 | } else { | |
b038ced7 SW |
482 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; |
483 | *ecode = RDMAP_INV_STAG; | |
484 | } | |
485 | break; | |
486 | case TPT_ERR_PDID: | |
4a97d47e SW |
487 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; |
488 | if ((opcode == T3_SEND_WITH_INV) || | |
489 | (opcode == T3_SEND_WITH_SE_INV)) | |
490 | *ecode = RDMAP_CANT_INV_STAG; | |
491 | else | |
492 | *ecode = RDMAP_STAG_NOT_ASSOC; | |
493 | break; | |
b038ced7 | 494 | case TPT_ERR_QPID: |
4a97d47e SW |
495 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; |
496 | *ecode = RDMAP_STAG_NOT_ASSOC; | |
497 | break; | |
b038ced7 | 498 | case TPT_ERR_ACCESS: |
4a97d47e SW |
499 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; |
500 | *ecode = RDMAP_ACC_VIOL; | |
b038ced7 SW |
501 | break; |
502 | case TPT_ERR_WRAP: | |
503 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; | |
504 | *ecode = RDMAP_TO_WRAP; | |
505 | break; | |
506 | case TPT_ERR_BOUND: | |
4a97d47e | 507 | if (tagged) { |
b038ced7 SW |
508 | *layer_type = LAYER_DDP|DDP_TAGGED_ERR; |
509 | *ecode = DDPT_BASE_BOUNDS; | |
4a97d47e | 510 | } else { |
b038ced7 SW |
511 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; |
512 | *ecode = RDMAP_BASE_BOUNDS; | |
b038ced7 SW |
513 | } |
514 | break; | |
515 | case TPT_ERR_INVALIDATE_SHARED_MR: | |
516 | case TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND: | |
517 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; | |
518 | *ecode = RDMAP_CANT_INV_STAG; | |
519 | break; | |
520 | case TPT_ERR_ECC: | |
521 | case TPT_ERR_ECC_PSTAG: | |
522 | case TPT_ERR_INTERNAL_ERR: | |
523 | *layer_type = LAYER_RDMAP|RDMAP_LOCAL_CATA; | |
524 | *ecode = 0; | |
525 | break; | |
526 | case TPT_ERR_OUT_OF_RQE: | |
527 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
528 | *ecode = DDPU_INV_MSN_NOBUF; | |
529 | break; | |
530 | case TPT_ERR_PBL_ADDR_BOUND: | |
531 | *layer_type = LAYER_DDP|DDP_TAGGED_ERR; | |
532 | *ecode = DDPT_BASE_BOUNDS; | |
533 | break; | |
534 | case TPT_ERR_CRC: | |
535 | *layer_type = LAYER_MPA|DDP_LLP; | |
536 | *ecode = MPA_CRC_ERR; | |
537 | break; | |
538 | case TPT_ERR_MARKER: | |
539 | *layer_type = LAYER_MPA|DDP_LLP; | |
540 | *ecode = MPA_MARKER_ERR; | |
541 | break; | |
542 | case TPT_ERR_PDU_LEN_ERR: | |
543 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
544 | *ecode = DDPU_MSG_TOOBIG; | |
545 | break; | |
546 | case TPT_ERR_DDP_VERSION: | |
547 | if (tagged) { | |
548 | *layer_type = LAYER_DDP|DDP_TAGGED_ERR; | |
549 | *ecode = DDPT_INV_VERS; | |
550 | } else { | |
551 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
552 | *ecode = DDPU_INV_VERS; | |
553 | } | |
554 | break; | |
555 | case TPT_ERR_RDMA_VERSION: | |
556 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; | |
557 | *ecode = RDMAP_INV_VERS; | |
558 | break; | |
559 | case TPT_ERR_OPCODE: | |
560 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; | |
561 | *ecode = RDMAP_INV_OPCODE; | |
562 | break; | |
563 | case TPT_ERR_DDP_QUEUE_NUM: | |
564 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
565 | *ecode = DDPU_INV_QN; | |
566 | break; | |
567 | case TPT_ERR_MSN: | |
568 | case TPT_ERR_MSN_GAP: | |
569 | case TPT_ERR_MSN_RANGE: | |
570 | case TPT_ERR_IRD_OVERFLOW: | |
571 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
572 | *ecode = DDPU_INV_MSN_RANGE; | |
573 | break; | |
574 | case TPT_ERR_TBIT: | |
575 | *layer_type = LAYER_DDP|DDP_LOCAL_CATA; | |
576 | *ecode = 0; | |
577 | break; | |
578 | case TPT_ERR_MO: | |
579 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
580 | *ecode = DDPU_INV_MO; | |
581 | break; | |
582 | default: | |
583 | *layer_type = LAYER_RDMAP|DDP_LOCAL_CATA; | |
584 | *ecode = 0; | |
585 | break; | |
586 | } | |
587 | } | |
588 | ||
589 | /* | |
590 | * This posts a TERMINATE with layer=RDMA, type=catastrophic. | |
591 | */ | |
592 | int iwch_post_terminate(struct iwch_qp *qhp, struct respQ_msg_t *rsp_msg) | |
593 | { | |
594 | union t3_wr *wqe; | |
595 | struct terminate_message *term; | |
b038ced7 SW |
596 | struct sk_buff *skb; |
597 | ||
33718363 | 598 | PDBG("%s %d\n", __func__, __LINE__); |
b038ced7 SW |
599 | skb = alloc_skb(40, GFP_ATOMIC); |
600 | if (!skb) { | |
33718363 | 601 | printk(KERN_ERR "%s cannot send TERMINATE!\n", __func__); |
b038ced7 SW |
602 | return -ENOMEM; |
603 | } | |
604 | wqe = (union t3_wr *)skb_put(skb, 40); | |
605 | memset(wqe, 0, 40); | |
606 | wqe->send.rdmaop = T3_TERMINATE; | |
607 | ||
608 | /* immediate data length */ | |
609 | wqe->send.plen = htonl(4); | |
610 | ||
611 | /* immediate data starts here. */ | |
612 | term = (struct terminate_message *)wqe->send.sgl; | |
4a97d47e | 613 | build_term_codes(rsp_msg, &term->layer_etype, &term->ecode); |
fb497d72 SW |
614 | wqe->send.wrh.op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(T3_WR_SEND) | |
615 | V_FW_RIWR_FLAGS(T3_COMPLETION_FLAG | T3_NOTIFY_FLAG)); | |
616 | wqe->send.wrh.gen_tid_len = cpu_to_be32(V_FW_RIWR_TID(qhp->ep->hwtid)); | |
b038ced7 SW |
617 | skb->priority = CPL_PRIORITY_DATA; |
618 | return cxgb3_ofld_send(qhp->rhp->rdev.t3cdev_p, skb); | |
619 | } | |
620 | ||
621 | /* | |
622 | * Assumes qhp lock is held. | |
623 | */ | |
624 | static void __flush_qp(struct iwch_qp *qhp, unsigned long *flag) | |
625 | { | |
626 | struct iwch_cq *rchp, *schp; | |
627 | int count; | |
628 | ||
629 | rchp = get_chp(qhp->rhp, qhp->attr.rcq); | |
630 | schp = get_chp(qhp->rhp, qhp->attr.scq); | |
631 | ||
33718363 | 632 | PDBG("%s qhp %p rchp %p schp %p\n", __func__, qhp, rchp, schp); |
b038ced7 SW |
633 | /* take a ref on the qhp since we must release the lock */ |
634 | atomic_inc(&qhp->refcnt); | |
635 | spin_unlock_irqrestore(&qhp->lock, *flag); | |
636 | ||
637 | /* locking heirarchy: cq lock first, then qp lock. */ | |
638 | spin_lock_irqsave(&rchp->lock, *flag); | |
639 | spin_lock(&qhp->lock); | |
640 | cxio_flush_hw_cq(&rchp->cq); | |
641 | cxio_count_rcqes(&rchp->cq, &qhp->wq, &count); | |
642 | cxio_flush_rq(&qhp->wq, &rchp->cq, count); | |
643 | spin_unlock(&qhp->lock); | |
644 | spin_unlock_irqrestore(&rchp->lock, *flag); | |
856b5925 | 645 | (*rchp->ibcq.comp_handler)(&rchp->ibcq, rchp->ibcq.cq_context); |
b038ced7 SW |
646 | |
647 | /* locking heirarchy: cq lock first, then qp lock. */ | |
648 | spin_lock_irqsave(&schp->lock, *flag); | |
649 | spin_lock(&qhp->lock); | |
650 | cxio_flush_hw_cq(&schp->cq); | |
651 | cxio_count_scqes(&schp->cq, &qhp->wq, &count); | |
652 | cxio_flush_sq(&qhp->wq, &schp->cq, count); | |
653 | spin_unlock(&qhp->lock); | |
654 | spin_unlock_irqrestore(&schp->lock, *flag); | |
856b5925 | 655 | (*schp->ibcq.comp_handler)(&schp->ibcq, schp->ibcq.cq_context); |
b038ced7 SW |
656 | |
657 | /* deref */ | |
658 | if (atomic_dec_and_test(&qhp->refcnt)) | |
659 | wake_up(&qhp->wait); | |
660 | ||
661 | spin_lock_irqsave(&qhp->lock, *flag); | |
662 | } | |
663 | ||
2b540355 | 664 | static void flush_qp(struct iwch_qp *qhp, unsigned long *flag) |
b038ced7 | 665 | { |
856b5925 | 666 | if (qhp->ibqp.uobject) |
b038ced7 SW |
667 | cxio_set_wq_in_error(&qhp->wq); |
668 | else | |
669 | __flush_qp(qhp, flag); | |
670 | } | |
671 | ||
672 | ||
673 | /* | |
674 | * Return non zero if at least one RECV was pre-posted. | |
675 | */ | |
2b540355 | 676 | static int rqes_posted(struct iwch_qp *qhp) |
b038ced7 SW |
677 | { |
678 | return fw_riwrh_opcode((struct fw_riwrh *)qhp->wq.queue) == T3_WR_RCV; | |
679 | } | |
680 | ||
681 | static int rdma_init(struct iwch_dev *rhp, struct iwch_qp *qhp, | |
682 | enum iwch_qp_attr_mask mask, | |
683 | struct iwch_qp_attributes *attrs) | |
684 | { | |
685 | struct t3_rdma_init_attr init_attr; | |
686 | int ret; | |
687 | ||
688 | init_attr.tid = qhp->ep->hwtid; | |
689 | init_attr.qpid = qhp->wq.qpid; | |
690 | init_attr.pdid = qhp->attr.pd; | |
691 | init_attr.scqid = qhp->attr.scq; | |
692 | init_attr.rcqid = qhp->attr.rcq; | |
693 | init_attr.rq_addr = qhp->wq.rq_addr; | |
694 | init_attr.rq_size = 1 << qhp->wq.rq_size_log2; | |
695 | init_attr.mpaattrs = uP_RI_MPA_IETF_ENABLE | | |
696 | qhp->attr.mpa_attr.recv_marker_enabled | | |
697 | (qhp->attr.mpa_attr.xmit_marker_enabled << 1) | | |
698 | (qhp->attr.mpa_attr.crc_enabled << 2); | |
699 | ||
700 | /* | |
701 | * XXX - The IWCM doesn't quite handle getting these | |
702 | * attrs set before going into RTS. For now, just turn | |
703 | * them on always... | |
704 | */ | |
705 | #if 0 | |
706 | init_attr.qpcaps = qhp->attr.enableRdmaRead | | |
707 | (qhp->attr.enableRdmaWrite << 1) | | |
708 | (qhp->attr.enableBind << 2) | | |
709 | (qhp->attr.enable_stag0_fastreg << 3) | | |
710 | (qhp->attr.enable_stag0_fastreg << 4); | |
711 | #else | |
712 | init_attr.qpcaps = 0x1f; | |
713 | #endif | |
714 | init_attr.tcp_emss = qhp->ep->emss; | |
715 | init_attr.ord = qhp->attr.max_ord; | |
716 | init_attr.ird = qhp->attr.max_ird; | |
717 | init_attr.qp_dma_addr = qhp->wq.dma_addr; | |
718 | init_attr.qp_dma_size = (1UL << qhp->wq.size_log2); | |
719 | init_attr.flags = rqes_posted(qhp) ? RECVS_POSTED : 0; | |
c6b5b504 | 720 | init_attr.flags |= capable(CAP_NET_BIND_SERVICE) ? PRIV_QP : 0; |
de3d3530 | 721 | init_attr.irs = qhp->ep->rcv_seq; |
b038ced7 | 722 | PDBG("%s init_attr.rq_addr 0x%x init_attr.rq_size = %d " |
33718363 | 723 | "flags 0x%x qpcaps 0x%x\n", __func__, |
b038ced7 SW |
724 | init_attr.rq_addr, init_attr.rq_size, |
725 | init_attr.flags, init_attr.qpcaps); | |
726 | ret = cxio_rdma_init(&rhp->rdev, &init_attr); | |
33718363 | 727 | PDBG("%s ret %d\n", __func__, ret); |
b038ced7 SW |
728 | return ret; |
729 | } | |
730 | ||
731 | int iwch_modify_qp(struct iwch_dev *rhp, struct iwch_qp *qhp, | |
732 | enum iwch_qp_attr_mask mask, | |
733 | struct iwch_qp_attributes *attrs, | |
734 | int internal) | |
735 | { | |
736 | int ret = 0; | |
737 | struct iwch_qp_attributes newattr = qhp->attr; | |
738 | unsigned long flag; | |
739 | int disconnect = 0; | |
740 | int terminate = 0; | |
741 | int abort = 0; | |
742 | int free = 0; | |
743 | struct iwch_ep *ep = NULL; | |
744 | ||
33718363 | 745 | PDBG("%s qhp %p qpid 0x%x ep %p state %d -> %d\n", __func__, |
b038ced7 SW |
746 | qhp, qhp->wq.qpid, qhp->ep, qhp->attr.state, |
747 | (mask & IWCH_QP_ATTR_NEXT_STATE) ? attrs->next_state : -1); | |
748 | ||
749 | spin_lock_irqsave(&qhp->lock, flag); | |
750 | ||
751 | /* Process attr changes if in IDLE */ | |
752 | if (mask & IWCH_QP_ATTR_VALID_MODIFY) { | |
753 | if (qhp->attr.state != IWCH_QP_STATE_IDLE) { | |
754 | ret = -EIO; | |
755 | goto out; | |
756 | } | |
757 | if (mask & IWCH_QP_ATTR_ENABLE_RDMA_READ) | |
758 | newattr.enable_rdma_read = attrs->enable_rdma_read; | |
759 | if (mask & IWCH_QP_ATTR_ENABLE_RDMA_WRITE) | |
760 | newattr.enable_rdma_write = attrs->enable_rdma_write; | |
761 | if (mask & IWCH_QP_ATTR_ENABLE_RDMA_BIND) | |
762 | newattr.enable_bind = attrs->enable_bind; | |
763 | if (mask & IWCH_QP_ATTR_MAX_ORD) { | |
764 | if (attrs->max_ord > | |
765 | rhp->attr.max_rdma_read_qp_depth) { | |
766 | ret = -EINVAL; | |
767 | goto out; | |
768 | } | |
769 | newattr.max_ord = attrs->max_ord; | |
770 | } | |
771 | if (mask & IWCH_QP_ATTR_MAX_IRD) { | |
772 | if (attrs->max_ird > | |
773 | rhp->attr.max_rdma_reads_per_qp) { | |
774 | ret = -EINVAL; | |
775 | goto out; | |
776 | } | |
777 | newattr.max_ird = attrs->max_ird; | |
778 | } | |
779 | qhp->attr = newattr; | |
780 | } | |
781 | ||
782 | if (!(mask & IWCH_QP_ATTR_NEXT_STATE)) | |
783 | goto out; | |
784 | if (qhp->attr.state == attrs->next_state) | |
785 | goto out; | |
786 | ||
787 | switch (qhp->attr.state) { | |
788 | case IWCH_QP_STATE_IDLE: | |
789 | switch (attrs->next_state) { | |
790 | case IWCH_QP_STATE_RTS: | |
791 | if (!(mask & IWCH_QP_ATTR_LLP_STREAM_HANDLE)) { | |
792 | ret = -EINVAL; | |
793 | goto out; | |
794 | } | |
795 | if (!(mask & IWCH_QP_ATTR_MPA_ATTR)) { | |
796 | ret = -EINVAL; | |
797 | goto out; | |
798 | } | |
799 | qhp->attr.mpa_attr = attrs->mpa_attr; | |
800 | qhp->attr.llp_stream_handle = attrs->llp_stream_handle; | |
801 | qhp->ep = qhp->attr.llp_stream_handle; | |
802 | qhp->attr.state = IWCH_QP_STATE_RTS; | |
803 | ||
804 | /* | |
805 | * Ref the endpoint here and deref when we | |
806 | * disassociate the endpoint from the QP. This | |
807 | * happens in CLOSING->IDLE transition or *->ERROR | |
808 | * transition. | |
809 | */ | |
810 | get_ep(&qhp->ep->com); | |
811 | spin_unlock_irqrestore(&qhp->lock, flag); | |
812 | ret = rdma_init(rhp, qhp, mask, attrs); | |
813 | spin_lock_irqsave(&qhp->lock, flag); | |
814 | if (ret) | |
815 | goto err; | |
816 | break; | |
817 | case IWCH_QP_STATE_ERROR: | |
818 | qhp->attr.state = IWCH_QP_STATE_ERROR; | |
819 | flush_qp(qhp, &flag); | |
820 | break; | |
821 | default: | |
822 | ret = -EINVAL; | |
823 | goto out; | |
824 | } | |
825 | break; | |
826 | case IWCH_QP_STATE_RTS: | |
827 | switch (attrs->next_state) { | |
828 | case IWCH_QP_STATE_CLOSING: | |
829 | BUG_ON(atomic_read(&qhp->ep->com.kref.refcount) < 2); | |
830 | qhp->attr.state = IWCH_QP_STATE_CLOSING; | |
831 | if (!internal) { | |
832 | abort=0; | |
833 | disconnect = 1; | |
834 | ep = qhp->ep; | |
989a1780 | 835 | get_ep(&ep->com); |
b038ced7 | 836 | } |
856b5925 | 837 | flush_qp(qhp, &flag); |
b038ced7 SW |
838 | break; |
839 | case IWCH_QP_STATE_TERMINATE: | |
840 | qhp->attr.state = IWCH_QP_STATE_TERMINATE; | |
856b5925 | 841 | if (qhp->ibqp.uobject) |
a1a75052 | 842 | cxio_set_wq_in_error(&qhp->wq); |
b038ced7 SW |
843 | if (!internal) |
844 | terminate = 1; | |
845 | break; | |
846 | case IWCH_QP_STATE_ERROR: | |
847 | qhp->attr.state = IWCH_QP_STATE_ERROR; | |
848 | if (!internal) { | |
849 | abort=1; | |
850 | disconnect = 1; | |
851 | ep = qhp->ep; | |
989a1780 | 852 | get_ep(&ep->com); |
b038ced7 SW |
853 | } |
854 | goto err; | |
855 | break; | |
856 | default: | |
857 | ret = -EINVAL; | |
858 | goto out; | |
859 | } | |
860 | break; | |
861 | case IWCH_QP_STATE_CLOSING: | |
862 | if (!internal) { | |
863 | ret = -EINVAL; | |
864 | goto out; | |
865 | } | |
866 | switch (attrs->next_state) { | |
867 | case IWCH_QP_STATE_IDLE: | |
868 | qhp->attr.state = IWCH_QP_STATE_IDLE; | |
869 | qhp->attr.llp_stream_handle = NULL; | |
870 | put_ep(&qhp->ep->com); | |
871 | qhp->ep = NULL; | |
872 | wake_up(&qhp->wait); | |
873 | break; | |
874 | case IWCH_QP_STATE_ERROR: | |
875 | goto err; | |
876 | default: | |
877 | ret = -EINVAL; | |
878 | goto err; | |
879 | } | |
880 | break; | |
881 | case IWCH_QP_STATE_ERROR: | |
882 | if (attrs->next_state != IWCH_QP_STATE_IDLE) { | |
883 | ret = -EINVAL; | |
884 | goto out; | |
885 | } | |
886 | ||
887 | if (!Q_EMPTY(qhp->wq.sq_rptr, qhp->wq.sq_wptr) || | |
888 | !Q_EMPTY(qhp->wq.rq_rptr, qhp->wq.rq_wptr)) { | |
889 | ret = -EINVAL; | |
890 | goto out; | |
891 | } | |
892 | qhp->attr.state = IWCH_QP_STATE_IDLE; | |
893 | memset(&qhp->attr, 0, sizeof(qhp->attr)); | |
894 | break; | |
895 | case IWCH_QP_STATE_TERMINATE: | |
896 | if (!internal) { | |
897 | ret = -EINVAL; | |
898 | goto out; | |
899 | } | |
900 | goto err; | |
901 | break; | |
902 | default: | |
903 | printk(KERN_ERR "%s in a bad state %d\n", | |
33718363 | 904 | __func__, qhp->attr.state); |
b038ced7 SW |
905 | ret = -EINVAL; |
906 | goto err; | |
907 | break; | |
908 | } | |
909 | goto out; | |
910 | err: | |
33718363 | 911 | PDBG("%s disassociating ep %p qpid 0x%x\n", __func__, qhp->ep, |
b038ced7 SW |
912 | qhp->wq.qpid); |
913 | ||
914 | /* disassociate the LLP connection */ | |
915 | qhp->attr.llp_stream_handle = NULL; | |
916 | ep = qhp->ep; | |
917 | qhp->ep = NULL; | |
918 | qhp->attr.state = IWCH_QP_STATE_ERROR; | |
919 | free=1; | |
920 | wake_up(&qhp->wait); | |
921 | BUG_ON(!ep); | |
922 | flush_qp(qhp, &flag); | |
923 | out: | |
924 | spin_unlock_irqrestore(&qhp->lock, flag); | |
925 | ||
926 | if (terminate) | |
927 | iwch_post_terminate(qhp, NULL); | |
928 | ||
929 | /* | |
930 | * If disconnect is 1, then we need to initiate a disconnect | |
931 | * on the EP. This can be a normal close (RTS->CLOSING) or | |
932 | * an abnormal close (RTS/CLOSING->ERROR). | |
933 | */ | |
989a1780 | 934 | if (disconnect) { |
b038ced7 | 935 | iwch_ep_disconnect(ep, abort, GFP_KERNEL); |
989a1780 SW |
936 | put_ep(&ep->com); |
937 | } | |
b038ced7 SW |
938 | |
939 | /* | |
940 | * If free is 1, then we've disassociated the EP from the QP | |
941 | * and we need to dereference the EP. | |
942 | */ | |
943 | if (free) | |
944 | put_ep(&ep->com); | |
945 | ||
33718363 | 946 | PDBG("%s exit state %d\n", __func__, qhp->attr.state); |
b038ced7 SW |
947 | return ret; |
948 | } | |
949 | ||
950 | static int quiesce_qp(struct iwch_qp *qhp) | |
951 | { | |
952 | spin_lock_irq(&qhp->lock); | |
953 | iwch_quiesce_tid(qhp->ep); | |
954 | qhp->flags |= QP_QUIESCED; | |
955 | spin_unlock_irq(&qhp->lock); | |
956 | return 0; | |
957 | } | |
958 | ||
959 | static int resume_qp(struct iwch_qp *qhp) | |
960 | { | |
961 | spin_lock_irq(&qhp->lock); | |
962 | iwch_resume_tid(qhp->ep); | |
963 | qhp->flags &= ~QP_QUIESCED; | |
964 | spin_unlock_irq(&qhp->lock); | |
965 | return 0; | |
966 | } | |
967 | ||
968 | int iwch_quiesce_qps(struct iwch_cq *chp) | |
969 | { | |
970 | int i; | |
971 | struct iwch_qp *qhp; | |
972 | ||
973 | for (i=0; i < T3_MAX_NUM_QP; i++) { | |
974 | qhp = get_qhp(chp->rhp, i); | |
975 | if (!qhp) | |
976 | continue; | |
977 | if ((qhp->attr.rcq == chp->cq.cqid) && !qp_quiesced(qhp)) { | |
978 | quiesce_qp(qhp); | |
979 | continue; | |
980 | } | |
981 | if ((qhp->attr.scq == chp->cq.cqid) && !qp_quiesced(qhp)) | |
982 | quiesce_qp(qhp); | |
983 | } | |
984 | return 0; | |
985 | } | |
986 | ||
987 | int iwch_resume_qps(struct iwch_cq *chp) | |
988 | { | |
989 | int i; | |
990 | struct iwch_qp *qhp; | |
991 | ||
992 | for (i=0; i < T3_MAX_NUM_QP; i++) { | |
993 | qhp = get_qhp(chp->rhp, i); | |
994 | if (!qhp) | |
995 | continue; | |
996 | if ((qhp->attr.rcq == chp->cq.cqid) && qp_quiesced(qhp)) { | |
997 | resume_qp(qhp); | |
998 | continue; | |
999 | } | |
1000 | if ((qhp->attr.scq == chp->cq.cqid) && qp_quiesced(qhp)) | |
1001 | resume_qp(qhp); | |
1002 | } | |
1003 | return 0; | |
1004 | } |