[net-next-2.6.git] / drivers / infiniband / hw / ehca / ehca_cq.c

/*
 *  IBM eServer eHCA Infiniband device driver for Linux on POWER
 *
 *  Completion queue handling
 *
 *  Authors: Waleri Fomin <fomin@de.ibm.com>
 *           Khadija Souissi <souissi@de.ibm.com>
 *           Reinhard Ernst <rernst@de.ibm.com>
 *           Heiko J Schick <schickhj@de.ibm.com>
 *           Hoang-Nam Nguyen <hnguyen@de.ibm.com>
 *
 *
 *  Copyright (c) 2005 IBM Corporation
 *
 *  All rights reserved.
 *
 *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
 *  BSD.
 *
 * OpenIB BSD License
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 *
 * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials
 * provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <asm/current.h>

#include "ehca_iverbs.h"
#include "ehca_classes.h"
#include "ehca_irq.h"
#include "hcp_if.h"

static struct kmem_cache *cq_cache;

int ehca_cq_assign_qp(struct ehca_cq *cq, struct ehca_qp *qp)
{
	unsigned int qp_num = qp->real_qp_num;
	unsigned int key = qp_num & (QP_HASHTAB_LEN-1);
	unsigned long spl_flags;

	spin_lock_irqsave(&cq->spinlock, spl_flags);
	hlist_add_head(&qp->list_entries, &cq->qp_hashtab[key]);
	spin_unlock_irqrestore(&cq->spinlock, spl_flags);

	ehca_dbg(cq->ib_cq.device, "cq_num=%x real_qp_num=%x",
		 cq->cq_number, qp_num);

	return 0;
}

int ehca_cq_unassign_qp(struct ehca_cq *cq, unsigned int real_qp_num)
{
	int ret = -EINVAL;
	unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
	struct hlist_node *iter;
	struct ehca_qp *qp;
	unsigned long spl_flags;

	spin_lock_irqsave(&cq->spinlock, spl_flags);
	hlist_for_each(iter, &cq->qp_hashtab[key]) {
		qp = hlist_entry(iter, struct ehca_qp, list_entries);
		if (qp->real_qp_num == real_qp_num) {
			hlist_del(iter);
			ehca_dbg(cq->ib_cq.device,
				 "removed qp from cq .cq_num=%x real_qp_num=%x",
				 cq->cq_number, real_qp_num);
			ret = 0;
			break;
		}
	}
	spin_unlock_irqrestore(&cq->spinlock, spl_flags);
	if (ret)
		ehca_err(cq->ib_cq.device,
			 "qp not found cq_num=%x real_qp_num=%x",
			 cq->cq_number, real_qp_num);

	return ret;
}

struct ehca_qp* ehca_cq_get_qp(struct ehca_cq *cq, int real_qp_num)
{
	struct ehca_qp *ret = NULL;
	unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
	struct hlist_node *iter;
	struct ehca_qp *qp;
	hlist_for_each(iter, &cq->qp_hashtab[key]) {
		qp = hlist_entry(iter, struct ehca_qp, list_entries);
		if (qp->real_qp_num == real_qp_num) {
			ret = qp;
			break;
		}
	}
	return ret;
}

struct ib_cq *ehca_create_cq(struct ib_device *device, int cqe,
			     struct ib_ucontext *context,
			     struct ib_udata *udata)
{
	static const u32 additional_cqe = 20;
	struct ib_cq *cq;
	struct ehca_cq *my_cq;
	struct ehca_shca *shca =
		container_of(device, struct ehca_shca, ib_device);
	struct ipz_adapter_handle adapter_handle;
	struct ehca_alloc_cq_parms param; /* h_call's out parameters */
	struct h_galpa gal;
	void *vpage;
	u32 counter;
	u64 rpage, cqx_fec, h_ret;
	int ipz_rc, ret, i;
	unsigned long flags;

	if (cqe >= 0xFFFFFFFF - 64 - additional_cqe)
		return ERR_PTR(-EINVAL);

	my_cq = kmem_cache_alloc(cq_cache, GFP_KERNEL);
	if (!my_cq) {
		ehca_err(device, "Out of memory for ehca_cq struct device=%p",
			 device);
		return ERR_PTR(-ENOMEM);
	}

	memset(my_cq, 0, sizeof(struct ehca_cq));
	memset(&param, 0, sizeof(struct ehca_alloc_cq_parms));

	spin_lock_init(&my_cq->spinlock);
	spin_lock_init(&my_cq->cb_lock);
	spin_lock_init(&my_cq->task_lock);
	my_cq->ownpid = current->tgid;

	cq = &my_cq->ib_cq;

	adapter_handle = shca->ipz_hca_handle;
	param.eq_handle = shca->eq.ipz_eq_handle;

	do {
		if (!idr_pre_get(&ehca_cq_idr, GFP_KERNEL)) {
			cq = ERR_PTR(-ENOMEM);
			ehca_err(device, "Can't reserve idr nr. device=%p",
				 device);
			goto create_cq_exit1;
		}

		spin_lock_irqsave(&ehca_cq_idr_lock, flags);
		ret = idr_get_new(&ehca_cq_idr, my_cq, &my_cq->token);
		spin_unlock_irqrestore(&ehca_cq_idr_lock, flags);

	} while (ret == -EAGAIN);

	if (ret) {
		cq = ERR_PTR(-ENOMEM);
		ehca_err(device, "Can't allocate new idr entry. device=%p",
			 device);
		goto create_cq_exit1;
	}

	/*
	 * CQs maximum depth is 4GB-64, but we need additional 20 as buffer
	 * for receiving errors CQEs.
	 */
	param.nr_cqe = cqe + additional_cqe;
	h_ret = hipz_h_alloc_resource_cq(adapter_handle, my_cq, &param);

	if (h_ret != H_SUCCESS) {
		ehca_err(device, "hipz_h_alloc_resource_cq() failed "
			 "h_ret=%lx device=%p", h_ret, device);
		cq = ERR_PTR(ehca2ib_return_code(h_ret));
		goto create_cq_exit2;
	}

	ipz_rc = ipz_queue_ctor(&my_cq->ipz_queue, param.act_pages,
				EHCA_PAGESIZE, sizeof(struct ehca_cqe), 0);
	if (!ipz_rc) {
		ehca_err(device, "ipz_queue_ctor() failed ipz_rc=%x device=%p",
			 ipz_rc, device);
		cq = ERR_PTR(-EINVAL);
		goto create_cq_exit3;
	}

	for (counter = 0; counter < param.act_pages; counter++) {
		vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue);
		if (!vpage) {
			ehca_err(device, "ipz_qpageit_get_inc() "
				 "returns NULL device=%p", device);
			cq = ERR_PTR(-EAGAIN);
			goto create_cq_exit4;
		}
		rpage = virt_to_abs(vpage);

		h_ret = hipz_h_register_rpage_cq(adapter_handle,
						 my_cq->ipz_cq_handle,
						 &my_cq->pf,
						 0,
						 0,
						 rpage,
						 1,
						 my_cq->galpas.
						 kernel);

		if (h_ret < H_SUCCESS) {
			ehca_err(device, "hipz_h_register_rpage_cq() failed "
				 "ehca_cq=%p cq_num=%x h_ret=%lx counter=%i "
				 "act_pages=%i", my_cq, my_cq->cq_number,
				 h_ret, counter, param.act_pages);
			cq = ERR_PTR(-EINVAL);
			goto create_cq_exit4;
		}

		if (counter == (param.act_pages - 1)) {
			vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue);
			if ((h_ret != H_SUCCESS) || vpage) {
				ehca_err(device, "Registration of pages not "
					 "complete ehca_cq=%p cq_num=%x "
					 "h_ret=%lx", my_cq, my_cq->cq_number,
					 h_ret);
				cq = ERR_PTR(-EAGAIN);
				goto create_cq_exit4;
			}
		} else {
			if (h_ret != H_PAGE_REGISTERED) {
				ehca_err(device, "Registration of page failed "
					 "ehca_cq=%p cq_num=%x h_ret=%lx"
					 "counter=%i act_pages=%i",
					 my_cq, my_cq->cq_number,
					 h_ret, counter, param.act_pages);
				cq = ERR_PTR(-ENOMEM);
				goto create_cq_exit4;
			}
		}
	}

	ipz_qeit_reset(&my_cq->ipz_queue);

	gal = my_cq->galpas.kernel;
	cqx_fec = hipz_galpa_load(gal, CQTEMM_OFFSET(cqx_fec));
	ehca_dbg(device, "ehca_cq=%p cq_num=%x CQX_FEC=%lx",
		 my_cq, my_cq->cq_number, cqx_fec);

	my_cq->ib_cq.cqe = my_cq->nr_of_entries =
		param.act_nr_of_entries - additional_cqe;
	my_cq->cq_number = (my_cq->ipz_cq_handle.handle) & 0xffff;

	for (i = 0; i < QP_HASHTAB_LEN; i++)
		INIT_HLIST_HEAD(&my_cq->qp_hashtab[i]);

	if (context) {
		struct ipz_queue *ipz_queue = &my_cq->ipz_queue;
		struct ehca_create_cq_resp resp;
		struct vm_area_struct *vma;
		memset(&resp, 0, sizeof(resp));
		resp.cq_number = my_cq->cq_number;
		resp.token = my_cq->token;
		resp.ipz_queue.qe_size = ipz_queue->qe_size;
		resp.ipz_queue.act_nr_of_sg = ipz_queue->act_nr_of_sg;
		resp.ipz_queue.queue_length = ipz_queue->queue_length;
		resp.ipz_queue.pagesize = ipz_queue->pagesize;
		resp.ipz_queue.toggle_state = ipz_queue->toggle_state;
		ret = ehca_mmap_nopage(((u64)(my_cq->token) << 32) | 0x12000000,
				       ipz_queue->queue_length,
				       (void**)&resp.ipz_queue.queue,
				       &vma);
		if (ret) {
			ehca_err(device, "Could not mmap queue pages");
			cq = ERR_PTR(ret);
			goto create_cq_exit4;
		}
		my_cq->uspace_queue = resp.ipz_queue.queue;
		resp.galpas = my_cq->galpas;
		ret = ehca_mmap_register(my_cq->galpas.user.fw_handle,
					 (void**)&resp.galpas.kernel.fw_handle,
					 &vma);
		if (ret) {
			ehca_err(device, "Could not mmap fw_handle");
			cq = ERR_PTR(ret);
			goto create_cq_exit5;
		}
		my_cq->uspace_fwh = (u64)resp.galpas.kernel.fw_handle;
		if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
			ehca_err(device, "Copy to udata failed.");
			goto create_cq_exit6;
		}
	}

	return cq;

create_cq_exit6:
	ehca_munmap(my_cq->uspace_fwh, EHCA_PAGESIZE);

create_cq_exit5:
	ehca_munmap(my_cq->uspace_queue, my_cq->ipz_queue.queue_length);

create_cq_exit4:
	ipz_queue_dtor(&my_cq->ipz_queue);

create_cq_exit3:
	h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
	if (h_ret != H_SUCCESS)
		ehca_err(device, "hipz_h_destroy_cq() failed ehca_cq=%p "
			 "cq_num=%x h_ret=%lx", my_cq, my_cq->cq_number, h_ret);

create_cq_exit2:
	spin_lock_irqsave(&ehca_cq_idr_lock, flags);
	idr_remove(&ehca_cq_idr, my_cq->token);
	spin_unlock_irqrestore(&ehca_cq_idr_lock, flags);

create_cq_exit1:
	kmem_cache_free(cq_cache, my_cq);

	return cq;
}

int ehca_destroy_cq(struct ib_cq *cq)
{
	u64 h_ret;
	int ret;
	struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
	int cq_num = my_cq->cq_number;
	struct ib_device *device = cq->device;
	struct ehca_shca *shca = container_of(device, struct ehca_shca,
					      ib_device);
	struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle;
	u32 cur_pid = current->tgid;
	unsigned long flags;

	spin_lock_irqsave(&ehca_cq_idr_lock, flags);
	while (my_cq->nr_callbacks)
		yield();

	idr_remove(&ehca_cq_idr, my_cq->token);
	spin_unlock_irqrestore(&ehca_cq_idr_lock, flags);

	if (my_cq->uspace_queue && my_cq->ownpid != cur_pid) {
		ehca_err(device, "Invalid caller pid=%x ownpid=%x",
			 cur_pid, my_cq->ownpid);
		return -EINVAL;
	}

	/* un-mmap if vma alloc */
	if (my_cq->uspace_queue ) {
		ret = ehca_munmap(my_cq->uspace_queue,
				  my_cq->ipz_queue.queue_length);
		if (ret)
			ehca_err(device, "Could not munmap queue ehca_cq=%p "
				 "cq_num=%x", my_cq, cq_num);
		ret = ehca_munmap(my_cq->uspace_fwh, EHCA_PAGESIZE);
		if (ret)
			ehca_err(device, "Could not munmap fwh ehca_cq=%p "
				 "cq_num=%x", my_cq, cq_num);
	}

	h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 0);
	if (h_ret == H_R_STATE) {
		/* cq in err: read err data and destroy it forcibly */
		ehca_dbg(device, "ehca_cq=%p cq_num=%x ressource=%lx in err "
			 "state. Try to delete it forcibly.",
			 my_cq, cq_num, my_cq->ipz_cq_handle.handle);
		ehca_error_data(shca, my_cq, my_cq->ipz_cq_handle.handle);
		h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
		if (h_ret == H_SUCCESS)
			ehca_dbg(device, "cq_num=%x deleted successfully.",
				 cq_num);
	}
	if (h_ret != H_SUCCESS) {
		ehca_err(device, "hipz_h_destroy_cq() failed h_ret=%lx "
			 "ehca_cq=%p cq_num=%x", h_ret, my_cq, cq_num);
		return ehca2ib_return_code(h_ret);
	}
	ipz_queue_dtor(&my_cq->ipz_queue);
	kmem_cache_free(cq_cache, my_cq);

	return 0;
}

int ehca_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)
{
	struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
	u32 cur_pid = current->tgid;

	if (my_cq->uspace_queue && my_cq->ownpid != cur_pid) {
		ehca_err(cq->device, "Invalid caller pid=%x ownpid=%x",
			 cur_pid, my_cq->ownpid);
		return -EINVAL;
	}

	/* TODO: proper resize needs to be done */
	ehca_err(cq->device, "not implemented yet");

	return -EFAULT;
}

int ehca_init_cq_cache(void)
{
	cq_cache = kmem_cache_create("ehca_cache_cq",
				     sizeof(struct ehca_cq), 0,
				     SLAB_HWCACHE_ALIGN,
				     NULL, NULL);
	if (!cq_cache)
		return -ENOMEM;
	return 0;
}

void ehca_cleanup_cq_cache(void)
{
	if (cq_cache)
		kmem_cache_destroy(cq_cache);
}
Commit	Line	Data
fab97220 HS	1	/*
	2	* IBM eServer eHCA Infiniband device driver for Linux on POWER
	3	*
	4	* Completion queue handling
	5	*
	6	* Authors: Waleri Fomin <fomin@de.ibm.com>
	7	* Khadija Souissi <souissi@de.ibm.com>
	8	* Reinhard Ernst <rernst@de.ibm.com>
	9	* Heiko J Schick <schickhj@de.ibm.com>
	10	* Hoang-Nam Nguyen <hnguyen@de.ibm.com>
	11	*
	12	*
	13	* Copyright (c) 2005 IBM Corporation
	14	*
	15	* All rights reserved.
	16	*
	17	* This source code is distributed under a dual license of GPL v2.0 and OpenIB
	18	* BSD.
	19	*
	20	* OpenIB BSD License
	21	*
	22	* Redistribution and use in source and binary forms, with or without
	23	* modification, are permitted provided that the following conditions are met:
	24	*
	25	* Redistributions of source code must retain the above copyright notice, this
	26	* list of conditions and the following disclaimer.
	27	*
	28	* Redistributions in binary form must reproduce the above copyright notice,
	29	* this list of conditions and the following disclaimer in the documentation
	30	* and/or other materials
	31	* provided with the distribution.
	32	*
	33	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
	34	* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	35	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	36	* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
	37	* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
	38	* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
	39	* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
	40	* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
	41	* IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
	42	* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
	43	* POSSIBILITY OF SUCH DAMAGE.
	44	*/
	45
	46	#include <asm/current.h>
	47
	48	#include "ehca_iverbs.h"
	49	#include "ehca_classes.h"
	50	#include "ehca_irq.h"
	51	#include "hcp_if.h"
	52
	53	static struct kmem_cache *cq_cache;
	54
	55	int ehca_cq_assign_qp(struct ehca_cq cq, struct ehca_qp qp)
	56	{
	57	unsigned int qp_num = qp->real_qp_num;
	58	unsigned int key = qp_num & (QP_HASHTAB_LEN-1);
	59	unsigned long spl_flags;
	60
	61	spin_lock_irqsave(&cq->spinlock, spl_flags);
	62	hlist_add_head(&qp->list_entries, &cq->qp_hashtab[key]);
	63	spin_unlock_irqrestore(&cq->spinlock, spl_flags);
	64
65	ehca_dbg(cq->ib_cq.device, "cq_num=%x real_qp_num=%x",
66	cq->cq_number, qp_num);
67
68	return 0;
69	}
70
71	int ehca_cq_unassign_qp(struct ehca_cq *cq, unsigned int real_qp_num)
72	{
73	int ret = -EINVAL;
74	unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
75	struct hlist_node *iter;
76	struct ehca_qp *qp;
77	unsigned long spl_flags;
78
79	spin_lock_irqsave(&cq->spinlock, spl_flags);
80	hlist_for_each(iter, &cq->qp_hashtab[key]) {
81	qp = hlist_entry(iter, struct ehca_qp, list_entries);
82	if (qp->real_qp_num == real_qp_num) {
83	hlist_del(iter);
84	ehca_dbg(cq->ib_cq.device,
85	"removed qp from cq .cq_num=%x real_qp_num=%x",
86	cq->cq_number, real_qp_num);
87	ret = 0;
88	break;
89	}
90	}
91	spin_unlock_irqrestore(&cq->spinlock, spl_flags);
92	if (ret)
93	ehca_err(cq->ib_cq.device,
94	"qp not found cq_num=%x real_qp_num=%x",
95	cq->cq_number, real_qp_num);
96
97	return ret;
98	}
99
100	struct ehca_qp* ehca_cq_get_qp(struct ehca_cq *cq, int real_qp_num)
101	{
102	struct ehca_qp *ret = NULL;
103	unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
104	struct hlist_node *iter;
105	struct ehca_qp *qp;
106	hlist_for_each(iter, &cq->qp_hashtab[key]) {
107	qp = hlist_entry(iter, struct ehca_qp, list_entries);
108	if (qp->real_qp_num == real_qp_num) {
109	ret = qp;
110	break;
111	}
112	}
113	return ret;
114	}
115
116	struct ib_cq ehca_create_cq(struct ib_device device, int cqe,
117	struct ib_ucontext *context,
118	struct ib_udata *udata)
119	{
120	static const u32 additional_cqe = 20;
121	struct ib_cq *cq;
122	struct ehca_cq *my_cq;
123	struct ehca_shca *shca =
124	container_of(device, struct ehca_shca, ib_device);
125	struct ipz_adapter_handle adapter_handle;
126	struct ehca_alloc_cq_parms param; /* h_call's out parameters */
127	struct h_galpa gal;
128	void *vpage;
129	u32 counter;
130	u64 rpage, cqx_fec, h_ret;
131	int ipz_rc, ret, i;
132	unsigned long flags;
133
134	if (cqe >= 0xFFFFFFFF - 64 - additional_cqe)
135	return ERR_PTR(-EINVAL);
136
e94b1766	137	my_cq = kmem_cache_alloc(cq_cache, GFP_KERNEL);
fab97220 HS	138	if (!my_cq) {
	139	ehca_err(device, "Out of memory for ehca_cq struct device=%p",
	140	device);
	141	return ERR_PTR(-ENOMEM);
	142	}
	143
	144	memset(my_cq, 0, sizeof(struct ehca_cq));
	145	memset(&param, 0, sizeof(struct ehca_alloc_cq_parms));
	146
	147	spin_lock_init(&my_cq->spinlock);
	148	spin_lock_init(&my_cq->cb_lock);
	149	spin_lock_init(&my_cq->task_lock);
	150	my_cq->ownpid = current->tgid;
	151
	152	cq = &my_cq->ib_cq;
	153
	154	adapter_handle = shca->ipz_hca_handle;
	155	param.eq_handle = shca->eq.ipz_eq_handle;
	156
	157	do {
	158	if (!idr_pre_get(&ehca_cq_idr, GFP_KERNEL)) {
	159	cq = ERR_PTR(-ENOMEM);
	160	ehca_err(device, "Can't reserve idr nr. device=%p",
	161	device);
	162	goto create_cq_exit1;
	163	}
	164
	165	spin_lock_irqsave(&ehca_cq_idr_lock, flags);
	166	ret = idr_get_new(&ehca_cq_idr, my_cq, &my_cq->token);
	167	spin_unlock_irqrestore(&ehca_cq_idr_lock, flags);
	168
	169	} while (ret == -EAGAIN);
	170
	171	if (ret) {
	172	cq = ERR_PTR(-ENOMEM);
	173	ehca_err(device, "Can't allocate new idr entry. device=%p",
	174	device);
	175	goto create_cq_exit1;
	176	}
	177
	178	/*
	179	* CQs maximum depth is 4GB-64, but we need additional 20 as buffer
	180	* for receiving errors CQEs.
	181	*/
	182	param.nr_cqe = cqe + additional_cqe;
	183	h_ret = hipz_h_alloc_resource_cq(adapter_handle, my_cq, &param);
	184
	185	if (h_ret != H_SUCCESS) {
	186	ehca_err(device, "hipz_h_alloc_resource_cq() failed "
	187	"h_ret=%lx device=%p", h_ret, device);
	188	cq = ERR_PTR(ehca2ib_return_code(h_ret));
	189	goto create_cq_exit2;
	190	}
	191
	192	ipz_rc = ipz_queue_ctor(&my_cq->ipz_queue, param.act_pages,
	193	EHCA_PAGESIZE, sizeof(struct ehca_cqe), 0);
	194	if (!ipz_rc) {
	195	ehca_err(device, "ipz_queue_ctor() failed ipz_rc=%x device=%p",
	196	ipz_rc, device);
	197	cq = ERR_PTR(-EINVAL);
	198	goto create_cq_exit3;
	199	}
	200
	201	for (counter = 0; counter < param.act_pages; counter++) {
202	vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue);
203	if (!vpage) {
204	ehca_err(device, "ipz_qpageit_get_inc() "
205	"returns NULL device=%p", device);
206	cq = ERR_PTR(-EAGAIN);
207	goto create_cq_exit4;
208	}
209	rpage = virt_to_abs(vpage);
210
211	h_ret = hipz_h_register_rpage_cq(adapter_handle,
212	my_cq->ipz_cq_handle,
213	&my_cq->pf,
214	0,
215	0,
216	rpage,
217	1,
218	my_cq->galpas.
219	kernel);
220
221	if (h_ret < H_SUCCESS) {
222	ehca_err(device, "hipz_h_register_rpage_cq() failed "
223	"ehca_cq=%p cq_num=%x h_ret=%lx counter=%i "
224	"act_pages=%i", my_cq, my_cq->cq_number,
225	h_ret, counter, param.act_pages);
226	cq = ERR_PTR(-EINVAL);
227	goto create_cq_exit4;
228	}
229
230	if (counter == (param.act_pages - 1)) {
231	vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue);
232	if ((h_ret != H_SUCCESS) \|\| vpage) {
233	ehca_err(device, "Registration of pages not "
234	"complete ehca_cq=%p cq_num=%x "
235	"h_ret=%lx", my_cq, my_cq->cq_number,
236	h_ret);
237	cq = ERR_PTR(-EAGAIN);
238	goto create_cq_exit4;
239	}
240	} else {
241	if (h_ret != H_PAGE_REGISTERED) {
242	ehca_err(device, "Registration of page failed "
243	"ehca_cq=%p cq_num=%x h_ret=%lx"
244	"counter=%i act_pages=%i",
245	my_cq, my_cq->cq_number,
246	h_ret, counter, param.act_pages);
247	cq = ERR_PTR(-ENOMEM);
248	goto create_cq_exit4;
249	}
250	}
251	}
252
253	ipz_qeit_reset(&my_cq->ipz_queue);
254
255	gal = my_cq->galpas.kernel;
256	cqx_fec = hipz_galpa_load(gal, CQTEMM_OFFSET(cqx_fec));
257	ehca_dbg(device, "ehca_cq=%p cq_num=%x CQX_FEC=%lx",
258	my_cq, my_cq->cq_number, cqx_fec);
259
260	my_cq->ib_cq.cqe = my_cq->nr_of_entries =
261	param.act_nr_of_entries - additional_cqe;
262	my_cq->cq_number = (my_cq->ipz_cq_handle.handle) & 0xffff;
263
264	for (i = 0; i < QP_HASHTAB_LEN; i++)
265	INIT_HLIST_HEAD(&my_cq->qp_hashtab[i]);
266
267	if (context) {
268	struct ipz_queue *ipz_queue = &my_cq->ipz_queue;
269	struct ehca_create_cq_resp resp;
270	struct vm_area_struct *vma;
271	memset(&resp, 0, sizeof(resp));
272	resp.cq_number = my_cq->cq_number;
273	resp.token = my_cq->token;
274	resp.ipz_queue.qe_size = ipz_queue->qe_size;
275	resp.ipz_queue.act_nr_of_sg = ipz_queue->act_nr_of_sg;
276	resp.ipz_queue.queue_length = ipz_queue->queue_length;
277	resp.ipz_queue.pagesize = ipz_queue->pagesize;
278	resp.ipz_queue.toggle_state = ipz_queue->toggle_state;
279	ret = ehca_mmap_nopage(((u64)(my_cq->token) << 32) \| 0x12000000,
280	ipz_queue->queue_length,
281	(void**)&resp.ipz_queue.queue,
282	&vma);
283	if (ret) {
284	ehca_err(device, "Could not mmap queue pages");
285	cq = ERR_PTR(ret);
286	goto create_cq_exit4;
287	}
288	my_cq->uspace_queue = resp.ipz_queue.queue;
289	resp.galpas = my_cq->galpas;
290	ret = ehca_mmap_register(my_cq->galpas.user.fw_handle,
291	(void**)&resp.galpas.kernel.fw_handle,
292	&vma);
293	if (ret) {
294	ehca_err(device, "Could not mmap fw_handle");
295	cq = ERR_PTR(ret);
296	goto create_cq_exit5;
297	}
298	my_cq->uspace_fwh = (u64)resp.galpas.kernel.fw_handle;
299	if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
300	ehca_err(device, "Copy to udata failed.");
301	goto create_cq_exit6;
302	}
303	}
304
305	return cq;
306
307	create_cq_exit6:
308	ehca_munmap(my_cq->uspace_fwh, EHCA_PAGESIZE);
309
310	create_cq_exit5:
311	ehca_munmap(my_cq->uspace_queue, my_cq->ipz_queue.queue_length);
312
313	create_cq_exit4:
314	ipz_queue_dtor(&my_cq->ipz_queue);
315
316	create_cq_exit3:
317	h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
318	if (h_ret != H_SUCCESS)
319	ehca_err(device, "hipz_h_destroy_cq() failed ehca_cq=%p "
320	"cq_num=%x h_ret=%lx", my_cq, my_cq->cq_number, h_ret);
321
322	create_cq_exit2:
323	spin_lock_irqsave(&ehca_cq_idr_lock, flags);
324	idr_remove(&ehca_cq_idr, my_cq->token);
325	spin_unlock_irqrestore(&ehca_cq_idr_lock, flags);
326
327	create_cq_exit1:
328	kmem_cache_free(cq_cache, my_cq);
329
330	return cq;
331	}
332
333	int ehca_destroy_cq(struct ib_cq *cq)
334	{
335	u64 h_ret;
336	int ret;
337	struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
338	int cq_num = my_cq->cq_number;
339	struct ib_device *device = cq->device;
340	struct ehca_shca *shca = container_of(device, struct ehca_shca,
341	ib_device);
342	struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle;
343	u32 cur_pid = current->tgid;
344	unsigned long flags;
345
346	spin_lock_irqsave(&ehca_cq_idr_lock, flags);
347	while (my_cq->nr_callbacks)
348	yield();
349
350	idr_remove(&ehca_cq_idr, my_cq->token);
351	spin_unlock_irqrestore(&ehca_cq_idr_lock, flags);
352
353	if (my_cq->uspace_queue && my_cq->ownpid != cur_pid) {
354	ehca_err(device, "Invalid caller pid=%x ownpid=%x",
355	cur_pid, my_cq->ownpid);
356	return -EINVAL;
357	}
358
359	/* un-mmap if vma alloc */
360	if (my_cq->uspace_queue ) {
361	ret = ehca_munmap(my_cq->uspace_queue,
362	my_cq->ipz_queue.queue_length);
363	if (ret)
364	ehca_err(device, "Could not munmap queue ehca_cq=%p "
365	"cq_num=%x", my_cq, cq_num);
366	ret = ehca_munmap(my_cq->uspace_fwh, EHCA_PAGESIZE);
367	if (ret)
368	ehca_err(device, "Could not munmap fwh ehca_cq=%p "
369	"cq_num=%x", my_cq, cq_num);
370	}
371
372	h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 0);
373	if (h_ret == H_R_STATE) {
374	/* cq in err: read err data and destroy it forcibly */
375	ehca_dbg(device, "ehca_cq=%p cq_num=%x ressource=%lx in err "
376	"state. Try to delete it forcibly.",
377	my_cq, cq_num, my_cq->ipz_cq_handle.handle);
378	ehca_error_data(shca, my_cq, my_cq->ipz_cq_handle.handle);
379	h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
380	if (h_ret == H_SUCCESS)
381	ehca_dbg(device, "cq_num=%x deleted successfully.",
382	cq_num);
383	}
384	if (h_ret != H_SUCCESS) {
385	ehca_err(device, "hipz_h_destroy_cq() failed h_ret=%lx "
386	"ehca_cq=%p cq_num=%x", h_ret, my_cq, cq_num);
387	return ehca2ib_return_code(h_ret);
388	}
389	ipz_queue_dtor(&my_cq->ipz_queue);
390	kmem_cache_free(cq_cache, my_cq);
391
392	return 0;
393	}
394
395	int ehca_resize_cq(struct ib_cq cq, int cqe, struct ib_udata udata)
396	{
397	struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
398	u32 cur_pid = current->tgid;
399
400	if (my_cq->uspace_queue && my_cq->ownpid != cur_pid) {
401	ehca_err(cq->device, "Invalid caller pid=%x ownpid=%x",
402	cur_pid, my_cq->ownpid);
403	return -EINVAL;
404	}
405
406	/* TODO: proper resize needs to be done */
407	ehca_err(cq->device, "not implemented yet");
408
409	return -EFAULT;
410	}
411
412	int ehca_init_cq_cache(void)
413	{
414	cq_cache = kmem_cache_create("ehca_cache_cq",
415	sizeof(struct ehca_cq), 0,
416	SLAB_HWCACHE_ALIGN,
417	NULL, NULL);
418	if (!cq_cache)
419	return -ENOMEM;
420	return 0;
421	}
422
423	void ehca_cleanup_cq_cache(void)
424	{
425	if (cq_cache)
426	kmem_cache_destroy(cq_cache);
427	}