[net-next-2.6.git] / crypto / async_tx / async_raid6_recov.c

/*
 * Asynchronous RAID-6 recovery calculations ASYNC_TX API.
 * Copyright(c) 2009 Intel Corporation
 *
 * based on raid6recov.c:
 *   Copyright 2002 H. Peter Anvin
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc., 51
 * Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
 *
 */
#include <linux/kernel.h>
#include <linux/interrupt.h>
#include <linux/dma-mapping.h>
#include <linux/raid/pq.h>
#include <linux/async_tx.h>

static struct dma_async_tx_descriptor *
async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef,
		  size_t len, struct async_submit_ctl *submit)
{
	struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
						      &dest, 1, srcs, 2, len);
	struct dma_device *dma = chan ? chan->device : NULL;
	const u8 *amul, *bmul;
	u8 ax, bx;
	u8 *a, *b, *c;

	if (dma) {
		dma_addr_t dma_dest[2];
		dma_addr_t dma_src[2];
		struct device *dev = dma->dev;
		struct dma_async_tx_descriptor *tx;
		enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;

		if (submit->flags & ASYNC_TX_FENCE)
			dma_flags |= DMA_PREP_FENCE;
		dma_dest[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
		dma_src[0] = dma_map_page(dev, srcs[0], 0, len, DMA_TO_DEVICE);
		dma_src[1] = dma_map_page(dev, srcs[1], 0, len, DMA_TO_DEVICE);
		tx = dma->device_prep_dma_pq(chan, dma_dest, dma_src, 2, coef,
					     len, dma_flags);
		if (tx) {
			async_tx_submit(chan, tx, submit);
			return tx;
		}

		/* could not get a descriptor, unmap and fall through to
		 * the synchronous path
		 */
		dma_unmap_page(dev, dma_dest[1], len, DMA_BIDIRECTIONAL);
		dma_unmap_page(dev, dma_src[0], len, DMA_TO_DEVICE);
		dma_unmap_page(dev, dma_src[1], len, DMA_TO_DEVICE);
	}

	/* run the operation synchronously */
	async_tx_quiesce(&submit->depend_tx);
	amul = raid6_gfmul[coef[0]];
	bmul = raid6_gfmul[coef[1]];
	a = page_address(srcs[0]);
	b = page_address(srcs[1]);
	c = page_address(dest);

	while (len--) {
		ax    = amul[*a++];
		bx    = bmul[*b++];
		*c++ = ax ^ bx;
	}

	return NULL;
}

static struct dma_async_tx_descriptor *
async_mult(struct page *dest, struct page *src, u8 coef, size_t len,
	   struct async_submit_ctl *submit)
{
	struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
						      &dest, 1, &src, 1, len);
	struct dma_device *dma = chan ? chan->device : NULL;
	const u8 *qmul; /* Q multiplier table */
	u8 *d, *s;

	if (dma) {
		dma_addr_t dma_dest[2];
		dma_addr_t dma_src[1];
		struct device *dev = dma->dev;
		struct dma_async_tx_descriptor *tx;
		enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;

		if (submit->flags & ASYNC_TX_FENCE)
			dma_flags |= DMA_PREP_FENCE;
		dma_dest[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
		dma_src[0] = dma_map_page(dev, src, 0, len, DMA_TO_DEVICE);
		tx = dma->device_prep_dma_pq(chan, dma_dest, dma_src, 1, &coef,
					     len, dma_flags);
		if (tx) {
			async_tx_submit(chan, tx, submit);
			return tx;
		}

		/* could not get a descriptor, unmap and fall through to
		 * the synchronous path
		 */
		dma_unmap_page(dev, dma_dest[1], len, DMA_BIDIRECTIONAL);
		dma_unmap_page(dev, dma_src[0], len, DMA_TO_DEVICE);
	}

	/* no channel available, or failed to allocate a descriptor, so
	 * perform the operation synchronously
	 */
	async_tx_quiesce(&submit->depend_tx);
	qmul  = raid6_gfmul[coef];
	d = page_address(dest);
	s = page_address(src);

	while (len--)
		*d++ = qmul[*s++];

	return NULL;
}

static struct dma_async_tx_descriptor *
__2data_recov_4(int disks, size_t bytes, int faila, int failb,
		struct page **blocks, struct async_submit_ctl *submit)
{
	struct dma_async_tx_descriptor *tx = NULL;
	struct page *p, *q, *a, *b;
	struct page *srcs[2];
	unsigned char coef[2];
	enum async_tx_flags flags = submit->flags;
	dma_async_tx_callback cb_fn = submit->cb_fn;
	void *cb_param = submit->cb_param;
	void *scribble = submit->scribble;

	p = blocks[disks-2];
	q = blocks[disks-1];

	a = blocks[faila];
	b = blocks[failb];

	/* in the 4 disk case P + Pxy == P and Q + Qxy == Q */
	/* Dx = A*(P+Pxy) + B*(Q+Qxy) */
	srcs[0] = p;
	srcs[1] = q;
	coef[0] = raid6_gfexi[failb-faila];
	coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
	tx = async_sum_product(b, srcs, coef, bytes, submit);

	/* Dy = P+Pxy+Dx */
	srcs[0] = p;
	srcs[1] = b;
	init_async_submit(submit, flags | ASYNC_TX_XOR_ZERO_DST, tx, cb_fn,
			  cb_param, scribble);
	tx = async_xor(a, srcs, 0, 2, bytes, submit);

	return tx;

}

static struct dma_async_tx_descriptor *
__2data_recov_5(int disks, size_t bytes, int faila, int failb,
		struct page **blocks, struct async_submit_ctl *submit)
{
	struct dma_async_tx_descriptor *tx = NULL;
	struct page *p, *q, *g, *dp, *dq;
	struct page *srcs[2];
	unsigned char coef[2];
	enum async_tx_flags flags = submit->flags;
	dma_async_tx_callback cb_fn = submit->cb_fn;
	void *cb_param = submit->cb_param;
	void *scribble = submit->scribble;
	int good_srcs, good, i;

	good_srcs = 0;
	good = -1;
	for (i = 0; i < disks-2; i++) {
		if (blocks[i] == NULL)
			continue;
		if (i == faila || i == failb)
			continue;
		good = i;
		good_srcs++;
	}
	BUG_ON(good_srcs > 1);

	p = blocks[disks-2];
	q = blocks[disks-1];
	g = blocks[good];

	/* Compute syndrome with zero for the missing data pages
	 * Use the dead data pages as temporary storage for delta p and
	 * delta q
	 */
	dp = blocks[faila];
	dq = blocks[failb];

	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
	tx = async_memcpy(dp, g, 0, 0, bytes, submit);
	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
	tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit);

	/* compute P + Pxy */
	srcs[0] = dp;
	srcs[1] = p;
	init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
			  NULL, NULL, scribble);
	tx = async_xor(dp, srcs, 0, 2, bytes, submit);

	/* compute Q + Qxy */
	srcs[0] = dq;
	srcs[1] = q;
	init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
			  NULL, NULL, scribble);
	tx = async_xor(dq, srcs, 0, 2, bytes, submit);

	/* Dx = A*(P+Pxy) + B*(Q+Qxy) */
	srcs[0] = dp;
	srcs[1] = dq;
	coef[0] = raid6_gfexi[failb-faila];
	coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
	tx = async_sum_product(dq, srcs, coef, bytes, submit);

	/* Dy = P+Pxy+Dx */
	srcs[0] = dp;
	srcs[1] = dq;
	init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
			  cb_param, scribble);
	tx = async_xor(dp, srcs, 0, 2, bytes, submit);

	return tx;
}

static struct dma_async_tx_descriptor *
__2data_recov_n(int disks, size_t bytes, int faila, int failb,
	      struct page **blocks, struct async_submit_ctl *submit)
{
	struct dma_async_tx_descriptor *tx = NULL;
	struct page *p, *q, *dp, *dq;
	struct page *srcs[2];
	unsigned char coef[2];
	enum async_tx_flags flags = submit->flags;
	dma_async_tx_callback cb_fn = submit->cb_fn;
	void *cb_param = submit->cb_param;
	void *scribble = submit->scribble;

	p = blocks[disks-2];
	q = blocks[disks-1];

	/* Compute syndrome with zero for the missing data pages
	 * Use the dead data pages as temporary storage for
	 * delta p and delta q
	 */
	dp = blocks[faila];
	blocks[faila] = NULL;
	blocks[disks-2] = dp;
	dq = blocks[failb];
	blocks[failb] = NULL;
	blocks[disks-1] = dq;

	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
	tx = async_gen_syndrome(blocks, 0, disks, bytes, submit);

	/* Restore pointer table */
	blocks[faila]   = dp;
	blocks[failb]   = dq;
	blocks[disks-2] = p;
	blocks[disks-1] = q;

	/* compute P + Pxy */
	srcs[0] = dp;
	srcs[1] = p;
	init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
			  NULL, NULL, scribble);
	tx = async_xor(dp, srcs, 0, 2, bytes, submit);

	/* compute Q + Qxy */
	srcs[0] = dq;
	srcs[1] = q;
	init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
			  NULL, NULL, scribble);
	tx = async_xor(dq, srcs, 0, 2, bytes, submit);

	/* Dx = A*(P+Pxy) + B*(Q+Qxy) */
	srcs[0] = dp;
	srcs[1] = dq;
	coef[0] = raid6_gfexi[failb-faila];
	coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
	tx = async_sum_product(dq, srcs, coef, bytes, submit);

	/* Dy = P+Pxy+Dx */
	srcs[0] = dp;
	srcs[1] = dq;
	init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
			  cb_param, scribble);
	tx = async_xor(dp, srcs, 0, 2, bytes, submit);

	return tx;
}

/**
 * async_raid6_2data_recov - asynchronously calculate two missing data blocks
 * @disks: number of disks in the RAID-6 array
 * @bytes: block size
 * @faila: first failed drive index
 * @failb: second failed drive index
 * @blocks: array of source pointers where the last two entries are p and q
 * @submit: submission/completion modifiers
 */
struct dma_async_tx_descriptor *
async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb,
			struct page **blocks, struct async_submit_ctl *submit)
{
	void *scribble = submit->scribble;
	int non_zero_srcs, i;

	BUG_ON(faila == failb);
	if (failb < faila)
		swap(faila, failb);

	pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);

	/* if a dma resource is not available or a scribble buffer is not
	 * available punt to the synchronous path.  In the 'dma not
	 * available' case be sure to use the scribble buffer to
	 * preserve the content of 'blocks' as the caller intended.
	 */
	if (!async_dma_find_channel(DMA_PQ) || !scribble) {
		void **ptrs = scribble ? scribble : (void **) blocks;

		async_tx_quiesce(&submit->depend_tx);
		for (i = 0; i < disks; i++)
			if (blocks[i] == NULL)
				ptrs[i] = (void *) raid6_empty_zero_page;
			else
				ptrs[i] = page_address(blocks[i]);

		raid6_2data_recov(disks, bytes, faila, failb, ptrs);

		async_tx_sync_epilog(submit);

		return NULL;
	}

	non_zero_srcs = 0;
	for (i = 0; i < disks-2 && non_zero_srcs < 4; i++)
		if (blocks[i])
			non_zero_srcs++;
	switch (non_zero_srcs) {
	case 0:
	case 1:
		/* There must be at least 2 sources - the failed devices. */
		BUG();

	case 2:
		/* dma devices do not uniformly understand a zero source pq
		 * operation (in contrast to the synchronous case), so
		 * explicitly handle the special case of a 4 disk array with
		 * both data disks missing.
		 */
		return __2data_recov_4(disks, bytes, faila, failb, blocks, submit);
	case 3:
		/* dma devices do not uniformly understand a single
		 * source pq operation (in contrast to the synchronous
		 * case), so explicitly handle the special case of a 5 disk
		 * array with 2 of 3 data disks missing.
		 */
		return __2data_recov_5(disks, bytes, faila, failb, blocks, submit);
	default:
		return __2data_recov_n(disks, bytes, faila, failb, blocks, submit);
	}
}
EXPORT_SYMBOL_GPL(async_raid6_2data_recov);

/**
 * async_raid6_datap_recov - asynchronously calculate a data and the 'p' block
 * @disks: number of disks in the RAID-6 array
 * @bytes: block size
 * @faila: failed drive index
 * @blocks: array of source pointers where the last two entries are p and q
 * @submit: submission/completion modifiers
 */
struct dma_async_tx_descriptor *
async_raid6_datap_recov(int disks, size_t bytes, int faila,
			struct page **blocks, struct async_submit_ctl *submit)
{
	struct dma_async_tx_descriptor *tx = NULL;
	struct page *p, *q, *dq;
	u8 coef;
	enum async_tx_flags flags = submit->flags;
	dma_async_tx_callback cb_fn = submit->cb_fn;
	void *cb_param = submit->cb_param;
	void *scribble = submit->scribble;
	int good_srcs, good, i;
	struct page *srcs[2];

	pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);

	/* if a dma resource is not available or a scribble buffer is not
	 * available punt to the synchronous path.  In the 'dma not
	 * available' case be sure to use the scribble buffer to
	 * preserve the content of 'blocks' as the caller intended.
	 */
	if (!async_dma_find_channel(DMA_PQ) || !scribble) {
		void **ptrs = scribble ? scribble : (void **) blocks;

		async_tx_quiesce(&submit->depend_tx);
		for (i = 0; i < disks; i++)
			if (blocks[i] == NULL)
				ptrs[i] = (void*)raid6_empty_zero_page;
			else
				ptrs[i] = page_address(blocks[i]);

		raid6_datap_recov(disks, bytes, faila, ptrs);

		async_tx_sync_epilog(submit);

		return NULL;
	}

	good_srcs = 0;
	good = -1;
	for (i = 0; i < disks-2; i++) {
		if (i == faila)
			continue;
		if (blocks[i]) {
			good = i;
			good_srcs++;
			if (good_srcs > 1)
				break;
		}
	}
	BUG_ON(good_srcs == 0);

	p = blocks[disks-2];
	q = blocks[disks-1];

	/* Compute syndrome with zero for the missing data page
	 * Use the dead data page as temporary storage for delta q
	 */
	dq = blocks[faila];
	blocks[faila] = NULL;
	blocks[disks-1] = dq;

	/* in the 4-disk case we only need to perform a single source
	 * multiplication with the one good data block.
	 */
	if (good_srcs == 1) {
		struct page *g = blocks[good];

		init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
				  scribble);
		tx = async_memcpy(p, g, 0, 0, bytes, submit);

		init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
				  scribble);
		tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit);
	} else {
		init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
				  scribble);
		tx = async_gen_syndrome(blocks, 0, disks, bytes, submit);
	}

	/* Restore pointer table */
	blocks[faila]   = dq;
	blocks[disks-1] = q;

	/* calculate g^{-faila} */
	coef = raid6_gfinv[raid6_gfexp[faila]];

	srcs[0] = dq;
	srcs[1] = q;
	init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
			  NULL, NULL, scribble);
	tx = async_xor(dq, srcs, 0, 2, bytes, submit);

	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
	tx = async_mult(dq, dq, coef, bytes, submit);

	srcs[0] = p;
	srcs[1] = dq;
	init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
			  cb_param, scribble);
	tx = async_xor(p, srcs, 0, 2, bytes, submit);

	return tx;
}
EXPORT_SYMBOL_GPL(async_raid6_datap_recov);

MODULE_AUTHOR("Dan Williams <dan.j.williams@intel.com>");
MODULE_DESCRIPTION("asynchronous RAID-6 recovery api");
MODULE_LICENSE("GPL");
Commit	Line	Data
0a82a623 DW	1	/*
	2	* Asynchronous RAID-6 recovery calculations ASYNC_TX API.
	3	* Copyright(c) 2009 Intel Corporation
	4	*
	5	* based on raid6recov.c:
	6	* Copyright 2002 H. Peter Anvin
	7	*
	8	* This program is free software; you can redistribute it and/or modify it
	9	* under the terms of the GNU General Public License as published by the Free
	10	* Software Foundation; either version 2 of the License, or (at your option)
	11	* any later version.
	12	*
	13	* This program is distributed in the hope that it will be useful, but WITHOUT
	14	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	15	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	16	* more details.
	17	*
	18	* You should have received a copy of the GNU General Public License along with
	19	* this program; if not, write to the Free Software Foundation, Inc., 51
	20	* Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
	21	*
	22	*/
	23	#include <linux/kernel.h>
	24	#include <linux/interrupt.h>
	25	#include <linux/dma-mapping.h>
	26	#include <linux/raid/pq.h>
	27	#include <linux/async_tx.h>
	28
	29	static struct dma_async_tx_descriptor *
	30	async_sum_product(struct page dest, struct page srcs, unsigned char coef,
	31	size_t len, struct async_submit_ctl *submit)
	32	{
	33	struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
	34	&dest, 1, srcs, 2, len);
	35	struct dma_device *dma = chan ? chan->device : NULL;
	36	const u8 amul, bmul;
	37	u8 ax, bx;
	38	u8 a, b, *c;
	39
	40	if (dma) {
	41	dma_addr_t dma_dest[2];
	42	dma_addr_t dma_src[2];
	43	struct device *dev = dma->dev;
	44	struct dma_async_tx_descriptor *tx;
	45	enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
	46
0403e382 DW	47	if (submit->flags & ASYNC_TX_FENCE)
0403e382 DW	48	dma_flags \|= DMA_PREP_FENCE;
0a82a623 DW	49	dma_dest[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
	50	dma_src[0] = dma_map_page(dev, srcs[0], 0, len, DMA_TO_DEVICE);
	51	dma_src[1] = dma_map_page(dev, srcs[1], 0, len, DMA_TO_DEVICE);
	52	tx = dma->device_prep_dma_pq(chan, dma_dest, dma_src, 2, coef,
	53	len, dma_flags);
	54	if (tx) {
	55	async_tx_submit(chan, tx, submit);
	56	return tx;
	57	}
1f6672d4 DW	58
	59	/* could not get a descriptor, unmap and fall through to
	60	* the synchronous path
	61	*/
	62	dma_unmap_page(dev, dma_dest[1], len, DMA_BIDIRECTIONAL);
	63	dma_unmap_page(dev, dma_src[0], len, DMA_TO_DEVICE);
	64	dma_unmap_page(dev, dma_src[1], len, DMA_TO_DEVICE);
0a82a623 DW	65	}
	66
	67	/* run the operation synchronously */
	68	async_tx_quiesce(&submit->depend_tx);
	69	amul = raid6_gfmul[coef[0]];
	70	bmul = raid6_gfmul[coef[1]];
	71	a = page_address(srcs[0]);
	72	b = page_address(srcs[1]);
	73	c = page_address(dest);
	74
	75	while (len--) {
	76	ax = amul[*a++];
	77	bx = bmul[*b++];
	78	*c++ = ax ^ bx;
	79	}
	80
	81	return NULL;
	82	}
	83
	84	static struct dma_async_tx_descriptor *
	85	async_mult(struct page dest, struct page src, u8 coef, size_t len,
	86	struct async_submit_ctl *submit)
	87	{
	88	struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
	89	&dest, 1, &src, 1, len);
	90	struct dma_device *dma = chan ? chan->device : NULL;
	91	const u8 qmul; / Q multiplier table */
	92	u8 d, s;
	93
	94	if (dma) {
	95	dma_addr_t dma_dest[2];
	96	dma_addr_t dma_src[1];
	97	struct device *dev = dma->dev;
	98	struct dma_async_tx_descriptor *tx;
	99	enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
	100
0403e382 DW	101	if (submit->flags & ASYNC_TX_FENCE)
0403e382 DW	102	dma_flags \|= DMA_PREP_FENCE;
0a82a623 DW	103	dma_dest[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
	104	dma_src[0] = dma_map_page(dev, src, 0, len, DMA_TO_DEVICE);
	105	tx = dma->device_prep_dma_pq(chan, dma_dest, dma_src, 1, &coef,
	106	len, dma_flags);
	107	if (tx) {
	108	async_tx_submit(chan, tx, submit);
	109	return tx;
	110	}
1f6672d4 DW	111
	112	/* could not get a descriptor, unmap and fall through to
	113	* the synchronous path
	114	*/
	115	dma_unmap_page(dev, dma_dest[1], len, DMA_BIDIRECTIONAL);
	116	dma_unmap_page(dev, dma_src[0], len, DMA_TO_DEVICE);
0a82a623 DW	117	}
	118
	119	/* no channel available, or failed to allocate a descriptor, so
	120	* perform the operation synchronously
	121	*/
	122	async_tx_quiesce(&submit->depend_tx);
	123	qmul = raid6_gfmul[coef];
	124	d = page_address(dest);
	125	s = page_address(src);
	126
	127	while (len--)
	128	d++ = qmul[s++];
	129
	130	return NULL;
	131	}
	132
	133	static struct dma_async_tx_descriptor *
da17bf43 DW	134	__2data_recov_4(int disks, size_t bytes, int faila, int failb,
da17bf43 DW	135	struct page *blocks, struct async_submit_ctl submit)
0a82a623 DW	136	{
	137	struct dma_async_tx_descriptor *tx = NULL;
	138	struct page p, q, a, b;
	139	struct page *srcs[2];
	140	unsigned char coef[2];
	141	enum async_tx_flags flags = submit->flags;
	142	dma_async_tx_callback cb_fn = submit->cb_fn;
	143	void *cb_param = submit->cb_param;
	144	void *scribble = submit->scribble;
	145
da17bf43 DW	146	p = blocks[disks-2];
da17bf43 DW	147	q = blocks[disks-1];
0a82a623 DW	148
	149	a = blocks[faila];
	150	b = blocks[failb];
	151
	152	/* in the 4 disk case P + Pxy == P and Q + Qxy == Q */
	153	/* Dx = A(P+Pxy) + B(Q+Qxy) */
	154	srcs[0] = p;
	155	srcs[1] = q;
	156	coef[0] = raid6_gfexi[failb-faila];
	157	coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
0403e382	158	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
0a82a623 DW	159	tx = async_sum_product(b, srcs, coef, bytes, submit);
	160
	161	/* Dy = P+Pxy+Dx */
	162	srcs[0] = p;
	163	srcs[1] = b;
	164	init_async_submit(submit, flags \| ASYNC_TX_XOR_ZERO_DST, tx, cb_fn,
	165	cb_param, scribble);
	166	tx = async_xor(a, srcs, 0, 2, bytes, submit);
	167
	168	return tx;
	169
	170	}
	171
	172	static struct dma_async_tx_descriptor *
da17bf43 DW	173	__2data_recov_5(int disks, size_t bytes, int faila, int failb,
da17bf43 DW	174	struct page *blocks, struct async_submit_ctl submit)
0a82a623 DW	175	{
	176	struct dma_async_tx_descriptor *tx = NULL;
	177	struct page p, q, g, dp, *dq;
	178	struct page *srcs[2];
	179	unsigned char coef[2];
	180	enum async_tx_flags flags = submit->flags;
	181	dma_async_tx_callback cb_fn = submit->cb_fn;
	182	void *cb_param = submit->cb_param;
	183	void *scribble = submit->scribble;
da17bf43	184	int good_srcs, good, i;
0a82a623	185
da17bf43 DW	186	good_srcs = 0;
	187	good = -1;
	188	for (i = 0; i < disks-2; i++) {
	189	if (blocks[i] == NULL)
	190	continue;
0a82a623 DW	191	if (i == faila \|\| i == failb)
0a82a623 DW	192	continue;
da17bf43 DW	193	good = i;
da17bf43 DW	194	good_srcs++;
0a82a623	195	}
da17bf43	196	BUG_ON(good_srcs > 1);
0a82a623	197
da17bf43 DW	198	p = blocks[disks-2];
da17bf43 DW	199	q = blocks[disks-1];
0a82a623 DW	200	g = blocks[good];
	201
	202	/* Compute syndrome with zero for the missing data pages
	203	* Use the dead data pages as temporary storage for delta p and
	204	* delta q
	205	*/
	206	dp = blocks[faila];
	207	dq = blocks[failb];
	208
0403e382	209	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
0a82a623	210	tx = async_memcpy(dp, g, 0, 0, bytes, submit);
0403e382	211	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
0a82a623 DW	212	tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit);
	213
	214	/* compute P + Pxy */
	215	srcs[0] = dp;
	216	srcs[1] = p;
0403e382 DW	217	init_async_submit(submit, ASYNC_TX_FENCE\|ASYNC_TX_XOR_DROP_DST, tx,
0403e382 DW	218	NULL, NULL, scribble);
0a82a623 DW	219	tx = async_xor(dp, srcs, 0, 2, bytes, submit);
	220
	221	/* compute Q + Qxy */
	222	srcs[0] = dq;
	223	srcs[1] = q;
0403e382 DW	224	init_async_submit(submit, ASYNC_TX_FENCE\|ASYNC_TX_XOR_DROP_DST, tx,
0403e382 DW	225	NULL, NULL, scribble);
0a82a623 DW	226	tx = async_xor(dq, srcs, 0, 2, bytes, submit);
	227
	228	/* Dx = A(P+Pxy) + B(Q+Qxy) */
	229	srcs[0] = dp;
	230	srcs[1] = dq;
	231	coef[0] = raid6_gfexi[failb-faila];
	232	coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
0403e382	233	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
0a82a623 DW	234	tx = async_sum_product(dq, srcs, coef, bytes, submit);
	235
	236	/* Dy = P+Pxy+Dx */
	237	srcs[0] = dp;
	238	srcs[1] = dq;
	239	init_async_submit(submit, flags \| ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
	240	cb_param, scribble);
	241	tx = async_xor(dp, srcs, 0, 2, bytes, submit);
	242
	243	return tx;
	244	}
	245
	246	static struct dma_async_tx_descriptor *
	247	__2data_recov_n(int disks, size_t bytes, int faila, int failb,
	248	struct page *blocks, struct async_submit_ctl submit)
	249	{
	250	struct dma_async_tx_descriptor *tx = NULL;
	251	struct page p, q, dp, dq;
	252	struct page *srcs[2];
	253	unsigned char coef[2];
	254	enum async_tx_flags flags = submit->flags;
	255	dma_async_tx_callback cb_fn = submit->cb_fn;
	256	void *cb_param = submit->cb_param;
	257	void *scribble = submit->scribble;
	258
	259	p = blocks[disks-2];
	260	q = blocks[disks-1];
	261
	262	/* Compute syndrome with zero for the missing data pages
	263	* Use the dead data pages as temporary storage for
	264	* delta p and delta q
	265	*/
	266	dp = blocks[faila];
5dd33c9a	267	blocks[faila] = NULL;
0a82a623 DW	268	blocks[disks-2] = dp;
0a82a623 DW	269	dq = blocks[failb];
5dd33c9a	270	blocks[failb] = NULL;
0a82a623 DW	271	blocks[disks-1] = dq;
0a82a623 DW	272
0403e382	273	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
0a82a623 DW	274	tx = async_gen_syndrome(blocks, 0, disks, bytes, submit);
	275
	276	/* Restore pointer table */
	277	blocks[faila] = dp;
	278	blocks[failb] = dq;
	279	blocks[disks-2] = p;
	280	blocks[disks-1] = q;
	281
	282	/* compute P + Pxy */
	283	srcs[0] = dp;
	284	srcs[1] = p;
0403e382 DW	285	init_async_submit(submit, ASYNC_TX_FENCE\|ASYNC_TX_XOR_DROP_DST, tx,
0403e382 DW	286	NULL, NULL, scribble);
0a82a623 DW	287	tx = async_xor(dp, srcs, 0, 2, bytes, submit);
	288
	289	/* compute Q + Qxy */
	290	srcs[0] = dq;
	291	srcs[1] = q;
0403e382 DW	292	init_async_submit(submit, ASYNC_TX_FENCE\|ASYNC_TX_XOR_DROP_DST, tx,
0403e382 DW	293	NULL, NULL, scribble);
0a82a623 DW	294	tx = async_xor(dq, srcs, 0, 2, bytes, submit);
	295
	296	/* Dx = A(P+Pxy) + B(Q+Qxy) */
	297	srcs[0] = dp;
	298	srcs[1] = dq;
	299	coef[0] = raid6_gfexi[failb-faila];
	300	coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
0403e382	301	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
0a82a623 DW	302	tx = async_sum_product(dq, srcs, coef, bytes, submit);
	303
	304	/* Dy = P+Pxy+Dx */
	305	srcs[0] = dp;
	306	srcs[1] = dq;
	307	init_async_submit(submit, flags \| ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
	308	cb_param, scribble);
	309	tx = async_xor(dp, srcs, 0, 2, bytes, submit);
	310
	311	return tx;
	312	}
	313
	314	/**
	315	* async_raid6_2data_recov - asynchronously calculate two missing data blocks
	316	* @disks: number of disks in the RAID-6 array
	317	* @bytes: block size
	318	* @faila: first failed drive index
	319	* @failb: second failed drive index
	320	* @blocks: array of source pointers where the last two entries are p and q
	321	* @submit: submission/completion modifiers
	322	*/
	323	struct dma_async_tx_descriptor *
	324	async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb,
	325	struct page *blocks, struct async_submit_ctl submit)
	326	{
5157b4aa	327	void *scribble = submit->scribble;
da17bf43 DW	328	int non_zero_srcs, i;
da17bf43 DW	329
0a82a623 DW	330	BUG_ON(faila == failb);
	331	if (failb < faila)
	332	swap(faila, failb);
	333
	334	pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
	335
5157b4aa DW	336	/* if a dma resource is not available or a scribble buffer is not
	337	* available punt to the synchronous path. In the 'dma not
	338	* available' case be sure to use the scribble buffer to
	339	* preserve the content of 'blocks' as the caller intended.
0a82a623	340	*/
5157b4aa DW	341	if (!async_dma_find_channel(DMA_PQ) \|\| !scribble) {
5157b4aa DW	342	void ptrs = scribble ? scribble : (void ) blocks;
0a82a623 DW	343
	344	async_tx_quiesce(&submit->depend_tx);
	345	for (i = 0; i < disks; i++)
5dd33c9a	346	if (blocks[i] == NULL)
da17bf43	347	ptrs[i] = (void *) raid6_empty_zero_page;
5dd33c9a N	348	else
5dd33c9a N	349	ptrs[i] = page_address(blocks[i]);
0a82a623 DW	350
	351	raid6_2data_recov(disks, bytes, faila, failb, ptrs);
	352
	353	async_tx_sync_epilog(submit);
	354
	355	return NULL;
	356	}
	357
da17bf43 DW	358	non_zero_srcs = 0;
	359	for (i = 0; i < disks-2 && non_zero_srcs < 4; i++)
	360	if (blocks[i])
	361	non_zero_srcs++;
	362	switch (non_zero_srcs) {
	363	case 0:
	364	case 1:
	365	/* There must be at least 2 sources - the failed devices. */
	366	BUG();
	367
	368	case 2:
0a82a623 DW	369	/* dma devices do not uniformly understand a zero source pq
0a82a623 DW	370	* operation (in contrast to the synchronous case), so
da17bf43 DW	371	* explicitly handle the special case of a 4 disk array with
da17bf43 DW	372	* both data disks missing.
0a82a623	373	*/
da17bf43 DW	374	return __2data_recov_4(disks, bytes, faila, failb, blocks, submit);
da17bf43 DW	375	case 3:
0a82a623 DW	376	/* dma devices do not uniformly understand a single
0a82a623 DW	377	* source pq operation (in contrast to the synchronous
da17bf43 DW	378	* case), so explicitly handle the special case of a 5 disk
da17bf43 DW	379	* array with 2 of 3 data disks missing.
0a82a623	380	*/
da17bf43	381	return __2data_recov_5(disks, bytes, faila, failb, blocks, submit);
0a82a623 DW	382	default:
	383	return __2data_recov_n(disks, bytes, faila, failb, blocks, submit);
	384	}
	385	}
	386	EXPORT_SYMBOL_GPL(async_raid6_2data_recov);
	387
	388	/**
	389	* async_raid6_datap_recov - asynchronously calculate a data and the 'p' block
	390	* @disks: number of disks in the RAID-6 array
	391	* @bytes: block size
	392	* @faila: failed drive index
	393	* @blocks: array of source pointers where the last two entries are p and q
	394	* @submit: submission/completion modifiers
	395	*/
	396	struct dma_async_tx_descriptor *
	397	async_raid6_datap_recov(int disks, size_t bytes, int faila,
	398	struct page *blocks, struct async_submit_ctl submit)
	399	{
	400	struct dma_async_tx_descriptor *tx = NULL;
	401	struct page p, q, *dq;
	402	u8 coef;
	403	enum async_tx_flags flags = submit->flags;
	404	dma_async_tx_callback cb_fn = submit->cb_fn;
	405	void *cb_param = submit->cb_param;
	406	void *scribble = submit->scribble;
da17bf43	407	int good_srcs, good, i;
0a82a623 DW	408	struct page *srcs[2];
	409
	410	pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
	411
5157b4aa DW	412	/* if a dma resource is not available or a scribble buffer is not
	413	* available punt to the synchronous path. In the 'dma not
	414	* available' case be sure to use the scribble buffer to
	415	* preserve the content of 'blocks' as the caller intended.
0a82a623	416	*/
5157b4aa DW	417	if (!async_dma_find_channel(DMA_PQ) \|\| !scribble) {
5157b4aa DW	418	void ptrs = scribble ? scribble : (void ) blocks;
0a82a623 DW	419
	420	async_tx_quiesce(&submit->depend_tx);
	421	for (i = 0; i < disks; i++)
5dd33c9a N	422	if (blocks[i] == NULL)
	423	ptrs[i] = (void*)raid6_empty_zero_page;
	424	else
	425	ptrs[i] = page_address(blocks[i]);
0a82a623 DW	426
	427	raid6_datap_recov(disks, bytes, faila, ptrs);
	428
	429	async_tx_sync_epilog(submit);
	430
	431	return NULL;
	432	}
	433
da17bf43 DW	434	good_srcs = 0;
	435	good = -1;
	436	for (i = 0; i < disks-2; i++) {
	437	if (i == faila)
	438	continue;
	439	if (blocks[i]) {
	440	good = i;
	441	good_srcs++;
	442	if (good_srcs > 1)
	443	break;
	444	}
	445	}
	446	BUG_ON(good_srcs == 0);
	447
0a82a623 DW	448	p = blocks[disks-2];
	449	q = blocks[disks-1];
	450
	451	/* Compute syndrome with zero for the missing data page
	452	* Use the dead data page as temporary storage for delta q
	453	*/
	454	dq = blocks[faila];
5dd33c9a	455	blocks[faila] = NULL;
0a82a623 DW	456	blocks[disks-1] = dq;
0a82a623 DW	457
da17bf43 DW	458	/* in the 4-disk case we only need to perform a single source
da17bf43 DW	459	* multiplication with the one good data block.
0a82a623	460	*/
da17bf43	461	if (good_srcs == 1) {
0a82a623 DW	462	struct page *g = blocks[good];
0a82a623 DW	463
0403e382 DW	464	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
0403e382 DW	465	scribble);
0a82a623 DW	466	tx = async_memcpy(p, g, 0, 0, bytes, submit);
0a82a623 DW	467
0403e382 DW	468	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
0403e382 DW	469	scribble);
0a82a623 DW	470	tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit);
0a82a623 DW	471	} else {
0403e382 DW	472	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
0403e382 DW	473	scribble);
0a82a623 DW	474	tx = async_gen_syndrome(blocks, 0, disks, bytes, submit);
	475	}
	476
	477	/* Restore pointer table */
	478	blocks[faila] = dq;
	479	blocks[disks-1] = q;
	480
	481	/* calculate g^{-faila} */
	482	coef = raid6_gfinv[raid6_gfexp[faila]];
	483
	484	srcs[0] = dq;
	485	srcs[1] = q;
0403e382 DW	486	init_async_submit(submit, ASYNC_TX_FENCE\|ASYNC_TX_XOR_DROP_DST, tx,
0403e382 DW	487	NULL, NULL, scribble);
0a82a623 DW	488	tx = async_xor(dq, srcs, 0, 2, bytes, submit);
0a82a623 DW	489
0403e382	490	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
0a82a623 DW	491	tx = async_mult(dq, dq, coef, bytes, submit);
	492
	493	srcs[0] = p;
	494	srcs[1] = dq;
	495	init_async_submit(submit, flags \| ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
	496	cb_param, scribble);
	497	tx = async_xor(p, srcs, 0, 2, bytes, submit);
	498
	499	return tx;
	500	}
	501	EXPORT_SYMBOL_GPL(async_raid6_datap_recov);
	502
	503	MODULE_AUTHOR("Dan Williams <dan.j.williams@intel.com>");
	504	MODULE_DESCRIPTION("asynchronous RAID-6 recovery api");
	505	MODULE_LICENSE("GPL");