[net-next-2.6.git] / arch / mips / kernel / unaligned.c

/*
 * Handle unaligned accesses by emulation.
 *
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Copyright (C) 1996, 1998, 1999, 2002 by Ralf Baechle
 * Copyright (C) 1999 Silicon Graphics, Inc.
 *
 * This file contains exception handler for address error exception with the
 * special capability to execute faulting instructions in software.  The
 * handler does not try to handle the case when the program counter points
 * to an address not aligned to a word boundary.
 *
 * Putting data to unaligned addresses is a bad practice even on Intel where
 * only the performance is affected.  Much worse is that such code is non-
 * portable.  Due to several programs that die on MIPS due to alignment
 * problems I decided to implement this handler anyway though I originally
 * didn't intend to do this at all for user code.
 *
 * For now I enable fixing of address errors by default to make life easier.
 * I however intend to disable this somewhen in the future when the alignment
 * problems with user programs have been fixed.  For programmers this is the
 * right way to go.
 *
 * Fixing address errors is a per process option.  The option is inherited
 * across fork(2) and execve(2) calls.  If you really want to use the
 * option in your user programs - I discourage the use of the software
 * emulation strongly - use the following code in your userland stuff:
 *
 * #include <sys/sysmips.h>
 *
 * ...
 * sysmips(MIPS_FIXADE, x);
 * ...
 *
 * The argument x is 0 for disabling software emulation, enabled otherwise.
 *
 * Below a little program to play around with this feature.
 *
 * #include <stdio.h>
 * #include <sys/sysmips.h>
 *
 * struct foo {
 *         unsigned char bar[8];
 * };
 *
 * main(int argc, char *argv[])
 * {
 *         struct foo x = {0, 1, 2, 3, 4, 5, 6, 7};
 *         unsigned int *p = (unsigned int *) (x.bar + 3);
 *         int i;
 *
 *         if (argc > 1)
 *                 sysmips(MIPS_FIXADE, atoi(argv[1]));
 *
 *         printf("*p = %08lx\n", *p);
 *
 *         *p = 0xdeadface;
 *
 *         for(i = 0; i <= 7; i++)
 *         printf("%02x ", x.bar[i]);
 *         printf("\n");
 * }
 *
 * Coprocessor loads are not supported; I think this case is unimportant
 * in the practice.
 *
 * TODO: Handle ndc (attempted store to doubleword in uncached memory)
 *       exception for the R6000.
 *       A store crossing a page boundary might be executed only partially.
 *       Undo the partial store in this case.
 */
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/signal.h>
#include <linux/smp.h>
#include <linux/sched.h>
#include <linux/debugfs.h>
#include <asm/asm.h>
#include <asm/branch.h>
#include <asm/byteorder.h>
#include <asm/cop2.h>
#include <asm/inst.h>
#include <asm/uaccess.h>
#include <asm/system.h>

#define STR(x)  __STR(x)
#define __STR(x)  #x

enum {
	UNALIGNED_ACTION_QUIET,
	UNALIGNED_ACTION_SIGNAL,
	UNALIGNED_ACTION_SHOW,
};
#ifdef CONFIG_DEBUG_FS
static u32 unaligned_instructions;
static u32 unaligned_action;
#else
#define unaligned_action UNALIGNED_ACTION_QUIET
#endif
extern void show_registers(struct pt_regs *regs);

static void emulate_load_store_insn(struct pt_regs *regs,
	void __user *addr, unsigned int __user *pc)
{
	union mips_instruction insn;
	unsigned long value;
	unsigned int res;

	/*
	 * This load never faults.
	 */
	__get_user(insn.word, pc);

	switch (insn.i_format.opcode) {
	/*
	 * These are instructions that a compiler doesn't generate.  We
	 * can assume therefore that the code is MIPS-aware and
	 * really buggy.  Emulating these instructions would break the
	 * semantics anyway.
	 */
	case ll_op:
	case lld_op:
	case sc_op:
	case scd_op:

	/*
	 * For these instructions the only way to create an address
	 * error is an attempted access to kernel/supervisor address
	 * space.
	 */
	case ldl_op:
	case ldr_op:
	case lwl_op:
	case lwr_op:
	case sdl_op:
	case sdr_op:
	case swl_op:
	case swr_op:
	case lb_op:
	case lbu_op:
	case sb_op:
		goto sigbus;

	/*
	 * The remaining opcodes are the ones that are really of interest.
	 */
	case lh_op:
		if (!access_ok(VERIFY_READ, addr, 2))
			goto sigbus;

		__asm__ __volatile__ (".set\tnoat\n"
#ifdef __BIG_ENDIAN
			"1:\tlb\t%0, 0(%2)\n"
			"2:\tlbu\t$1, 1(%2)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
			"1:\tlb\t%0, 1(%2)\n"
			"2:\tlbu\t$1, 0(%2)\n\t"
#endif
			"sll\t%0, 0x8\n\t"
			"or\t%0, $1\n\t"
			"li\t%1, 0\n"
			"3:\t.set\tat\n\t"
			".section\t.fixup,\"ax\"\n\t"
			"4:\tli\t%1, %3\n\t"
			"j\t3b\n\t"
			".previous\n\t"
			".section\t__ex_table,\"a\"\n\t"
			STR(PTR)"\t1b, 4b\n\t"
			STR(PTR)"\t2b, 4b\n\t"
			".previous"
			: "=&r" (value), "=r" (res)
			: "r" (addr), "i" (-EFAULT));
		if (res)
			goto fault;
		compute_return_epc(regs);
		regs->regs[insn.i_format.rt] = value;
		break;

	case lw_op:
		if (!access_ok(VERIFY_READ, addr, 4))
			goto sigbus;

		__asm__ __volatile__ (
#ifdef __BIG_ENDIAN
			"1:\tlwl\t%0, (%2)\n"
			"2:\tlwr\t%0, 3(%2)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
			"1:\tlwl\t%0, 3(%2)\n"
			"2:\tlwr\t%0, (%2)\n\t"
#endif
			"li\t%1, 0\n"
			"3:\t.section\t.fixup,\"ax\"\n\t"
			"4:\tli\t%1, %3\n\t"
			"j\t3b\n\t"
			".previous\n\t"
			".section\t__ex_table,\"a\"\n\t"
			STR(PTR)"\t1b, 4b\n\t"
			STR(PTR)"\t2b, 4b\n\t"
			".previous"
			: "=&r" (value), "=r" (res)
			: "r" (addr), "i" (-EFAULT));
		if (res)
			goto fault;
		compute_return_epc(regs);
		regs->regs[insn.i_format.rt] = value;
		break;

	case lhu_op:
		if (!access_ok(VERIFY_READ, addr, 2))
			goto sigbus;

		__asm__ __volatile__ (
			".set\tnoat\n"
#ifdef __BIG_ENDIAN
			"1:\tlbu\t%0, 0(%2)\n"
			"2:\tlbu\t$1, 1(%2)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
			"1:\tlbu\t%0, 1(%2)\n"
			"2:\tlbu\t$1, 0(%2)\n\t"
#endif
			"sll\t%0, 0x8\n\t"
			"or\t%0, $1\n\t"
			"li\t%1, 0\n"
			"3:\t.set\tat\n\t"
			".section\t.fixup,\"ax\"\n\t"
			"4:\tli\t%1, %3\n\t"
			"j\t3b\n\t"
			".previous\n\t"
			".section\t__ex_table,\"a\"\n\t"
			STR(PTR)"\t1b, 4b\n\t"
			STR(PTR)"\t2b, 4b\n\t"
			".previous"
			: "=&r" (value), "=r" (res)
			: "r" (addr), "i" (-EFAULT));
		if (res)
			goto fault;
		compute_return_epc(regs);
		regs->regs[insn.i_format.rt] = value;
		break;

	case lwu_op:
#ifdef CONFIG_64BIT
		/*
		 * A 32-bit kernel might be running on a 64-bit processor.  But
		 * if we're on a 32-bit processor and an i-cache incoherency
		 * or race makes us see a 64-bit instruction here the sdl/sdr
		 * would blow up, so for now we don't handle unaligned 64-bit
		 * instructions on 32-bit kernels.
		 */
		if (!access_ok(VERIFY_READ, addr, 4))
			goto sigbus;

		__asm__ __volatile__ (
#ifdef __BIG_ENDIAN
			"1:\tlwl\t%0, (%2)\n"
			"2:\tlwr\t%0, 3(%2)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
			"1:\tlwl\t%0, 3(%2)\n"
			"2:\tlwr\t%0, (%2)\n\t"
#endif
			"dsll\t%0, %0, 32\n\t"
			"dsrl\t%0, %0, 32\n\t"
			"li\t%1, 0\n"
			"3:\t.section\t.fixup,\"ax\"\n\t"
			"4:\tli\t%1, %3\n\t"
			"j\t3b\n\t"
			".previous\n\t"
			".section\t__ex_table,\"a\"\n\t"
			STR(PTR)"\t1b, 4b\n\t"
			STR(PTR)"\t2b, 4b\n\t"
			".previous"
			: "=&r" (value), "=r" (res)
			: "r" (addr), "i" (-EFAULT));
		if (res)
			goto fault;
		compute_return_epc(regs);
		regs->regs[insn.i_format.rt] = value;
		break;
#endif /* CONFIG_64BIT */

		/* Cannot handle 64-bit instructions in 32-bit kernel */
		goto sigill;

	case ld_op:
#ifdef CONFIG_64BIT
		/*
		 * A 32-bit kernel might be running on a 64-bit processor.  But
		 * if we're on a 32-bit processor and an i-cache incoherency
		 * or race makes us see a 64-bit instruction here the sdl/sdr
		 * would blow up, so for now we don't handle unaligned 64-bit
		 * instructions on 32-bit kernels.
		 */
		if (!access_ok(VERIFY_READ, addr, 8))
			goto sigbus;

		__asm__ __volatile__ (
#ifdef __BIG_ENDIAN
			"1:\tldl\t%0, (%2)\n"
			"2:\tldr\t%0, 7(%2)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
			"1:\tldl\t%0, 7(%2)\n"
			"2:\tldr\t%0, (%2)\n\t"
#endif
			"li\t%1, 0\n"
			"3:\t.section\t.fixup,\"ax\"\n\t"
			"4:\tli\t%1, %3\n\t"
			"j\t3b\n\t"
			".previous\n\t"
			".section\t__ex_table,\"a\"\n\t"
			STR(PTR)"\t1b, 4b\n\t"
			STR(PTR)"\t2b, 4b\n\t"
			".previous"
			: "=&r" (value), "=r" (res)
			: "r" (addr), "i" (-EFAULT));
		if (res)
			goto fault;
		compute_return_epc(regs);
		regs->regs[insn.i_format.rt] = value;
		break;
#endif /* CONFIG_64BIT */

		/* Cannot handle 64-bit instructions in 32-bit kernel */
		goto sigill;

	case sh_op:
		if (!access_ok(VERIFY_WRITE, addr, 2))
			goto sigbus;

		value = regs->regs[insn.i_format.rt];
		__asm__ __volatile__ (
#ifdef __BIG_ENDIAN
			".set\tnoat\n"
			"1:\tsb\t%1, 1(%2)\n\t"
			"srl\t$1, %1, 0x8\n"
			"2:\tsb\t$1, 0(%2)\n\t"
			".set\tat\n\t"
#endif
#ifdef __LITTLE_ENDIAN
			".set\tnoat\n"
			"1:\tsb\t%1, 0(%2)\n\t"
			"srl\t$1,%1, 0x8\n"
			"2:\tsb\t$1, 1(%2)\n\t"
			".set\tat\n\t"
#endif
			"li\t%0, 0\n"
			"3:\n\t"
			".section\t.fixup,\"ax\"\n\t"
			"4:\tli\t%0, %3\n\t"
			"j\t3b\n\t"
			".previous\n\t"
			".section\t__ex_table,\"a\"\n\t"
			STR(PTR)"\t1b, 4b\n\t"
			STR(PTR)"\t2b, 4b\n\t"
			".previous"
			: "=r" (res)
			: "r" (value), "r" (addr), "i" (-EFAULT));
		if (res)
			goto fault;
		compute_return_epc(regs);
		break;

	case sw_op:
		if (!access_ok(VERIFY_WRITE, addr, 4))
			goto sigbus;

		value = regs->regs[insn.i_format.rt];
		__asm__ __volatile__ (
#ifdef __BIG_ENDIAN
			"1:\tswl\t%1,(%2)\n"
			"2:\tswr\t%1, 3(%2)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
			"1:\tswl\t%1, 3(%2)\n"
			"2:\tswr\t%1, (%2)\n\t"
#endif
			"li\t%0, 0\n"
			"3:\n\t"
			".section\t.fixup,\"ax\"\n\t"
			"4:\tli\t%0, %3\n\t"
			"j\t3b\n\t"
			".previous\n\t"
			".section\t__ex_table,\"a\"\n\t"
			STR(PTR)"\t1b, 4b\n\t"
			STR(PTR)"\t2b, 4b\n\t"
			".previous"
		: "=r" (res)
		: "r" (value), "r" (addr), "i" (-EFAULT));
		if (res)
			goto fault;
		compute_return_epc(regs);
		break;

	case sd_op:
#ifdef CONFIG_64BIT
		/*
		 * A 32-bit kernel might be running on a 64-bit processor.  But
		 * if we're on a 32-bit processor and an i-cache incoherency
		 * or race makes us see a 64-bit instruction here the sdl/sdr
		 * would blow up, so for now we don't handle unaligned 64-bit
		 * instructions on 32-bit kernels.
		 */
		if (!access_ok(VERIFY_WRITE, addr, 8))
			goto sigbus;

		value = regs->regs[insn.i_format.rt];
		__asm__ __volatile__ (
#ifdef __BIG_ENDIAN
			"1:\tsdl\t%1,(%2)\n"
			"2:\tsdr\t%1, 7(%2)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
			"1:\tsdl\t%1, 7(%2)\n"
			"2:\tsdr\t%1, (%2)\n\t"
#endif
			"li\t%0, 0\n"
			"3:\n\t"
			".section\t.fixup,\"ax\"\n\t"
			"4:\tli\t%0, %3\n\t"
			"j\t3b\n\t"
			".previous\n\t"
			".section\t__ex_table,\"a\"\n\t"
			STR(PTR)"\t1b, 4b\n\t"
			STR(PTR)"\t2b, 4b\n\t"
			".previous"
		: "=r" (res)
		: "r" (value), "r" (addr), "i" (-EFAULT));
		if (res)
			goto fault;
		compute_return_epc(regs);
		break;
#endif /* CONFIG_64BIT */

		/* Cannot handle 64-bit instructions in 32-bit kernel */
		goto sigill;

	case lwc1_op:
	case ldc1_op:
	case swc1_op:
	case sdc1_op:
		/*
		 * I herewith declare: this does not happen.  So send SIGBUS.
		 */
		goto sigbus;

	/*
	 * COP2 is available to implementor for application specific use.
	 * It's up to applications to register a notifier chain and do
	 * whatever they have to do, including possible sending of signals.
	 */
	case lwc2_op:
		cu2_notifier_call_chain(CU2_LWC2_OP, regs);
		break;

	case ldc2_op:
		cu2_notifier_call_chain(CU2_LDC2_OP, regs);
		break;

	case swc2_op:
		cu2_notifier_call_chain(CU2_SWC2_OP, regs);
		break;

	case sdc2_op:
		cu2_notifier_call_chain(CU2_SDC2_OP, regs);
		break;

	default:
		/*
		 * Pheeee...  We encountered an yet unknown instruction or
		 * cache coherence problem.  Die sucker, die ...
		 */
		goto sigill;
	}

#ifdef CONFIG_DEBUG_FS
	unaligned_instructions++;
#endif

	return;

fault:
	/* Did we have an exception handler installed? */
	if (fixup_exception(regs))
		return;

	die_if_kernel("Unhandled kernel unaligned access", regs);
	force_sig(SIGSEGV, current);

	return;

sigbus:
	die_if_kernel("Unhandled kernel unaligned access", regs);
	force_sig(SIGBUS, current);

	return;

sigill:
	die_if_kernel("Unhandled kernel unaligned access or invalid instruction", regs);
	force_sig(SIGILL, current);
}

asmlinkage void do_ade(struct pt_regs *regs)
{
	unsigned int __user *pc;
	mm_segment_t seg;

	/*
	 * Did we catch a fault trying to load an instruction?
	 * Or are we running in MIPS16 mode?
	 */
	if ((regs->cp0_badvaddr == regs->cp0_epc) || (regs->cp0_epc & 0x1))
		goto sigbus;

	pc = (unsigned int __user *) exception_epc(regs);
	if (user_mode(regs) && !test_thread_flag(TIF_FIXADE))
		goto sigbus;
	if (unaligned_action == UNALIGNED_ACTION_SIGNAL)
		goto sigbus;
	else if (unaligned_action == UNALIGNED_ACTION_SHOW)
		show_registers(regs);

	/*
	 * Do branch emulation only if we didn't forward the exception.
	 * This is all so but ugly ...
	 */
	seg = get_fs();
	if (!user_mode(regs))
		set_fs(KERNEL_DS);
	emulate_load_store_insn(regs, (void __user *)regs->cp0_badvaddr, pc);
	set_fs(seg);

	return;

sigbus:
	die_if_kernel("Kernel unaligned instruction access", regs);
	force_sig(SIGBUS, current);

	/*
	 * XXX On return from the signal handler we should advance the epc
	 */
}

#ifdef CONFIG_DEBUG_FS
extern struct dentry *mips_debugfs_dir;
static int __init debugfs_unaligned(void)
{
	struct dentry *d;

	if (!mips_debugfs_dir)
		return -ENODEV;
	d = debugfs_create_u32("unaligned_instructions", S_IRUGO,
			       mips_debugfs_dir, &unaligned_instructions);
	if (!d)
		return -ENOMEM;
	d = debugfs_create_u32("unaligned_action", S_IRUGO | S_IWUSR,
			       mips_debugfs_dir, &unaligned_action);
	if (!d)
		return -ENOMEM;
	return 0;
}
__initcall(debugfs_unaligned);
#endif
Commit	Line	Data
1da177e4 LT	1	/*
	2	* Handle unaligned accesses by emulation.
	3	*
	4	* This file is subject to the terms and conditions of the GNU General Public
	5	* License. See the file "COPYING" in the main directory of this archive
	6	* for more details.
	7	*
	8	* Copyright (C) 1996, 1998, 1999, 2002 by Ralf Baechle
	9	* Copyright (C) 1999 Silicon Graphics, Inc.
	10	*
	11	* This file contains exception handler for address error exception with the
	12	* special capability to execute faulting instructions in software. The
	13	* handler does not try to handle the case when the program counter points
	14	* to an address not aligned to a word boundary.
	15	*
	16	* Putting data to unaligned addresses is a bad practice even on Intel where
	17	* only the performance is affected. Much worse is that such code is non-
	18	* portable. Due to several programs that die on MIPS due to alignment
	19	* problems I decided to implement this handler anyway though I originally
	20	* didn't intend to do this at all for user code.
	21	*
	22	* For now I enable fixing of address errors by default to make life easier.
	23	* I however intend to disable this somewhen in the future when the alignment
	24	* problems with user programs have been fixed. For programmers this is the
	25	* right way to go.
	26	*
	27	* Fixing address errors is a per process option. The option is inherited
	28	* across fork(2) and execve(2) calls. If you really want to use the
	29	* option in your user programs - I discourage the use of the software
	30	* emulation strongly - use the following code in your userland stuff:
	31	*
	32	* #include <sys/sysmips.h>
	33	*
	34	* ...
	35	* sysmips(MIPS_FIXADE, x);
	36	* ...
	37	*
	38	* The argument x is 0 for disabling software emulation, enabled otherwise.
	39	*
	40	* Below a little program to play around with this feature.
	41	*
	42	* #include <stdio.h>
	43	* #include <sys/sysmips.h>
	44	*
	45	* struct foo {
	46	* unsigned char bar[8];
	47	* };
	48	*
	49	* main(int argc, char *argv[])
	50	* {
	51	* struct foo x = {0, 1, 2, 3, 4, 5, 6, 7};
	52	* unsigned int p = (unsigned int ) (x.bar + 3);
	53	* int i;
	54	*
	55	* if (argc > 1)
	56	* sysmips(MIPS_FIXADE, atoi(argv[1]));
	57	*
	58	* printf("p = %08lx\n", p);
	59	*
	60	* *p = 0xdeadface;
	61	*
	62	* for(i = 0; i <= 7; i++)
	63	* printf("%02x ", x.bar[i]);
	64	* printf("\n");
65	* }
66	*
67	* Coprocessor loads are not supported; I think this case is unimportant
68	* in the practice.
69	*
70	* TODO: Handle ndc (attempted store to doubleword in uncached memory)
71	* exception for the R6000.
72	* A store crossing a page boundary might be executed only partially.
73	* Undo the partial store in this case.
74	*/
1da177e4 LT	75	#include <linux/mm.h>
	76	#include <linux/module.h>
	77	#include <linux/signal.h>
	78	#include <linux/smp.h>
e8edc6e0	79	#include <linux/sched.h>
6312e0ee	80	#include <linux/debugfs.h>
1da177e4 LT	81	#include <asm/asm.h>
	82	#include <asm/branch.h>
	83	#include <asm/byteorder.h>
69f3a7de	84	#include <asm/cop2.h>
1da177e4 LT	85	#include <asm/inst.h>
	86	#include <asm/uaccess.h>
	87	#include <asm/system.h>
	88
	89	#define STR(x) __STR(x)
	90	#define __STR(x) #x
	91
6312e0ee AN	92	enum {
	93	UNALIGNED_ACTION_QUIET,
	94	UNALIGNED_ACTION_SIGNAL,
	95	UNALIGNED_ACTION_SHOW,
	96	};
	97	#ifdef CONFIG_DEBUG_FS
	98	static u32 unaligned_instructions;
	99	static u32 unaligned_action;
	100	#else
	101	#define unaligned_action UNALIGNED_ACTION_QUIET
1da177e4	102	#endif
6312e0ee	103	extern void show_registers(struct pt_regs *regs);
1da177e4	104
7f18f151 RB	105	static void emulate_load_store_insn(struct pt_regs *regs,
7f18f151 RB	106	void __user addr, unsigned int __user pc)
1da177e4 LT	107	{
	108	union mips_instruction insn;
	109	unsigned long value;
	110	unsigned int res;
	111
1da177e4 LT	112	/*
	113	* This load never faults.
	114	*/
fe00f943	115	__get_user(insn.word, pc);
1da177e4 LT	116
	117	switch (insn.i_format.opcode) {
	118	/*
	119	* These are instructions that a compiler doesn't generate. We
	120	* can assume therefore that the code is MIPS-aware and
	121	* really buggy. Emulating these instructions would break the
	122	* semantics anyway.
	123	*/
	124	case ll_op:
	125	case lld_op:
	126	case sc_op:
	127	case scd_op:
	128
	129	/*
	130	* For these instructions the only way to create an address
	131	* error is an attempted access to kernel/supervisor address
	132	* space.
	133	*/
	134	case ldl_op:
	135	case ldr_op:
	136	case lwl_op:
	137	case lwr_op:
	138	case sdl_op:
	139	case sdr_op:
	140	case swl_op:
	141	case swr_op:
	142	case lb_op:
	143	case lbu_op:
	144	case sb_op:
	145	goto sigbus;
	146
	147	/*
	148	* The remaining opcodes are the ones that are really of interest.
	149	*/
	150	case lh_op:
	151	if (!access_ok(VERIFY_READ, addr, 2))
	152	goto sigbus;
	153
	154	__asm__ __volatile__ (".set\tnoat\n"
	155	#ifdef __BIG_ENDIAN
	156	"1:\tlb\t%0, 0(%2)\n"
	157	"2:\tlbu\t$1, 1(%2)\n\t"
	158	#endif
	159	#ifdef __LITTLE_ENDIAN
	160	"1:\tlb\t%0, 1(%2)\n"
	161	"2:\tlbu\t$1, 0(%2)\n\t"
	162	#endif
	163	"sll\t%0, 0x8\n\t"
	164	"or\t%0, $1\n\t"
	165	"li\t%1, 0\n"
	166	"3:\t.set\tat\n\t"
	167	".section\t.fixup,\"ax\"\n\t"
	168	"4:\tli\t%1, %3\n\t"
	169	"j\t3b\n\t"
	170	".previous\n\t"
	171	".section\t__ex_table,\"a\"\n\t"
	172	STR(PTR)"\t1b, 4b\n\t"
	173	STR(PTR)"\t2b, 4b\n\t"
	174	".previous"
	175	: "=&r" (value), "=r" (res)
	176	: "r" (addr), "i" (-EFAULT));
	177	if (res)
	178	goto fault;
7f18f151 RB	179	compute_return_epc(regs);
7f18f151 RB	180	regs->regs[insn.i_format.rt] = value;
1da177e4 LT	181	break;
	182
	183	case lw_op:
	184	if (!access_ok(VERIFY_READ, addr, 4))
	185	goto sigbus;
	186
	187	__asm__ __volatile__ (
	188	#ifdef __BIG_ENDIAN
	189	"1:\tlwl\t%0, (%2)\n"
	190	"2:\tlwr\t%0, 3(%2)\n\t"
	191	#endif
	192	#ifdef __LITTLE_ENDIAN
	193	"1:\tlwl\t%0, 3(%2)\n"
	194	"2:\tlwr\t%0, (%2)\n\t"
	195	#endif
	196	"li\t%1, 0\n"
	197	"3:\t.section\t.fixup,\"ax\"\n\t"
	198	"4:\tli\t%1, %3\n\t"
	199	"j\t3b\n\t"
	200	".previous\n\t"
	201	".section\t__ex_table,\"a\"\n\t"
	202	STR(PTR)"\t1b, 4b\n\t"
	203	STR(PTR)"\t2b, 4b\n\t"
	204	".previous"
	205	: "=&r" (value), "=r" (res)
	206	: "r" (addr), "i" (-EFAULT));
	207	if (res)
	208	goto fault;
7f18f151 RB	209	compute_return_epc(regs);
7f18f151 RB	210	regs->regs[insn.i_format.rt] = value;
1da177e4 LT	211	break;
	212
	213	case lhu_op:
	214	if (!access_ok(VERIFY_READ, addr, 2))
	215	goto sigbus;
	216
	217	__asm__ __volatile__ (
	218	".set\tnoat\n"
	219	#ifdef __BIG_ENDIAN
	220	"1:\tlbu\t%0, 0(%2)\n"
	221	"2:\tlbu\t$1, 1(%2)\n\t"
	222	#endif
	223	#ifdef __LITTLE_ENDIAN
	224	"1:\tlbu\t%0, 1(%2)\n"
	225	"2:\tlbu\t$1, 0(%2)\n\t"
	226	#endif
	227	"sll\t%0, 0x8\n\t"
	228	"or\t%0, $1\n\t"
	229	"li\t%1, 0\n"
	230	"3:\t.set\tat\n\t"
	231	".section\t.fixup,\"ax\"\n\t"
	232	"4:\tli\t%1, %3\n\t"
	233	"j\t3b\n\t"
	234	".previous\n\t"
	235	".section\t__ex_table,\"a\"\n\t"
	236	STR(PTR)"\t1b, 4b\n\t"
	237	STR(PTR)"\t2b, 4b\n\t"
	238	".previous"
	239	: "=&r" (value), "=r" (res)
	240	: "r" (addr), "i" (-EFAULT));
	241	if (res)
	242	goto fault;
7f18f151 RB	243	compute_return_epc(regs);
7f18f151 RB	244	regs->regs[insn.i_format.rt] = value;
1da177e4 LT	245	break;
	246
	247	case lwu_op:
875d43e7	248	#ifdef CONFIG_64BIT
1da177e4 LT	249	/*
	250	* A 32-bit kernel might be running on a 64-bit processor. But
	251	* if we're on a 32-bit processor and an i-cache incoherency
	252	* or race makes us see a 64-bit instruction here the sdl/sdr
	253	* would blow up, so for now we don't handle unaligned 64-bit
	254	* instructions on 32-bit kernels.
	255	*/
	256	if (!access_ok(VERIFY_READ, addr, 4))
	257	goto sigbus;
	258
	259	__asm__ __volatile__ (
	260	#ifdef __BIG_ENDIAN
	261	"1:\tlwl\t%0, (%2)\n"
	262	"2:\tlwr\t%0, 3(%2)\n\t"
	263	#endif
	264	#ifdef __LITTLE_ENDIAN
	265	"1:\tlwl\t%0, 3(%2)\n"
	266	"2:\tlwr\t%0, (%2)\n\t"
	267	#endif
	268	"dsll\t%0, %0, 32\n\t"
	269	"dsrl\t%0, %0, 32\n\t"
	270	"li\t%1, 0\n"
	271	"3:\t.section\t.fixup,\"ax\"\n\t"
	272	"4:\tli\t%1, %3\n\t"
	273	"j\t3b\n\t"
	274	".previous\n\t"
	275	".section\t__ex_table,\"a\"\n\t"
	276	STR(PTR)"\t1b, 4b\n\t"
	277	STR(PTR)"\t2b, 4b\n\t"
	278	".previous"
	279	: "=&r" (value), "=r" (res)
	280	: "r" (addr), "i" (-EFAULT));
	281	if (res)
	282	goto fault;
7f18f151 RB	283	compute_return_epc(regs);
7f18f151 RB	284	regs->regs[insn.i_format.rt] = value;
1da177e4	285	break;
875d43e7	286	#endif /* CONFIG_64BIT */
1da177e4 LT	287
	288	/* Cannot handle 64-bit instructions in 32-bit kernel */
	289	goto sigill;
	290
	291	case ld_op:
875d43e7	292	#ifdef CONFIG_64BIT
1da177e4 LT	293	/*
	294	* A 32-bit kernel might be running on a 64-bit processor. But
	295	* if we're on a 32-bit processor and an i-cache incoherency
	296	* or race makes us see a 64-bit instruction here the sdl/sdr
	297	* would blow up, so for now we don't handle unaligned 64-bit
	298	* instructions on 32-bit kernels.
	299	*/
	300	if (!access_ok(VERIFY_READ, addr, 8))
	301	goto sigbus;
	302
	303	__asm__ __volatile__ (
	304	#ifdef __BIG_ENDIAN
	305	"1:\tldl\t%0, (%2)\n"
	306	"2:\tldr\t%0, 7(%2)\n\t"
	307	#endif
	308	#ifdef __LITTLE_ENDIAN
	309	"1:\tldl\t%0, 7(%2)\n"
	310	"2:\tldr\t%0, (%2)\n\t"
	311	#endif
	312	"li\t%1, 0\n"
	313	"3:\t.section\t.fixup,\"ax\"\n\t"
	314	"4:\tli\t%1, %3\n\t"
	315	"j\t3b\n\t"
	316	".previous\n\t"
	317	".section\t__ex_table,\"a\"\n\t"
	318	STR(PTR)"\t1b, 4b\n\t"
	319	STR(PTR)"\t2b, 4b\n\t"
	320	".previous"
	321	: "=&r" (value), "=r" (res)
	322	: "r" (addr), "i" (-EFAULT));
	323	if (res)
	324	goto fault;
7f18f151 RB	325	compute_return_epc(regs);
7f18f151 RB	326	regs->regs[insn.i_format.rt] = value;
1da177e4	327	break;
875d43e7	328	#endif /* CONFIG_64BIT */
1da177e4 LT	329
	330	/* Cannot handle 64-bit instructions in 32-bit kernel */
	331	goto sigill;
	332
	333	case sh_op:
	334	if (!access_ok(VERIFY_WRITE, addr, 2))
	335	goto sigbus;
	336
	337	value = regs->regs[insn.i_format.rt];
	338	__asm__ __volatile__ (
	339	#ifdef __BIG_ENDIAN
	340	".set\tnoat\n"
	341	"1:\tsb\t%1, 1(%2)\n\t"
	342	"srl\t$1, %1, 0x8\n"
	343	"2:\tsb\t$1, 0(%2)\n\t"
	344	".set\tat\n\t"
	345	#endif
	346	#ifdef __LITTLE_ENDIAN
	347	".set\tnoat\n"
	348	"1:\tsb\t%1, 0(%2)\n\t"
	349	"srl\t$1,%1, 0x8\n"
	350	"2:\tsb\t$1, 1(%2)\n\t"
	351	".set\tat\n\t"
	352	#endif
	353	"li\t%0, 0\n"
	354	"3:\n\t"
	355	".section\t.fixup,\"ax\"\n\t"
	356	"4:\tli\t%0, %3\n\t"
	357	"j\t3b\n\t"
	358	".previous\n\t"
	359	".section\t__ex_table,\"a\"\n\t"
	360	STR(PTR)"\t1b, 4b\n\t"
	361	STR(PTR)"\t2b, 4b\n\t"
	362	".previous"
	363	: "=r" (res)
	364	: "r" (value), "r" (addr), "i" (-EFAULT));
	365	if (res)
	366	goto fault;
7f18f151	367	compute_return_epc(regs);
1da177e4 LT	368	break;
	369
	370	case sw_op:
	371	if (!access_ok(VERIFY_WRITE, addr, 4))
	372	goto sigbus;
	373
	374	value = regs->regs[insn.i_format.rt];
	375	__asm__ __volatile__ (
	376	#ifdef __BIG_ENDIAN
	377	"1:\tswl\t%1,(%2)\n"
	378	"2:\tswr\t%1, 3(%2)\n\t"
	379	#endif
	380	#ifdef __LITTLE_ENDIAN
	381	"1:\tswl\t%1, 3(%2)\n"
	382	"2:\tswr\t%1, (%2)\n\t"
	383	#endif
	384	"li\t%0, 0\n"
	385	"3:\n\t"
	386	".section\t.fixup,\"ax\"\n\t"
	387	"4:\tli\t%0, %3\n\t"
	388	"j\t3b\n\t"
	389	".previous\n\t"
	390	".section\t__ex_table,\"a\"\n\t"
	391	STR(PTR)"\t1b, 4b\n\t"
	392	STR(PTR)"\t2b, 4b\n\t"
	393	".previous"
	394	: "=r" (res)
	395	: "r" (value), "r" (addr), "i" (-EFAULT));
	396	if (res)
	397	goto fault;
7f18f151	398	compute_return_epc(regs);
1da177e4 LT	399	break;
	400
	401	case sd_op:
875d43e7	402	#ifdef CONFIG_64BIT
1da177e4 LT	403	/*
	404	* A 32-bit kernel might be running on a 64-bit processor. But
	405	* if we're on a 32-bit processor and an i-cache incoherency
	406	* or race makes us see a 64-bit instruction here the sdl/sdr
	407	* would blow up, so for now we don't handle unaligned 64-bit
	408	* instructions on 32-bit kernels.
	409	*/
	410	if (!access_ok(VERIFY_WRITE, addr, 8))
	411	goto sigbus;
	412
	413	value = regs->regs[insn.i_format.rt];
	414	__asm__ __volatile__ (
	415	#ifdef __BIG_ENDIAN
	416	"1:\tsdl\t%1,(%2)\n"
	417	"2:\tsdr\t%1, 7(%2)\n\t"
	418	#endif
	419	#ifdef __LITTLE_ENDIAN
	420	"1:\tsdl\t%1, 7(%2)\n"
	421	"2:\tsdr\t%1, (%2)\n\t"
	422	#endif
	423	"li\t%0, 0\n"
	424	"3:\n\t"
	425	".section\t.fixup,\"ax\"\n\t"
	426	"4:\tli\t%0, %3\n\t"
	427	"j\t3b\n\t"
	428	".previous\n\t"
	429	".section\t__ex_table,\"a\"\n\t"
	430	STR(PTR)"\t1b, 4b\n\t"
	431	STR(PTR)"\t2b, 4b\n\t"
	432	".previous"
	433	: "=r" (res)
	434	: "r" (value), "r" (addr), "i" (-EFAULT));
	435	if (res)
	436	goto fault;
7f18f151	437	compute_return_epc(regs);
1da177e4	438	break;
875d43e7	439	#endif /* CONFIG_64BIT */
1da177e4 LT	440
	441	/* Cannot handle 64-bit instructions in 32-bit kernel */
	442	goto sigill;
	443
	444	case lwc1_op:
	445	case ldc1_op:
	446	case swc1_op:
	447	case sdc1_op:
	448	/*
	449	* I herewith declare: this does not happen. So send SIGBUS.
	450	*/
	451	goto sigbus;
	452
69f3a7de RB	453	/*
	454	* COP2 is available to implementor for application specific use.
	455	* It's up to applications to register a notifier chain and do
	456	* whatever they have to do, including possible sending of signals.
	457	*/
1da177e4	458	case lwc2_op:
69f3a7de RB	459	cu2_notifier_call_chain(CU2_LWC2_OP, regs);
	460	break;
	461
1da177e4	462	case ldc2_op:
69f3a7de RB	463	cu2_notifier_call_chain(CU2_LDC2_OP, regs);
	464	break;
	465
1da177e4	466	case swc2_op:
69f3a7de RB	467	cu2_notifier_call_chain(CU2_SWC2_OP, regs);
	468	break;
	469
1da177e4	470	case sdc2_op:
69f3a7de RB	471	cu2_notifier_call_chain(CU2_SDC2_OP, regs);
	472	break;
	473
1da177e4 LT	474	default:
	475	/*
	476	* Pheeee... We encountered an yet unknown instruction or
	477	* cache coherence problem. Die sucker, die ...
	478	*/
	479	goto sigill;
	480	}
	481
6312e0ee	482	#ifdef CONFIG_DEBUG_FS
1da177e4 LT	483	unaligned_instructions++;
	484	#endif
	485
7f18f151	486	return;
1da177e4 LT	487
	488	fault:
	489	/* Did we have an exception handler installed? */
	490	if (fixup_exception(regs))
7f18f151	491	return;
1da177e4	492
49a89efb	493	die_if_kernel("Unhandled kernel unaligned access", regs);
a6d5ff04	494	force_sig(SIGSEGV, current);
1da177e4	495
7f18f151	496	return;
1da177e4 LT	497
	498	sigbus:
	499	die_if_kernel("Unhandled kernel unaligned access", regs);
a6d5ff04	500	force_sig(SIGBUS, current);
1da177e4	501
7f18f151	502	return;
1da177e4 LT	503
	504	sigill:
	505	die_if_kernel("Unhandled kernel unaligned access or invalid instruction", regs);
a6d5ff04	506	force_sig(SIGILL, current);
1da177e4 LT	507	}
	508
	509	asmlinkage void do_ade(struct pt_regs *regs)
	510	{
fe00f943	511	unsigned int __user *pc;
1da177e4	512	mm_segment_t seg;
1da177e4	513
1da177e4 LT	514	/*
	515	* Did we catch a fault trying to load an instruction?
	516	* Or are we running in MIPS16 mode?
	517	*/
	518	if ((regs->cp0_badvaddr == regs->cp0_epc) \|\| (regs->cp0_epc & 0x1))
	519	goto sigbus;
	520
fe00f943	521	pc = (unsigned int __user *) exception_epc(regs);
293c5bd1	522	if (user_mode(regs) && !test_thread_flag(TIF_FIXADE))
1da177e4	523	goto sigbus;
6312e0ee AN	524	if (unaligned_action == UNALIGNED_ACTION_SIGNAL)
	525	goto sigbus;
	526	else if (unaligned_action == UNALIGNED_ACTION_SHOW)
	527	show_registers(regs);
1da177e4 LT	528
	529	/*
	530	* Do branch emulation only if we didn't forward the exception.
	531	* This is all so but ugly ...
	532	*/
	533	seg = get_fs();
	534	if (!user_mode(regs))
	535	set_fs(KERNEL_DS);
7f18f151	536	emulate_load_store_insn(regs, (void __user *)regs->cp0_badvaddr, pc);
1da177e4 LT	537	set_fs(seg);
	538
	539	return;
	540
	541	sigbus:
	542	die_if_kernel("Kernel unaligned instruction access", regs);
	543	force_sig(SIGBUS, current);
	544
	545	/*
	546	* XXX On return from the signal handler we should advance the epc
	547	*/
	548	}
6312e0ee AN	549
	550	#ifdef CONFIG_DEBUG_FS
	551	extern struct dentry *mips_debugfs_dir;
	552	static int __init debugfs_unaligned(void)
	553	{
	554	struct dentry *d;
	555
	556	if (!mips_debugfs_dir)
	557	return -ENODEV;
	558	d = debugfs_create_u32("unaligned_instructions", S_IRUGO,
	559	mips_debugfs_dir, &unaligned_instructions);
b517531c Z	560	if (!d)
b517531c Z	561	return -ENOMEM;
6312e0ee AN	562	d = debugfs_create_u32("unaligned_action", S_IRUGO \| S_IWUSR,
6312e0ee AN	563	mips_debugfs_dir, &unaligned_action);
b517531c Z	564	if (!d)
b517531c Z	565	return -ENOMEM;
6312e0ee AN	566	return 0;
	567	}
	568	__initcall(debugfs_unaligned);
	569	#endif