[net-next-2.6.git] / arch / mips / kernel / unaligned.c

/*
 * Handle unaligned accesses by emulation.
 *
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Copyright (C) 1996, 1998, 1999, 2002 by Ralf Baechle
 * Copyright (C) 1999 Silicon Graphics, Inc.
 *
 * This file contains exception handler for address error exception with the
 * special capability to execute faulting instructions in software.  The
 * handler does not try to handle the case when the program counter points
 * to an address not aligned to a word boundary.
 *
 * Putting data to unaligned addresses is a bad practice even on Intel where
 * only the performance is affected.  Much worse is that such code is non-
 * portable.  Due to several programs that die on MIPS due to alignment
 * problems I decided to implement this handler anyway though I originally
 * didn't intend to do this at all for user code.
 *
 * For now I enable fixing of address errors by default to make life easier.
 * I however intend to disable this somewhen in the future when the alignment
 * problems with user programs have been fixed.  For programmers this is the
 * right way to go.
 *
 * Fixing address errors is a per process option.  The option is inherited
 * across fork(2) and execve(2) calls.  If you really want to use the
 * option in your user programs - I discourage the use of the software
 * emulation strongly - use the following code in your userland stuff:
 *
 * #include <sys/sysmips.h>
 *
 * ...
 * sysmips(MIPS_FIXADE, x);
 * ...
 *
 * The argument x is 0 for disabling software emulation, enabled otherwise.
 *
 * Below a little program to play around with this feature.
 *
 * #include <stdio.h>
 * #include <sys/sysmips.h>
 *
 * struct foo {
 *         unsigned char bar[8];
 * };
 *
 * main(int argc, char *argv[])
 * {
 *         struct foo x = {0, 1, 2, 3, 4, 5, 6, 7};
 *         unsigned int *p = (unsigned int *) (x.bar + 3);
 *         int i;
 *
 *         if (argc > 1)
 *                 sysmips(MIPS_FIXADE, atoi(argv[1]));
 *
 *         printf("*p = %08lx\n", *p);
 *
 *         *p = 0xdeadface;
 *
 *         for(i = 0; i <= 7; i++)
 *         printf("%02x ", x.bar[i]);
 *         printf("\n");
 * }
 *
 * Coprocessor loads are not supported; I think this case is unimportant
 * in the practice.
 *
 * TODO: Handle ndc (attempted store to doubleword in uncached memory)
 *       exception for the R6000.
 *       A store crossing a page boundary might be executed only partially.
 *       Undo the partial store in this case.
 */
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/signal.h>
#include <linux/smp.h>
#include <linux/sched.h>
#include <linux/debugfs.h>
#include <linux/perf_event.h>

#include <asm/asm.h>
#include <asm/branch.h>
#include <asm/byteorder.h>
#include <asm/cop2.h>
#include <asm/inst.h>
#include <asm/uaccess.h>
#include <asm/system.h>

#define STR(x)  __STR(x)
#define __STR(x)  #x

enum {
	UNALIGNED_ACTION_QUIET,
	UNALIGNED_ACTION_SIGNAL,
	UNALIGNED_ACTION_SHOW,
};
#ifdef CONFIG_DEBUG_FS
static u32 unaligned_instructions;
static u32 unaligned_action;
#else
#define unaligned_action UNALIGNED_ACTION_QUIET
#endif
extern void show_registers(struct pt_regs *regs);

static void emulate_load_store_insn(struct pt_regs *regs,
	void __user *addr, unsigned int __user *pc)
{
	union mips_instruction insn;
	unsigned long value;
	unsigned int res;

	perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS,
		      1, 0, regs, 0);

	/*
	 * This load never faults.
	 */
	__get_user(insn.word, pc);

	switch (insn.i_format.opcode) {
	/*
	 * These are instructions that a compiler doesn't generate.  We
	 * can assume therefore that the code is MIPS-aware and
	 * really buggy.  Emulating these instructions would break the
	 * semantics anyway.
	 */
	case ll_op:
	case lld_op:
	case sc_op:
	case scd_op:

	/*
	 * For these instructions the only way to create an address
	 * error is an attempted access to kernel/supervisor address
	 * space.
	 */
	case ldl_op:
	case ldr_op:
	case lwl_op:
	case lwr_op:
	case sdl_op:
	case sdr_op:
	case swl_op:
	case swr_op:
	case lb_op:
	case lbu_op:
	case sb_op:
		goto sigbus;

	/*
	 * The remaining opcodes are the ones that are really of interest.
	 */
	case lh_op:
		if (!access_ok(VERIFY_READ, addr, 2))
			goto sigbus;

		__asm__ __volatile__ (".set\tnoat\n"
#ifdef __BIG_ENDIAN
			"1:\tlb\t%0, 0(%2)\n"
			"2:\tlbu\t$1, 1(%2)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
			"1:\tlb\t%0, 1(%2)\n"
			"2:\tlbu\t$1, 0(%2)\n\t"
#endif
			"sll\t%0, 0x8\n\t"
			"or\t%0, $1\n\t"
			"li\t%1, 0\n"
			"3:\t.set\tat\n\t"
			".section\t.fixup,\"ax\"\n\t"
			"4:\tli\t%1, %3\n\t"
			"j\t3b\n\t"
			".previous\n\t"
			".section\t__ex_table,\"a\"\n\t"
			STR(PTR)"\t1b, 4b\n\t"
			STR(PTR)"\t2b, 4b\n\t"
			".previous"
			: "=&r" (value), "=r" (res)
			: "r" (addr), "i" (-EFAULT));
		if (res)
			goto fault;
		compute_return_epc(regs);
		regs->regs[insn.i_format.rt] = value;
		break;

	case lw_op:
		if (!access_ok(VERIFY_READ, addr, 4))
			goto sigbus;

		__asm__ __volatile__ (
#ifdef __BIG_ENDIAN
			"1:\tlwl\t%0, (%2)\n"
			"2:\tlwr\t%0, 3(%2)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
			"1:\tlwl\t%0, 3(%2)\n"
			"2:\tlwr\t%0, (%2)\n\t"
#endif
			"li\t%1, 0\n"
			"3:\t.section\t.fixup,\"ax\"\n\t"
			"4:\tli\t%1, %3\n\t"
			"j\t3b\n\t"
			".previous\n\t"
			".section\t__ex_table,\"a\"\n\t"
			STR(PTR)"\t1b, 4b\n\t"
			STR(PTR)"\t2b, 4b\n\t"
			".previous"
			: "=&r" (value), "=r" (res)
			: "r" (addr), "i" (-EFAULT));
		if (res)
			goto fault;
		compute_return_epc(regs);
		regs->regs[insn.i_format.rt] = value;
		break;

	case lhu_op:
		if (!access_ok(VERIFY_READ, addr, 2))
			goto sigbus;

		__asm__ __volatile__ (
			".set\tnoat\n"
#ifdef __BIG_ENDIAN
			"1:\tlbu\t%0, 0(%2)\n"
			"2:\tlbu\t$1, 1(%2)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
			"1:\tlbu\t%0, 1(%2)\n"
			"2:\tlbu\t$1, 0(%2)\n\t"
#endif
			"sll\t%0, 0x8\n\t"
			"or\t%0, $1\n\t"
			"li\t%1, 0\n"
			"3:\t.set\tat\n\t"
			".section\t.fixup,\"ax\"\n\t"
			"4:\tli\t%1, %3\n\t"
			"j\t3b\n\t"
			".previous\n\t"
			".section\t__ex_table,\"a\"\n\t"
			STR(PTR)"\t1b, 4b\n\t"
			STR(PTR)"\t2b, 4b\n\t"
			".previous"
			: "=&r" (value), "=r" (res)
			: "r" (addr), "i" (-EFAULT));
		if (res)
			goto fault;
		compute_return_epc(regs);
		regs->regs[insn.i_format.rt] = value;
		break;

	case lwu_op:
#ifdef CONFIG_64BIT
		/*
		 * A 32-bit kernel might be running on a 64-bit processor.  But
		 * if we're on a 32-bit processor and an i-cache incoherency
		 * or race makes us see a 64-bit instruction here the sdl/sdr
		 * would blow up, so for now we don't handle unaligned 64-bit
		 * instructions on 32-bit kernels.
		 */
		if (!access_ok(VERIFY_READ, addr, 4))
			goto sigbus;

		__asm__ __volatile__ (
#ifdef __BIG_ENDIAN
			"1:\tlwl\t%0, (%2)\n"
			"2:\tlwr\t%0, 3(%2)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
			"1:\tlwl\t%0, 3(%2)\n"
			"2:\tlwr\t%0, (%2)\n\t"
#endif
			"dsll\t%0, %0, 32\n\t"
			"dsrl\t%0, %0, 32\n\t"
			"li\t%1, 0\n"
			"3:\t.section\t.fixup,\"ax\"\n\t"
			"4:\tli\t%1, %3\n\t"
			"j\t3b\n\t"
			".previous\n\t"
			".section\t__ex_table,\"a\"\n\t"
			STR(PTR)"\t1b, 4b\n\t"
			STR(PTR)"\t2b, 4b\n\t"
			".previous"
			: "=&r" (value), "=r" (res)
			: "r" (addr), "i" (-EFAULT));
		if (res)
			goto fault;
		compute_return_epc(regs);
		regs->regs[insn.i_format.rt] = value;
		break;
#endif /* CONFIG_64BIT */

		/* Cannot handle 64-bit instructions in 32-bit kernel */
		goto sigill;

	case ld_op:
#ifdef CONFIG_64BIT
		/*
		 * A 32-bit kernel might be running on a 64-bit processor.  But
		 * if we're on a 32-bit processor and an i-cache incoherency
		 * or race makes us see a 64-bit instruction here the sdl/sdr
		 * would blow up, so for now we don't handle unaligned 64-bit
		 * instructions on 32-bit kernels.
		 */
		if (!access_ok(VERIFY_READ, addr, 8))
			goto sigbus;

		__asm__ __volatile__ (
#ifdef __BIG_ENDIAN
			"1:\tldl\t%0, (%2)\n"
			"2:\tldr\t%0, 7(%2)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
			"1:\tldl\t%0, 7(%2)\n"
			"2:\tldr\t%0, (%2)\n\t"
#endif
			"li\t%1, 0\n"
			"3:\t.section\t.fixup,\"ax\"\n\t"
			"4:\tli\t%1, %3\n\t"
			"j\t3b\n\t"
			".previous\n\t"
			".section\t__ex_table,\"a\"\n\t"
			STR(PTR)"\t1b, 4b\n\t"
			STR(PTR)"\t2b, 4b\n\t"
			".previous"
			: "=&r" (value), "=r" (res)
			: "r" (addr), "i" (-EFAULT));
		if (res)
			goto fault;
		compute_return_epc(regs);
		regs->regs[insn.i_format.rt] = value;
		break;
#endif /* CONFIG_64BIT */

		/* Cannot handle 64-bit instructions in 32-bit kernel */
		goto sigill;

	case sh_op:
		if (!access_ok(VERIFY_WRITE, addr, 2))
			goto sigbus;

		value = regs->regs[insn.i_format.rt];
		__asm__ __volatile__ (
#ifdef __BIG_ENDIAN
			".set\tnoat\n"
			"1:\tsb\t%1, 1(%2)\n\t"
			"srl\t$1, %1, 0x8\n"
			"2:\tsb\t$1, 0(%2)\n\t"
			".set\tat\n\t"
#endif
#ifdef __LITTLE_ENDIAN
			".set\tnoat\n"
			"1:\tsb\t%1, 0(%2)\n\t"
			"srl\t$1,%1, 0x8\n"
			"2:\tsb\t$1, 1(%2)\n\t"
			".set\tat\n\t"
#endif
			"li\t%0, 0\n"
			"3:\n\t"
			".section\t.fixup,\"ax\"\n\t"
			"4:\tli\t%0, %3\n\t"
			"j\t3b\n\t"
			".previous\n\t"
			".section\t__ex_table,\"a\"\n\t"
			STR(PTR)"\t1b, 4b\n\t"
			STR(PTR)"\t2b, 4b\n\t"
			".previous"
			: "=r" (res)
			: "r" (value), "r" (addr), "i" (-EFAULT));
		if (res)
			goto fault;
		compute_return_epc(regs);
		break;

	case sw_op:
		if (!access_ok(VERIFY_WRITE, addr, 4))
			goto sigbus;

		value = regs->regs[insn.i_format.rt];
		__asm__ __volatile__ (
#ifdef __BIG_ENDIAN
			"1:\tswl\t%1,(%2)\n"
			"2:\tswr\t%1, 3(%2)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
			"1:\tswl\t%1, 3(%2)\n"
			"2:\tswr\t%1, (%2)\n\t"
#endif
			"li\t%0, 0\n"
			"3:\n\t"
			".section\t.fixup,\"ax\"\n\t"
			"4:\tli\t%0, %3\n\t"
			"j\t3b\n\t"
			".previous\n\t"
			".section\t__ex_table,\"a\"\n\t"
			STR(PTR)"\t1b, 4b\n\t"
			STR(PTR)"\t2b, 4b\n\t"
			".previous"
		: "=r" (res)
		: "r" (value), "r" (addr), "i" (-EFAULT));
		if (res)
			goto fault;
		compute_return_epc(regs);
		break;

	case sd_op:
#ifdef CONFIG_64BIT
		/*
		 * A 32-bit kernel might be running on a 64-bit processor.  But
		 * if we're on a 32-bit processor and an i-cache incoherency
		 * or race makes us see a 64-bit instruction here the sdl/sdr
		 * would blow up, so for now we don't handle unaligned 64-bit
		 * instructions on 32-bit kernels.
		 */
		if (!access_ok(VERIFY_WRITE, addr, 8))
			goto sigbus;

		value = regs->regs[insn.i_format.rt];
		__asm__ __volatile__ (
#ifdef __BIG_ENDIAN
			"1:\tsdl\t%1,(%2)\n"
			"2:\tsdr\t%1, 7(%2)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
			"1:\tsdl\t%1, 7(%2)\n"
			"2:\tsdr\t%1, (%2)\n\t"
#endif
			"li\t%0, 0\n"
			"3:\n\t"
			".section\t.fixup,\"ax\"\n\t"
			"4:\tli\t%0, %3\n\t"
			"j\t3b\n\t"
			".previous\n\t"
			".section\t__ex_table,\"a\"\n\t"
			STR(PTR)"\t1b, 4b\n\t"
			STR(PTR)"\t2b, 4b\n\t"
			".previous"
		: "=r" (res)
		: "r" (value), "r" (addr), "i" (-EFAULT));
		if (res)
			goto fault;
		compute_return_epc(regs);
		break;
#endif /* CONFIG_64BIT */

		/* Cannot handle 64-bit instructions in 32-bit kernel */
		goto sigill;

	case lwc1_op:
	case ldc1_op:
	case swc1_op:
	case sdc1_op:
		/*
		 * I herewith declare: this does not happen.  So send SIGBUS.
		 */
		goto sigbus;

	/*
	 * COP2 is available to implementor for application specific use.
	 * It's up to applications to register a notifier chain and do
	 * whatever they have to do, including possible sending of signals.
	 */
	case lwc2_op:
		cu2_notifier_call_chain(CU2_LWC2_OP, regs);
		break;

	case ldc2_op:
		cu2_notifier_call_chain(CU2_LDC2_OP, regs);
		break;

	case swc2_op:
		cu2_notifier_call_chain(CU2_SWC2_OP, regs);
		break;

	case sdc2_op:
		cu2_notifier_call_chain(CU2_SDC2_OP, regs);
		break;

	default:
		/*
		 * Pheeee...  We encountered an yet unknown instruction or
		 * cache coherence problem.  Die sucker, die ...
		 */
		goto sigill;
	}

#ifdef CONFIG_DEBUG_FS
	unaligned_instructions++;
#endif

	return;

fault:
	/* Did we have an exception handler installed? */
	if (fixup_exception(regs))
		return;

	die_if_kernel("Unhandled kernel unaligned access", regs);
	force_sig(SIGSEGV, current);

	return;

sigbus:
	die_if_kernel("Unhandled kernel unaligned access", regs);
	force_sig(SIGBUS, current);

	return;

sigill:
	die_if_kernel("Unhandled kernel unaligned access or invalid instruction", regs);
	force_sig(SIGILL, current);
}

asmlinkage void do_ade(struct pt_regs *regs)
{
	unsigned int __user *pc;
	mm_segment_t seg;

	perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS,
			1, 0, regs, regs->cp0_badvaddr);
	/*
	 * Did we catch a fault trying to load an instruction?
	 * Or are we running in MIPS16 mode?
	 */
	if ((regs->cp0_badvaddr == regs->cp0_epc) || (regs->cp0_epc & 0x1))
		goto sigbus;

	pc = (unsigned int __user *) exception_epc(regs);
	if (user_mode(regs) && !test_thread_flag(TIF_FIXADE))
		goto sigbus;
	if (unaligned_action == UNALIGNED_ACTION_SIGNAL)
		goto sigbus;
	else if (unaligned_action == UNALIGNED_ACTION_SHOW)
		show_registers(regs);

	/*
	 * Do branch emulation only if we didn't forward the exception.
	 * This is all so but ugly ...
	 */
	seg = get_fs();
	if (!user_mode(regs))
		set_fs(KERNEL_DS);
	emulate_load_store_insn(regs, (void __user *)regs->cp0_badvaddr, pc);
	set_fs(seg);

	return;

sigbus:
	die_if_kernel("Kernel unaligned instruction access", regs);
	force_sig(SIGBUS, current);

	/*
	 * XXX On return from the signal handler we should advance the epc
	 */
}

#ifdef CONFIG_DEBUG_FS
extern struct dentry *mips_debugfs_dir;
static int __init debugfs_unaligned(void)
{
	struct dentry *d;

	if (!mips_debugfs_dir)
		return -ENODEV;
	d = debugfs_create_u32("unaligned_instructions", S_IRUGO,
			       mips_debugfs_dir, &unaligned_instructions);
	if (!d)
		return -ENOMEM;
	d = debugfs_create_u32("unaligned_action", S_IRUGO | S_IWUSR,
			       mips_debugfs_dir, &unaligned_action);
	if (!d)
		return -ENOMEM;
	return 0;
}
__initcall(debugfs_unaligned);
#endif
Commit	Line	Data
1da177e4 LT	1	/*
	2	* Handle unaligned accesses by emulation.
	3	*
	4	* This file is subject to the terms and conditions of the GNU General Public
	5	* License. See the file "COPYING" in the main directory of this archive
	6	* for more details.
	7	*
	8	* Copyright (C) 1996, 1998, 1999, 2002 by Ralf Baechle
	9	* Copyright (C) 1999 Silicon Graphics, Inc.
	10	*
	11	* This file contains exception handler for address error exception with the
	12	* special capability to execute faulting instructions in software. The
	13	* handler does not try to handle the case when the program counter points
	14	* to an address not aligned to a word boundary.
	15	*
	16	* Putting data to unaligned addresses is a bad practice even on Intel where
	17	* only the performance is affected. Much worse is that such code is non-
	18	* portable. Due to several programs that die on MIPS due to alignment
	19	* problems I decided to implement this handler anyway though I originally
	20	* didn't intend to do this at all for user code.
	21	*
	22	* For now I enable fixing of address errors by default to make life easier.
	23	* I however intend to disable this somewhen in the future when the alignment
	24	* problems with user programs have been fixed. For programmers this is the
	25	* right way to go.
	26	*
	27	* Fixing address errors is a per process option. The option is inherited
	28	* across fork(2) and execve(2) calls. If you really want to use the
	29	* option in your user programs - I discourage the use of the software
	30	* emulation strongly - use the following code in your userland stuff:
	31	*
	32	* #include <sys/sysmips.h>
	33	*
	34	* ...
	35	* sysmips(MIPS_FIXADE, x);
	36	* ...
	37	*
	38	* The argument x is 0 for disabling software emulation, enabled otherwise.
	39	*
	40	* Below a little program to play around with this feature.
	41	*
	42	* #include <stdio.h>
	43	* #include <sys/sysmips.h>
	44	*
	45	* struct foo {
	46	* unsigned char bar[8];
	47	* };
	48	*
	49	* main(int argc, char *argv[])
	50	* {
	51	* struct foo x = {0, 1, 2, 3, 4, 5, 6, 7};
	52	* unsigned int p = (unsigned int ) (x.bar + 3);
	53	* int i;
	54	*
	55	* if (argc > 1)
	56	* sysmips(MIPS_FIXADE, atoi(argv[1]));
	57	*
	58	* printf("p = %08lx\n", p);
	59	*
	60	* *p = 0xdeadface;
	61	*
	62	* for(i = 0; i <= 7; i++)
	63	* printf("%02x ", x.bar[i]);
	64	* printf("\n");
65	* }
66	*
67	* Coprocessor loads are not supported; I think this case is unimportant
68	* in the practice.
69	*
70	* TODO: Handle ndc (attempted store to doubleword in uncached memory)
71	* exception for the R6000.
72	* A store crossing a page boundary might be executed only partially.
73	* Undo the partial store in this case.
74	*/
1da177e4 LT	75	#include <linux/mm.h>
	76	#include <linux/module.h>
	77	#include <linux/signal.h>
	78	#include <linux/smp.h>
e8edc6e0	79	#include <linux/sched.h>
6312e0ee	80	#include <linux/debugfs.h>
7f788d2d DCZ	81	#include <linux/perf_event.h>
7f788d2d DCZ	82
1da177e4 LT	83	#include <asm/asm.h>
	84	#include <asm/branch.h>
	85	#include <asm/byteorder.h>
69f3a7de	86	#include <asm/cop2.h>
1da177e4 LT	87	#include <asm/inst.h>
	88	#include <asm/uaccess.h>
	89	#include <asm/system.h>
	90
	91	#define STR(x) __STR(x)
	92	#define __STR(x) #x
	93
6312e0ee AN	94	enum {
	95	UNALIGNED_ACTION_QUIET,
	96	UNALIGNED_ACTION_SIGNAL,
	97	UNALIGNED_ACTION_SHOW,
	98	};
	99	#ifdef CONFIG_DEBUG_FS
	100	static u32 unaligned_instructions;
	101	static u32 unaligned_action;
	102	#else
	103	#define unaligned_action UNALIGNED_ACTION_QUIET
1da177e4	104	#endif
6312e0ee	105	extern void show_registers(struct pt_regs *regs);
1da177e4	106
7f18f151 RB	107	static void emulate_load_store_insn(struct pt_regs *regs,
7f18f151 RB	108	void __user addr, unsigned int __user pc)
1da177e4 LT	109	{
	110	union mips_instruction insn;
	111	unsigned long value;
	112	unsigned int res;
	113
7f788d2d DCZ	114	perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS,
	115	1, 0, regs, 0);
	116
1da177e4 LT	117	/*
	118	* This load never faults.
	119	*/
fe00f943	120	__get_user(insn.word, pc);
1da177e4 LT	121
	122	switch (insn.i_format.opcode) {
	123	/*
	124	* These are instructions that a compiler doesn't generate. We
	125	* can assume therefore that the code is MIPS-aware and
	126	* really buggy. Emulating these instructions would break the
	127	* semantics anyway.
	128	*/
	129	case ll_op:
	130	case lld_op:
	131	case sc_op:
	132	case scd_op:
	133
	134	/*
	135	* For these instructions the only way to create an address
	136	* error is an attempted access to kernel/supervisor address
	137	* space.
	138	*/
	139	case ldl_op:
	140	case ldr_op:
	141	case lwl_op:
	142	case lwr_op:
	143	case sdl_op:
	144	case sdr_op:
	145	case swl_op:
	146	case swr_op:
	147	case lb_op:
	148	case lbu_op:
	149	case sb_op:
	150	goto sigbus;
	151
	152	/*
	153	* The remaining opcodes are the ones that are really of interest.
	154	*/
	155	case lh_op:
	156	if (!access_ok(VERIFY_READ, addr, 2))
	157	goto sigbus;
	158
	159	__asm__ __volatile__ (".set\tnoat\n"
	160	#ifdef __BIG_ENDIAN
	161	"1:\tlb\t%0, 0(%2)\n"
	162	"2:\tlbu\t$1, 1(%2)\n\t"
	163	#endif
	164	#ifdef __LITTLE_ENDIAN
	165	"1:\tlb\t%0, 1(%2)\n"
	166	"2:\tlbu\t$1, 0(%2)\n\t"
	167	#endif
	168	"sll\t%0, 0x8\n\t"
	169	"or\t%0, $1\n\t"
	170	"li\t%1, 0\n"
	171	"3:\t.set\tat\n\t"
	172	".section\t.fixup,\"ax\"\n\t"
	173	"4:\tli\t%1, %3\n\t"
	174	"j\t3b\n\t"
	175	".previous\n\t"
	176	".section\t__ex_table,\"a\"\n\t"
	177	STR(PTR)"\t1b, 4b\n\t"
	178	STR(PTR)"\t2b, 4b\n\t"
	179	".previous"
	180	: "=&r" (value), "=r" (res)
	181	: "r" (addr), "i" (-EFAULT));
	182	if (res)
	183	goto fault;
7f18f151 RB	184	compute_return_epc(regs);
7f18f151 RB	185	regs->regs[insn.i_format.rt] = value;
1da177e4 LT	186	break;
	187
	188	case lw_op:
	189	if (!access_ok(VERIFY_READ, addr, 4))
	190	goto sigbus;
	191
	192	__asm__ __volatile__ (
	193	#ifdef __BIG_ENDIAN
	194	"1:\tlwl\t%0, (%2)\n"
	195	"2:\tlwr\t%0, 3(%2)\n\t"
	196	#endif
	197	#ifdef __LITTLE_ENDIAN
	198	"1:\tlwl\t%0, 3(%2)\n"
	199	"2:\tlwr\t%0, (%2)\n\t"
	200	#endif
	201	"li\t%1, 0\n"
	202	"3:\t.section\t.fixup,\"ax\"\n\t"
	203	"4:\tli\t%1, %3\n\t"
	204	"j\t3b\n\t"
	205	".previous\n\t"
	206	".section\t__ex_table,\"a\"\n\t"
	207	STR(PTR)"\t1b, 4b\n\t"
	208	STR(PTR)"\t2b, 4b\n\t"
	209	".previous"
	210	: "=&r" (value), "=r" (res)
	211	: "r" (addr), "i" (-EFAULT));
	212	if (res)
	213	goto fault;
7f18f151 RB	214	compute_return_epc(regs);
7f18f151 RB	215	regs->regs[insn.i_format.rt] = value;
1da177e4 LT	216	break;
	217
	218	case lhu_op:
	219	if (!access_ok(VERIFY_READ, addr, 2))
	220	goto sigbus;
	221
	222	__asm__ __volatile__ (
	223	".set\tnoat\n"
	224	#ifdef __BIG_ENDIAN
	225	"1:\tlbu\t%0, 0(%2)\n"
	226	"2:\tlbu\t$1, 1(%2)\n\t"
	227	#endif
	228	#ifdef __LITTLE_ENDIAN
	229	"1:\tlbu\t%0, 1(%2)\n"
	230	"2:\tlbu\t$1, 0(%2)\n\t"
	231	#endif
	232	"sll\t%0, 0x8\n\t"
	233	"or\t%0, $1\n\t"
	234	"li\t%1, 0\n"
	235	"3:\t.set\tat\n\t"
	236	".section\t.fixup,\"ax\"\n\t"
	237	"4:\tli\t%1, %3\n\t"
	238	"j\t3b\n\t"
	239	".previous\n\t"
	240	".section\t__ex_table,\"a\"\n\t"
	241	STR(PTR)"\t1b, 4b\n\t"
	242	STR(PTR)"\t2b, 4b\n\t"
	243	".previous"
	244	: "=&r" (value), "=r" (res)
	245	: "r" (addr), "i" (-EFAULT));
	246	if (res)
	247	goto fault;
7f18f151 RB	248	compute_return_epc(regs);
7f18f151 RB	249	regs->regs[insn.i_format.rt] = value;
1da177e4 LT	250	break;
	251
	252	case lwu_op:
875d43e7	253	#ifdef CONFIG_64BIT
1da177e4 LT	254	/*
	255	* A 32-bit kernel might be running on a 64-bit processor. But
	256	* if we're on a 32-bit processor and an i-cache incoherency
	257	* or race makes us see a 64-bit instruction here the sdl/sdr
	258	* would blow up, so for now we don't handle unaligned 64-bit
	259	* instructions on 32-bit kernels.
	260	*/
	261	if (!access_ok(VERIFY_READ, addr, 4))
	262	goto sigbus;
	263
	264	__asm__ __volatile__ (
	265	#ifdef __BIG_ENDIAN
	266	"1:\tlwl\t%0, (%2)\n"
	267	"2:\tlwr\t%0, 3(%2)\n\t"
	268	#endif
	269	#ifdef __LITTLE_ENDIAN
	270	"1:\tlwl\t%0, 3(%2)\n"
	271	"2:\tlwr\t%0, (%2)\n\t"
	272	#endif
	273	"dsll\t%0, %0, 32\n\t"
	274	"dsrl\t%0, %0, 32\n\t"
	275	"li\t%1, 0\n"
	276	"3:\t.section\t.fixup,\"ax\"\n\t"
	277	"4:\tli\t%1, %3\n\t"
	278	"j\t3b\n\t"
	279	".previous\n\t"
	280	".section\t__ex_table,\"a\"\n\t"
	281	STR(PTR)"\t1b, 4b\n\t"
	282	STR(PTR)"\t2b, 4b\n\t"
	283	".previous"
	284	: "=&r" (value), "=r" (res)
	285	: "r" (addr), "i" (-EFAULT));
	286	if (res)
	287	goto fault;
7f18f151 RB	288	compute_return_epc(regs);
7f18f151 RB	289	regs->regs[insn.i_format.rt] = value;
1da177e4	290	break;
875d43e7	291	#endif /* CONFIG_64BIT */
1da177e4 LT	292
	293	/* Cannot handle 64-bit instructions in 32-bit kernel */
	294	goto sigill;
	295
	296	case ld_op:
875d43e7	297	#ifdef CONFIG_64BIT
1da177e4 LT	298	/*
	299	* A 32-bit kernel might be running on a 64-bit processor. But
	300	* if we're on a 32-bit processor and an i-cache incoherency
	301	* or race makes us see a 64-bit instruction here the sdl/sdr
	302	* would blow up, so for now we don't handle unaligned 64-bit
	303	* instructions on 32-bit kernels.
	304	*/
	305	if (!access_ok(VERIFY_READ, addr, 8))
	306	goto sigbus;
	307
	308	__asm__ __volatile__ (
	309	#ifdef __BIG_ENDIAN
	310	"1:\tldl\t%0, (%2)\n"
	311	"2:\tldr\t%0, 7(%2)\n\t"
	312	#endif
	313	#ifdef __LITTLE_ENDIAN
	314	"1:\tldl\t%0, 7(%2)\n"
	315	"2:\tldr\t%0, (%2)\n\t"
	316	#endif
	317	"li\t%1, 0\n"
	318	"3:\t.section\t.fixup,\"ax\"\n\t"
	319	"4:\tli\t%1, %3\n\t"
	320	"j\t3b\n\t"
	321	".previous\n\t"
	322	".section\t__ex_table,\"a\"\n\t"
	323	STR(PTR)"\t1b, 4b\n\t"
	324	STR(PTR)"\t2b, 4b\n\t"
	325	".previous"
	326	: "=&r" (value), "=r" (res)
	327	: "r" (addr), "i" (-EFAULT));
	328	if (res)
	329	goto fault;
7f18f151 RB	330	compute_return_epc(regs);
7f18f151 RB	331	regs->regs[insn.i_format.rt] = value;
1da177e4	332	break;
875d43e7	333	#endif /* CONFIG_64BIT */
1da177e4 LT	334
	335	/* Cannot handle 64-bit instructions in 32-bit kernel */
	336	goto sigill;
	337
	338	case sh_op:
	339	if (!access_ok(VERIFY_WRITE, addr, 2))
	340	goto sigbus;
	341
	342	value = regs->regs[insn.i_format.rt];
	343	__asm__ __volatile__ (
	344	#ifdef __BIG_ENDIAN
	345	".set\tnoat\n"
	346	"1:\tsb\t%1, 1(%2)\n\t"
	347	"srl\t$1, %1, 0x8\n"
	348	"2:\tsb\t$1, 0(%2)\n\t"
	349	".set\tat\n\t"
	350	#endif
	351	#ifdef __LITTLE_ENDIAN
	352	".set\tnoat\n"
	353	"1:\tsb\t%1, 0(%2)\n\t"
	354	"srl\t$1,%1, 0x8\n"
	355	"2:\tsb\t$1, 1(%2)\n\t"
	356	".set\tat\n\t"
	357	#endif
	358	"li\t%0, 0\n"
	359	"3:\n\t"
	360	".section\t.fixup,\"ax\"\n\t"
	361	"4:\tli\t%0, %3\n\t"
	362	"j\t3b\n\t"
	363	".previous\n\t"
	364	".section\t__ex_table,\"a\"\n\t"
	365	STR(PTR)"\t1b, 4b\n\t"
	366	STR(PTR)"\t2b, 4b\n\t"
	367	".previous"
	368	: "=r" (res)
	369	: "r" (value), "r" (addr), "i" (-EFAULT));
	370	if (res)
	371	goto fault;
7f18f151	372	compute_return_epc(regs);
1da177e4 LT	373	break;
	374
	375	case sw_op:
	376	if (!access_ok(VERIFY_WRITE, addr, 4))
	377	goto sigbus;
	378
	379	value = regs->regs[insn.i_format.rt];
	380	__asm__ __volatile__ (
	381	#ifdef __BIG_ENDIAN
	382	"1:\tswl\t%1,(%2)\n"
	383	"2:\tswr\t%1, 3(%2)\n\t"
	384	#endif
	385	#ifdef __LITTLE_ENDIAN
	386	"1:\tswl\t%1, 3(%2)\n"
	387	"2:\tswr\t%1, (%2)\n\t"
	388	#endif
	389	"li\t%0, 0\n"
	390	"3:\n\t"
	391	".section\t.fixup,\"ax\"\n\t"
	392	"4:\tli\t%0, %3\n\t"
	393	"j\t3b\n\t"
	394	".previous\n\t"
	395	".section\t__ex_table,\"a\"\n\t"
	396	STR(PTR)"\t1b, 4b\n\t"
	397	STR(PTR)"\t2b, 4b\n\t"
	398	".previous"
	399	: "=r" (res)
	400	: "r" (value), "r" (addr), "i" (-EFAULT));
	401	if (res)
	402	goto fault;
7f18f151	403	compute_return_epc(regs);
1da177e4 LT	404	break;
	405
	406	case sd_op:
875d43e7	407	#ifdef CONFIG_64BIT
1da177e4 LT	408	/*
	409	* A 32-bit kernel might be running on a 64-bit processor. But
	410	* if we're on a 32-bit processor and an i-cache incoherency
	411	* or race makes us see a 64-bit instruction here the sdl/sdr
	412	* would blow up, so for now we don't handle unaligned 64-bit
	413	* instructions on 32-bit kernels.
	414	*/
	415	if (!access_ok(VERIFY_WRITE, addr, 8))
	416	goto sigbus;
	417
	418	value = regs->regs[insn.i_format.rt];
	419	__asm__ __volatile__ (
	420	#ifdef __BIG_ENDIAN
	421	"1:\tsdl\t%1,(%2)\n"
	422	"2:\tsdr\t%1, 7(%2)\n\t"
	423	#endif
	424	#ifdef __LITTLE_ENDIAN
	425	"1:\tsdl\t%1, 7(%2)\n"
	426	"2:\tsdr\t%1, (%2)\n\t"
	427	#endif
	428	"li\t%0, 0\n"
	429	"3:\n\t"
	430	".section\t.fixup,\"ax\"\n\t"
	431	"4:\tli\t%0, %3\n\t"
	432	"j\t3b\n\t"
	433	".previous\n\t"
	434	".section\t__ex_table,\"a\"\n\t"
	435	STR(PTR)"\t1b, 4b\n\t"
	436	STR(PTR)"\t2b, 4b\n\t"
	437	".previous"
	438	: "=r" (res)
	439	: "r" (value), "r" (addr), "i" (-EFAULT));
	440	if (res)
	441	goto fault;
7f18f151	442	compute_return_epc(regs);
1da177e4	443	break;
875d43e7	444	#endif /* CONFIG_64BIT */
1da177e4 LT	445
	446	/* Cannot handle 64-bit instructions in 32-bit kernel */
	447	goto sigill;
	448
	449	case lwc1_op:
	450	case ldc1_op:
	451	case swc1_op:
	452	case sdc1_op:
	453	/*
	454	* I herewith declare: this does not happen. So send SIGBUS.
	455	*/
	456	goto sigbus;
	457
69f3a7de RB	458	/*
	459	* COP2 is available to implementor for application specific use.
	460	* It's up to applications to register a notifier chain and do
	461	* whatever they have to do, including possible sending of signals.
	462	*/
1da177e4	463	case lwc2_op:
69f3a7de RB	464	cu2_notifier_call_chain(CU2_LWC2_OP, regs);
	465	break;
	466
1da177e4	467	case ldc2_op:
69f3a7de RB	468	cu2_notifier_call_chain(CU2_LDC2_OP, regs);
	469	break;
	470
1da177e4	471	case swc2_op:
69f3a7de RB	472	cu2_notifier_call_chain(CU2_SWC2_OP, regs);
	473	break;
	474
1da177e4	475	case sdc2_op:
69f3a7de RB	476	cu2_notifier_call_chain(CU2_SDC2_OP, regs);
	477	break;
	478
1da177e4 LT	479	default:
	480	/*
	481	* Pheeee... We encountered an yet unknown instruction or
	482	* cache coherence problem. Die sucker, die ...
	483	*/
	484	goto sigill;
	485	}
	486
6312e0ee	487	#ifdef CONFIG_DEBUG_FS
1da177e4 LT	488	unaligned_instructions++;
	489	#endif
	490
7f18f151	491	return;
1da177e4 LT	492
	493	fault:
	494	/* Did we have an exception handler installed? */
	495	if (fixup_exception(regs))
7f18f151	496	return;
1da177e4	497
49a89efb	498	die_if_kernel("Unhandled kernel unaligned access", regs);
a6d5ff04	499	force_sig(SIGSEGV, current);
1da177e4	500
7f18f151	501	return;
1da177e4 LT	502
	503	sigbus:
	504	die_if_kernel("Unhandled kernel unaligned access", regs);
a6d5ff04	505	force_sig(SIGBUS, current);
1da177e4	506
7f18f151	507	return;
1da177e4 LT	508
	509	sigill:
	510	die_if_kernel("Unhandled kernel unaligned access or invalid instruction", regs);
a6d5ff04	511	force_sig(SIGILL, current);
1da177e4 LT	512	}
	513
	514	asmlinkage void do_ade(struct pt_regs *regs)
	515	{
fe00f943	516	unsigned int __user *pc;
1da177e4	517	mm_segment_t seg;
1da177e4	518
7f788d2d DCZ	519	perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS,
7f788d2d DCZ	520	1, 0, regs, regs->cp0_badvaddr);
1da177e4 LT	521	/*
	522	* Did we catch a fault trying to load an instruction?
	523	* Or are we running in MIPS16 mode?
	524	*/
	525	if ((regs->cp0_badvaddr == regs->cp0_epc) \|\| (regs->cp0_epc & 0x1))
	526	goto sigbus;
	527
fe00f943	528	pc = (unsigned int __user *) exception_epc(regs);
293c5bd1	529	if (user_mode(regs) && !test_thread_flag(TIF_FIXADE))
1da177e4	530	goto sigbus;
6312e0ee AN	531	if (unaligned_action == UNALIGNED_ACTION_SIGNAL)
	532	goto sigbus;
	533	else if (unaligned_action == UNALIGNED_ACTION_SHOW)
	534	show_registers(regs);
1da177e4 LT	535
	536	/*
	537	* Do branch emulation only if we didn't forward the exception.
	538	* This is all so but ugly ...
	539	*/
	540	seg = get_fs();
	541	if (!user_mode(regs))
	542	set_fs(KERNEL_DS);
7f18f151	543	emulate_load_store_insn(regs, (void __user *)regs->cp0_badvaddr, pc);
1da177e4 LT	544	set_fs(seg);
	545
	546	return;
	547
	548	sigbus:
	549	die_if_kernel("Kernel unaligned instruction access", regs);
	550	force_sig(SIGBUS, current);
	551
	552	/*
	553	* XXX On return from the signal handler we should advance the epc
	554	*/
	555	}
6312e0ee AN	556
	557	#ifdef CONFIG_DEBUG_FS
	558	extern struct dentry *mips_debugfs_dir;
	559	static int __init debugfs_unaligned(void)
	560	{
	561	struct dentry *d;
	562
	563	if (!mips_debugfs_dir)
	564	return -ENODEV;
	565	d = debugfs_create_u32("unaligned_instructions", S_IRUGO,
	566	mips_debugfs_dir, &unaligned_instructions);
b517531c Z	567	if (!d)
b517531c Z	568	return -ENOMEM;
6312e0ee AN	569	d = debugfs_create_u32("unaligned_action", S_IRUGO \| S_IWUSR,
6312e0ee AN	570	mips_debugfs_dir, &unaligned_action);
b517531c Z	571	if (!d)
b517531c Z	572	return -ENOMEM;
6312e0ee AN	573	return 0;
	574	}
	575	__initcall(debugfs_unaligned);
	576	#endif