[net-next-2.6.git] / arch / i386 / kernel / alternative.c

#include <linux/module.h>
#include <linux/sched.h>
#include <linux/spinlock.h>
#include <linux/list.h>
#include <linux/kprobes.h>
#include <linux/mm.h>
#include <linux/vmalloc.h>
#include <asm/alternative.h>
#include <asm/sections.h>
#include <asm/pgtable.h>
#include <asm/mce.h>
#include <asm/nmi.h>

#define MAX_PATCH_LEN (255-1)

#ifdef CONFIG_HOTPLUG_CPU
static int smp_alt_once;

static int __init bootonly(char *str)
{
	smp_alt_once = 1;
	return 1;
}
__setup("smp-alt-boot", bootonly);
#else
#define smp_alt_once 1
#endif

static int debug_alternative;

static int __init debug_alt(char *str)
{
	debug_alternative = 1;
	return 1;
}
__setup("debug-alternative", debug_alt);

static int noreplace_smp;

static int __init setup_noreplace_smp(char *str)
{
	noreplace_smp = 1;
	return 1;
}
__setup("noreplace-smp", setup_noreplace_smp);

#ifdef CONFIG_PARAVIRT
static int noreplace_paravirt = 0;

static int __init setup_noreplace_paravirt(char *str)
{
	noreplace_paravirt = 1;
	return 1;
}
__setup("noreplace-paravirt", setup_noreplace_paravirt);
#endif

#define DPRINTK(fmt, args...) if (debug_alternative) \
	printk(KERN_DEBUG fmt, args)

#ifdef GENERIC_NOP1
/* Use inline assembly to define this because the nops are defined
   as inline assembly strings in the include files and we cannot
   get them easily into strings. */
asm("\t.data\nintelnops: "
	GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
	GENERIC_NOP7 GENERIC_NOP8);
extern unsigned char intelnops[];
static unsigned char *intel_nops[ASM_NOP_MAX+1] = {
	NULL,
	intelnops,
	intelnops + 1,
	intelnops + 1 + 2,
	intelnops + 1 + 2 + 3,
	intelnops + 1 + 2 + 3 + 4,
	intelnops + 1 + 2 + 3 + 4 + 5,
	intelnops + 1 + 2 + 3 + 4 + 5 + 6,
	intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
};
#endif

#ifdef K8_NOP1
asm("\t.data\nk8nops: "
	K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
	K8_NOP7 K8_NOP8);
extern unsigned char k8nops[];
static unsigned char *k8_nops[ASM_NOP_MAX+1] = {
	NULL,
	k8nops,
	k8nops + 1,
	k8nops + 1 + 2,
	k8nops + 1 + 2 + 3,
	k8nops + 1 + 2 + 3 + 4,
	k8nops + 1 + 2 + 3 + 4 + 5,
	k8nops + 1 + 2 + 3 + 4 + 5 + 6,
	k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
};
#endif

#ifdef K7_NOP1
asm("\t.data\nk7nops: "
	K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
	K7_NOP7 K7_NOP8);
extern unsigned char k7nops[];
static unsigned char *k7_nops[ASM_NOP_MAX+1] = {
	NULL,
	k7nops,
	k7nops + 1,
	k7nops + 1 + 2,
	k7nops + 1 + 2 + 3,
	k7nops + 1 + 2 + 3 + 4,
	k7nops + 1 + 2 + 3 + 4 + 5,
	k7nops + 1 + 2 + 3 + 4 + 5 + 6,
	k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
};
#endif

#ifdef CONFIG_X86_64

extern char __vsyscall_0;
static inline unsigned char** find_nop_table(void)
{
	return k8_nops;
}

#else /* CONFIG_X86_64 */

static struct nop {
	int cpuid;
	unsigned char **noptable;
} noptypes[] = {
	{ X86_FEATURE_K8, k8_nops },
	{ X86_FEATURE_K7, k7_nops },
	{ -1, NULL }
};

static unsigned char** find_nop_table(void)
{
	unsigned char **noptable = intel_nops;
	int i;

	for (i = 0; noptypes[i].cpuid >= 0; i++) {
		if (boot_cpu_has(noptypes[i].cpuid)) {
			noptable = noptypes[i].noptable;
			break;
		}
	}
	return noptable;
}

#endif /* CONFIG_X86_64 */

/* Use this to add nops to a buffer, then text_poke the whole buffer. */
static void add_nops(void *insns, unsigned int len)
{
	unsigned char **noptable = find_nop_table();

	while (len > 0) {
		unsigned int noplen = len;
		if (noplen > ASM_NOP_MAX)
			noplen = ASM_NOP_MAX;
		memcpy(insns, noptable[noplen], noplen);
		insns += noplen;
		len -= noplen;
	}
}

extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
extern u8 *__smp_locks[], *__smp_locks_end[];

/* Replace instructions with better alternatives for this CPU type.
   This runs before SMP is initialized to avoid SMP problems with
   self modifying code. This implies that assymetric systems where
   APs have less capabilities than the boot processor are not handled.
   Tough. Make sure you disable such features by hand. */

void apply_alternatives(struct alt_instr *start, struct alt_instr *end)
{
	struct alt_instr *a;
	char insnbuf[MAX_PATCH_LEN];

	DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end);
	for (a = start; a < end; a++) {
		u8 *instr = a->instr;
		BUG_ON(a->replacementlen > a->instrlen);
		BUG_ON(a->instrlen > sizeof(insnbuf));
		if (!boot_cpu_has(a->cpuid))
			continue;
#ifdef CONFIG_X86_64
		/* vsyscall code is not mapped yet. resolve it manually. */
		if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END) {
			instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0));
			DPRINTK("%s: vsyscall fixup: %p => %p\n",
				__FUNCTION__, a->instr, instr);
		}
#endif
		memcpy(insnbuf, a->replacement, a->replacementlen);
		add_nops(insnbuf + a->replacementlen,
			 a->instrlen - a->replacementlen);
		text_poke(instr, insnbuf, a->instrlen);
	}
}

#ifdef CONFIG_SMP

static void alternatives_smp_lock(u8 **start, u8 **end, u8 *text, u8 *text_end)
{
	u8 **ptr;

	for (ptr = start; ptr < end; ptr++) {
		if (*ptr < text)
			continue;
		if (*ptr > text_end)
			continue;
		text_poke(*ptr, ((unsigned char []){0xf0}), 1); /* add lock prefix */
	};
}

static void alternatives_smp_unlock(u8 **start, u8 **end, u8 *text, u8 *text_end)
{
	u8 **ptr;
	char insn[1];

	if (noreplace_smp)
		return;

	add_nops(insn, 1);
	for (ptr = start; ptr < end; ptr++) {
		if (*ptr < text)
			continue;
		if (*ptr > text_end)
			continue;
		text_poke(*ptr, insn, 1);
	};
}

struct smp_alt_module {
	/* what is this ??? */
	struct module	*mod;
	char		*name;

	/* ptrs to lock prefixes */
	u8		**locks;
	u8		**locks_end;

	/* .text segment, needed to avoid patching init code ;) */
	u8		*text;
	u8		*text_end;

	struct list_head next;
};
static LIST_HEAD(smp_alt_modules);
static DEFINE_SPINLOCK(smp_alt);

void alternatives_smp_module_add(struct module *mod, char *name,
				 void *locks, void *locks_end,
				 void *text,  void *text_end)
{
	struct smp_alt_module *smp;
	unsigned long flags;

	if (noreplace_smp)
		return;

	if (smp_alt_once) {
		if (boot_cpu_has(X86_FEATURE_UP))
			alternatives_smp_unlock(locks, locks_end,
						text, text_end);
		return;
	}

	smp = kzalloc(sizeof(*smp), GFP_KERNEL);
	if (NULL == smp)
		return; /* we'll run the (safe but slow) SMP code then ... */

	smp->mod	= mod;
	smp->name	= name;
	smp->locks	= locks;
	smp->locks_end	= locks_end;
	smp->text	= text;
	smp->text_end	= text_end;
	DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
		__FUNCTION__, smp->locks, smp->locks_end,
		smp->text, smp->text_end, smp->name);

	spin_lock_irqsave(&smp_alt, flags);
	list_add_tail(&smp->next, &smp_alt_modules);
	if (boot_cpu_has(X86_FEATURE_UP))
		alternatives_smp_unlock(smp->locks, smp->locks_end,
					smp->text, smp->text_end);
	spin_unlock_irqrestore(&smp_alt, flags);
}

void alternatives_smp_module_del(struct module *mod)
{
	struct smp_alt_module *item;
	unsigned long flags;

	if (smp_alt_once || noreplace_smp)
		return;

	spin_lock_irqsave(&smp_alt, flags);
	list_for_each_entry(item, &smp_alt_modules, next) {
		if (mod != item->mod)
			continue;
		list_del(&item->next);
		spin_unlock_irqrestore(&smp_alt, flags);
		DPRINTK("%s: %s\n", __FUNCTION__, item->name);
		kfree(item);
		return;
	}
	spin_unlock_irqrestore(&smp_alt, flags);
}

void alternatives_smp_switch(int smp)
{
	struct smp_alt_module *mod;
	unsigned long flags;

#ifdef CONFIG_LOCKDEP
	/*
	 * A not yet fixed binutils section handling bug prevents
	 * alternatives-replacement from working reliably, so turn
	 * it off:
	 */
	printk("lockdep: not fixing up alternatives.\n");
	return;
#endif

	if (noreplace_smp || smp_alt_once)
		return;
	BUG_ON(!smp && (num_online_cpus() > 1));

	spin_lock_irqsave(&smp_alt, flags);
	if (smp) {
		printk(KERN_INFO "SMP alternatives: switching to SMP code\n");
		clear_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
		clear_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
		list_for_each_entry(mod, &smp_alt_modules, next)
			alternatives_smp_lock(mod->locks, mod->locks_end,
					      mod->text, mod->text_end);
	} else {
		printk(KERN_INFO "SMP alternatives: switching to UP code\n");
		set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
		set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
		list_for_each_entry(mod, &smp_alt_modules, next)
			alternatives_smp_unlock(mod->locks, mod->locks_end,
						mod->text, mod->text_end);
	}
	spin_unlock_irqrestore(&smp_alt, flags);
}

#endif

#ifdef CONFIG_PARAVIRT
void apply_paravirt(struct paravirt_patch_site *start,
		    struct paravirt_patch_site *end)
{
	struct paravirt_patch_site *p;
	char insnbuf[MAX_PATCH_LEN];

	if (noreplace_paravirt)
		return;

	for (p = start; p < end; p++) {
		unsigned int used;

		BUG_ON(p->len > MAX_PATCH_LEN);
		/* prep the buffer with the original instructions */
		memcpy(insnbuf, p->instr, p->len);
		used = paravirt_ops.patch(p->instrtype, p->clobbers, insnbuf,
					  (unsigned long)p->instr, p->len);

		BUG_ON(used > p->len);

		/* Pad the rest with nops */
		add_nops(insnbuf + used, p->len - used);
		text_poke(p->instr, insnbuf, p->len);
	}
}
extern struct paravirt_patch_site __start_parainstructions[],
	__stop_parainstructions[];
#endif	/* CONFIG_PARAVIRT */

void __init alternative_instructions(void)
{
	unsigned long flags;

	/* The patching is not fully atomic, so try to avoid local interruptions
	   that might execute the to be patched code.
	   Other CPUs are not running. */
	stop_nmi();
#ifdef CONFIG_X86_MCE
	stop_mce();
#endif

	local_irq_save(flags);
	apply_alternatives(__alt_instructions, __alt_instructions_end);

	/* switch to patch-once-at-boottime-only mode and free the
	 * tables in case we know the number of CPUs will never ever
	 * change */
#ifdef CONFIG_HOTPLUG_CPU
	if (num_possible_cpus() < 2)
		smp_alt_once = 1;
#endif

#ifdef CONFIG_SMP
	if (smp_alt_once) {
		if (1 == num_possible_cpus()) {
			printk(KERN_INFO "SMP alternatives: switching to UP code\n");
			set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
			set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
			alternatives_smp_unlock(__smp_locks, __smp_locks_end,
						_text, _etext);
		}
		free_init_pages("SMP alternatives",
				(unsigned long)__smp_locks,
				(unsigned long)__smp_locks_end);
	} else {
		alternatives_smp_module_add(NULL, "core kernel",
					    __smp_locks, __smp_locks_end,
					    _text, _etext);
		alternatives_smp_switch(0);
	}
#endif
 	apply_paravirt(__parainstructions, __parainstructions_end);
	local_irq_restore(flags);

	restart_nmi();
#ifdef CONFIG_X86_MCE
	restart_mce();
#endif
}

/*
 * Warning:
 * When you use this code to patch more than one byte of an instruction
 * you need to make sure that other CPUs cannot execute this code in parallel.
 * Also no thread must be currently preempted in the middle of these instructions.
 * And on the local CPU you need to be protected again NMI or MCE handlers
 * seeing an inconsistent instruction while you patch.
 */
void __kprobes text_poke(void *addr, unsigned char *opcode, int len)
{
	memcpy(addr, opcode, len);
	sync_core();
	/* Not strictly needed, but can speed CPU recovery up. Ignore cross cacheline
	   case. */
	if (cpu_has_clflush)
		asm("clflush (%0) " :: "r" (addr) : "memory");
}
Commit	Line	Data
9a0b5817	1	#include <linux/module.h>
f6a57033	2	#include <linux/sched.h>
9a0b5817 GH	3	#include <linux/spinlock.h>
9a0b5817 GH	4	#include <linux/list.h>
19d36ccd AK	5	#include <linux/kprobes.h>
	6	#include <linux/mm.h>
	7	#include <linux/vmalloc.h>
9a0b5817 GH	8	#include <asm/alternative.h>
9a0b5817 GH	9	#include <asm/sections.h>
19d36ccd	10	#include <asm/pgtable.h>
8f4e956b AK	11	#include <asm/mce.h>
8f4e956b AK	12	#include <asm/nmi.h>
9a0b5817	13
ab144f5e AK	14	#define MAX_PATCH_LEN (255-1)
ab144f5e AK	15
09488165 JB	16	#ifdef CONFIG_HOTPLUG_CPU
09488165 JB	17	static int smp_alt_once;
9a0b5817	18
d167a518 GH	19	static int __init bootonly(char *str)
	20	{
	21	smp_alt_once = 1;
	22	return 1;
	23	}
b7fb4af0	24	__setup("smp-alt-boot", bootonly);
09488165 JB	25	#else
	26	#define smp_alt_once 1
	27	#endif
	28
	29	static int debug_alternative;
b7fb4af0	30
d167a518 GH	31	static int __init debug_alt(char *str)
	32	{
	33	debug_alternative = 1;
	34	return 1;
	35	}
d167a518 GH	36	__setup("debug-alternative", debug_alt);
d167a518 GH	37
09488165 JB	38	static int noreplace_smp;
09488165 JB	39
b7fb4af0 JF	40	static int __init setup_noreplace_smp(char *str)
	41	{
	42	noreplace_smp = 1;
	43	return 1;
	44	}
	45	__setup("noreplace-smp", setup_noreplace_smp);
	46
959b4fdf JF	47	#ifdef CONFIG_PARAVIRT
	48	static int noreplace_paravirt = 0;
	49
	50	static int __init setup_noreplace_paravirt(char *str)
	51	{
	52	noreplace_paravirt = 1;
	53	return 1;
	54	}
	55	__setup("noreplace-paravirt", setup_noreplace_paravirt);
	56	#endif
b7fb4af0	57
d167a518 GH	58	#define DPRINTK(fmt, args...) if (debug_alternative) \
	59	printk(KERN_DEBUG fmt, args)
	60
	61	#ifdef GENERIC_NOP1
9a0b5817 GH	62	/* Use inline assembly to define this because the nops are defined
	63	as inline assembly strings in the include files and we cannot
	64	get them easily into strings. */
	65	asm("\t.data\nintelnops: "
	66	GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
	67	GENERIC_NOP7 GENERIC_NOP8);
d167a518	68	extern unsigned char intelnops[];
9a0b5817 GH	69	static unsigned char *intel_nops[ASM_NOP_MAX+1] = {
	70	NULL,
	71	intelnops,
	72	intelnops + 1,
	73	intelnops + 1 + 2,
	74	intelnops + 1 + 2 + 3,
	75	intelnops + 1 + 2 + 3 + 4,
	76	intelnops + 1 + 2 + 3 + 4 + 5,
	77	intelnops + 1 + 2 + 3 + 4 + 5 + 6,
	78	intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
	79	};
d167a518 GH	80	#endif
	81
	82	#ifdef K8_NOP1
	83	asm("\t.data\nk8nops: "
	84	K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
	85	K8_NOP7 K8_NOP8);
	86	extern unsigned char k8nops[];
9a0b5817 GH	87	static unsigned char *k8_nops[ASM_NOP_MAX+1] = {
	88	NULL,
	89	k8nops,
	90	k8nops + 1,
	91	k8nops + 1 + 2,
	92	k8nops + 1 + 2 + 3,
	93	k8nops + 1 + 2 + 3 + 4,
	94	k8nops + 1 + 2 + 3 + 4 + 5,
	95	k8nops + 1 + 2 + 3 + 4 + 5 + 6,
	96	k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
	97	};
d167a518 GH	98	#endif
	99
	100	#ifdef K7_NOP1
	101	asm("\t.data\nk7nops: "
	102	K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
	103	K7_NOP7 K7_NOP8);
	104	extern unsigned char k7nops[];
9a0b5817 GH	105	static unsigned char *k7_nops[ASM_NOP_MAX+1] = {
	106	NULL,
	107	k7nops,
	108	k7nops + 1,
	109	k7nops + 1 + 2,
	110	k7nops + 1 + 2 + 3,
	111	k7nops + 1 + 2 + 3 + 4,
	112	k7nops + 1 + 2 + 3 + 4 + 5,
	113	k7nops + 1 + 2 + 3 + 4 + 5 + 6,
	114	k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
	115	};
d167a518 GH	116	#endif
	117
	118	#ifdef CONFIG_X86_64
	119
	120	extern char __vsyscall_0;
	121	static inline unsigned char** find_nop_table(void)
	122	{
	123	return k8_nops;
	124	}
	125
	126	#else /* CONFIG_X86_64 */
	127
9a0b5817 GH	128	static struct nop {
	129	int cpuid;
	130	unsigned char **noptable;
	131	} noptypes[] = {
	132	{ X86_FEATURE_K8, k8_nops },
	133	{ X86_FEATURE_K7, k7_nops },
	134	{ -1, NULL }
	135	};
	136
9a0b5817 GH	137	static unsigned char** find_nop_table(void)
	138	{
	139	unsigned char **noptable = intel_nops;
	140	int i;
	141
	142	for (i = 0; noptypes[i].cpuid >= 0; i++) {
	143	if (boot_cpu_has(noptypes[i].cpuid)) {
	144	noptable = noptypes[i].noptable;
	145	break;
	146	}
	147	}
	148	return noptable;
	149	}
	150
d167a518 GH	151	#endif /* CONFIG_X86_64 */
d167a518 GH	152
ab144f5e AK	153	/* Use this to add nops to a buffer, then text_poke the whole buffer. */
ab144f5e AK	154	static void add_nops(void *insns, unsigned int len)
139ec7c4 RR	155	{
	156	unsigned char **noptable = find_nop_table();
	157
	158	while (len > 0) {
	159	unsigned int noplen = len;
	160	if (noplen > ASM_NOP_MAX)
	161	noplen = ASM_NOP_MAX;
ab144f5e	162	memcpy(insns, noptable[noplen], noplen);
139ec7c4 RR	163	insns += noplen;
	164	len -= noplen;
	165	}
	166	}
	167
d167a518	168	extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
d167a518 GH	169	extern u8 __smp_locks[], __smp_locks_end[];
d167a518 GH	170
9a0b5817 GH	171	/* Replace instructions with better alternatives for this CPU type.
	172	This runs before SMP is initialized to avoid SMP problems with
	173	self modifying code. This implies that assymetric systems where
	174	APs have less capabilities than the boot processor are not handled.
	175	Tough. Make sure you disable such features by hand. */
	176
	177	void apply_alternatives(struct alt_instr start, struct alt_instr end)
	178	{
9a0b5817	179	struct alt_instr *a;
ab144f5e	180	char insnbuf[MAX_PATCH_LEN];
9a0b5817 GH	181
	182	DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end);
	183	for (a = start; a < end; a++) {
ab144f5e	184	u8 *instr = a->instr;
9a0b5817	185	BUG_ON(a->replacementlen > a->instrlen);
ab144f5e	186	BUG_ON(a->instrlen > sizeof(insnbuf));
9a0b5817 GH	187	if (!boot_cpu_has(a->cpuid))
9a0b5817 GH	188	continue;
d167a518 GH	189	#ifdef CONFIG_X86_64
	190	/* vsyscall code is not mapped yet. resolve it manually. */
	191	if (instr >= (u8 )VSYSCALL_START && instr < (u8)VSYSCALL_END) {
	192	instr = __va(instr - (u8)VSYSCALL_START + (u8)__pa_symbol(&__vsyscall_0));
	193	DPRINTK("%s: vsyscall fixup: %p => %p\n",
	194	__FUNCTION__, a->instr, instr);
	195	}
	196	#endif
ab144f5e AK	197	memcpy(insnbuf, a->replacement, a->replacementlen);
	198	add_nops(insnbuf + a->replacementlen,
	199	a->instrlen - a->replacementlen);
	200	text_poke(instr, insnbuf, a->instrlen);
9a0b5817 GH	201	}
	202	}
	203
8ec4d41f GH	204	#ifdef CONFIG_SMP
8ec4d41f GH	205
9a0b5817 GH	206	static void alternatives_smp_lock(u8 start, u8 end, u8 text, u8 text_end)
	207	{
	208	u8 **ptr;
	209
	210	for (ptr = start; ptr < end; ptr++) {
	211	if (*ptr < text)
	212	continue;
	213	if (*ptr > text_end)
	214	continue;
19d36ccd	215	text_poke(ptr, ((unsigned char []){0xf0}), 1); / add lock prefix */
9a0b5817 GH	216	};
	217	}
	218
	219	static void alternatives_smp_unlock(u8 start, u8 end, u8 text, u8 text_end)
	220	{
9a0b5817	221	u8 **ptr;
ab144f5e	222	char insn[1];
9a0b5817	223
b7fb4af0 JF	224	if (noreplace_smp)
	225	return;
	226
ab144f5e	227	add_nops(insn, 1);
9a0b5817 GH	228	for (ptr = start; ptr < end; ptr++) {
	229	if (*ptr < text)
	230	continue;
	231	if (*ptr > text_end)
	232	continue;
ab144f5e	233	text_poke(*ptr, insn, 1);
9a0b5817 GH	234	};
	235	}
	236
	237	struct smp_alt_module {
	238	/* what is this ??? */
	239	struct module *mod;
	240	char *name;
	241
	242	/* ptrs to lock prefixes */
	243	u8 **locks;
	244	u8 **locks_end;
	245
	246	/* .text segment, needed to avoid patching init code ;) */
	247	u8 *text;
	248	u8 *text_end;
	249
	250	struct list_head next;
	251	};
	252	static LIST_HEAD(smp_alt_modules);
	253	static DEFINE_SPINLOCK(smp_alt);
	254
9a0b5817 GH	255	void alternatives_smp_module_add(struct module mod, char name,
	256	void locks, void locks_end,
	257	void text, void text_end)
	258	{
	259	struct smp_alt_module *smp;
	260	unsigned long flags;
	261
b7fb4af0 JF	262	if (noreplace_smp)
	263	return;
	264
9a0b5817 GH	265	if (smp_alt_once) {
	266	if (boot_cpu_has(X86_FEATURE_UP))
	267	alternatives_smp_unlock(locks, locks_end,
	268	text, text_end);
	269	return;
	270	}
	271
	272	smp = kzalloc(sizeof(*smp), GFP_KERNEL);
	273	if (NULL == smp)
	274	return; /* we'll run the (safe but slow) SMP code then ... */
	275
	276	smp->mod = mod;
	277	smp->name = name;
	278	smp->locks = locks;
	279	smp->locks_end = locks_end;
	280	smp->text = text;
	281	smp->text_end = text_end;
	282	DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
	283	__FUNCTION__, smp->locks, smp->locks_end,
	284	smp->text, smp->text_end, smp->name);
	285
	286	spin_lock_irqsave(&smp_alt, flags);
	287	list_add_tail(&smp->next, &smp_alt_modules);
	288	if (boot_cpu_has(X86_FEATURE_UP))
	289	alternatives_smp_unlock(smp->locks, smp->locks_end,
	290	smp->text, smp->text_end);
	291	spin_unlock_irqrestore(&smp_alt, flags);
	292	}
	293
	294	void alternatives_smp_module_del(struct module *mod)
	295	{
	296	struct smp_alt_module *item;
	297	unsigned long flags;
	298
b7fb4af0	299	if (smp_alt_once \|\| noreplace_smp)
9a0b5817 GH	300	return;
	301
	302	spin_lock_irqsave(&smp_alt, flags);
	303	list_for_each_entry(item, &smp_alt_modules, next) {
	304	if (mod != item->mod)
	305	continue;
	306	list_del(&item->next);
	307	spin_unlock_irqrestore(&smp_alt, flags);
	308	DPRINTK("%s: %s\n", __FUNCTION__, item->name);
	309	kfree(item);
	310	return;
	311	}
	312	spin_unlock_irqrestore(&smp_alt, flags);
	313	}
	314
	315	void alternatives_smp_switch(int smp)
	316	{
	317	struct smp_alt_module *mod;
	318	unsigned long flags;
	319
3047e99e IM	320	#ifdef CONFIG_LOCKDEP
	321	/*
	322	* A not yet fixed binutils section handling bug prevents
	323	* alternatives-replacement from working reliably, so turn
	324	* it off:
	325	*/
	326	printk("lockdep: not fixing up alternatives.\n");
	327	return;
	328	#endif
	329
b7fb4af0	330	if (noreplace_smp \|\| smp_alt_once)
9a0b5817 GH	331	return;
	332	BUG_ON(!smp && (num_online_cpus() > 1));
	333
	334	spin_lock_irqsave(&smp_alt, flags);
	335	if (smp) {
	336	printk(KERN_INFO "SMP alternatives: switching to SMP code\n");
	337	clear_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
	338	clear_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
9a0b5817 GH	339	list_for_each_entry(mod, &smp_alt_modules, next)
	340	alternatives_smp_lock(mod->locks, mod->locks_end,
	341	mod->text, mod->text_end);
	342	} else {
	343	printk(KERN_INFO "SMP alternatives: switching to UP code\n");
	344	set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
	345	set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
9a0b5817 GH	346	list_for_each_entry(mod, &smp_alt_modules, next)
	347	alternatives_smp_unlock(mod->locks, mod->locks_end,
	348	mod->text, mod->text_end);
	349	}
	350	spin_unlock_irqrestore(&smp_alt, flags);
	351	}
	352
8ec4d41f GH	353	#endif
8ec4d41f GH	354
139ec7c4	355	#ifdef CONFIG_PARAVIRT
98de032b JF	356	void apply_paravirt(struct paravirt_patch_site *start,
98de032b JF	357	struct paravirt_patch_site *end)
139ec7c4	358	{
98de032b	359	struct paravirt_patch_site *p;
ab144f5e	360	char insnbuf[MAX_PATCH_LEN];
139ec7c4	361
959b4fdf JF	362	if (noreplace_paravirt)
	363	return;
	364
139ec7c4 RR	365	for (p = start; p < end; p++) {
	366	unsigned int used;
	367
ab144f5e	368	BUG_ON(p->len > MAX_PATCH_LEN);
d34fda4a CW	369	/* prep the buffer with the original instructions */
d34fda4a CW	370	memcpy(insnbuf, p->instr, p->len);
ab144f5e AK	371	used = paravirt_ops.patch(p->instrtype, p->clobbers, insnbuf,
ab144f5e AK	372	(unsigned long)p->instr, p->len);
7f63c41c	373
63f70270 JF	374	BUG_ON(used > p->len);
63f70270 JF	375
139ec7c4	376	/* Pad the rest with nops */
ab144f5e AK	377	add_nops(insnbuf + used, p->len - used);
ab144f5e AK	378	text_poke(p->instr, insnbuf, p->len);
139ec7c4	379	}
139ec7c4	380	}
98de032b	381	extern struct paravirt_patch_site __start_parainstructions[],
139ec7c4 RR	382	__stop_parainstructions[];
	383	#endif /* CONFIG_PARAVIRT */
	384
9a0b5817 GH	385	void __init alternative_instructions(void)
9a0b5817 GH	386	{
e51959fa	387	unsigned long flags;
e51959fa	388
8f4e956b AK	389	/* The patching is not fully atomic, so try to avoid local interruptions
	390	that might execute the to be patched code.
	391	Other CPUs are not running. */
	392	stop_nmi();
d2d0251f	393	#ifdef CONFIG_X86_MCE
8f4e956b AK	394	stop_mce();
	395	#endif
	396
e51959fa	397	local_irq_save(flags);
9a0b5817 GH	398	apply_alternatives(__alt_instructions, __alt_instructions_end);
	399
	400	/* switch to patch-once-at-boottime-only mode and free the
	401	* tables in case we know the number of CPUs will never ever
	402	* change */
	403	#ifdef CONFIG_HOTPLUG_CPU
	404	if (num_possible_cpus() < 2)
	405	smp_alt_once = 1;
9a0b5817 GH	406	#endif
9a0b5817 GH	407
8ec4d41f	408	#ifdef CONFIG_SMP
9a0b5817 GH	409	if (smp_alt_once) {
	410	if (1 == num_possible_cpus()) {
	411	printk(KERN_INFO "SMP alternatives: switching to UP code\n");
	412	set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
	413	set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
9a0b5817 GH	414	alternatives_smp_unlock(__smp_locks, __smp_locks_end,
	415	_text, _etext);
	416	}
	417	free_init_pages("SMP alternatives",
e3ebadd9 LT	418	(unsigned long)__smp_locks,
e3ebadd9 LT	419	(unsigned long)__smp_locks_end);
9a0b5817	420	} else {
9a0b5817 GH	421	alternatives_smp_module_add(NULL, "core kernel",
	422	__smp_locks, __smp_locks_end,
	423	_text, _etext);
	424	alternatives_smp_switch(0);
	425	}
8ec4d41f	426	#endif
441d40dc	427	apply_paravirt(__parainstructions, __parainstructions_end);
e51959fa	428	local_irq_restore(flags);
8f4e956b AK	429
8f4e956b AK	430	restart_nmi();
d2d0251f	431	#ifdef CONFIG_X86_MCE
8f4e956b AK	432	restart_mce();
8f4e956b AK	433	#endif
9a0b5817	434	}
19d36ccd AK	435
	436	/*
	437	* Warning:
	438	* When you use this code to patch more than one byte of an instruction
	439	* you need to make sure that other CPUs cannot execute this code in parallel.
	440	* Also no thread must be currently preempted in the middle of these instructions.
	441	* And on the local CPU you need to be protected again NMI or MCE handlers
	442	* seeing an inconsistent instruction while you patch.
	443	*/
602033ed	444	void __kprobes text_poke(void addr, unsigned char opcode, int len)
19d36ccd	445	{
19d36ccd AK	446	memcpy(addr, opcode, len);
	447	sync_core();
	448	/* Not strictly needed, but can speed CPU recovery up. Ignore cross cacheline
	449	case. */
	450	if (cpu_has_clflush)
602033ed	451	asm("clflush (%0) " :: "r" (addr) : "memory");
19d36ccd	452	}