[net-next-2.6.git] / arch / i386 / kernel / alternative.c

#include <linux/module.h>
#include <linux/sched.h>
#include <linux/spinlock.h>
#include <linux/list.h>
#include <linux/kprobes.h>
#include <linux/mm.h>
#include <linux/vmalloc.h>
#include <asm/alternative.h>
#include <asm/sections.h>
#include <asm/pgtable.h>
#include <asm/mce.h>
#include <asm/nmi.h>

#define MAX_PATCH_LEN (255-1)

#ifdef CONFIG_HOTPLUG_CPU
static int smp_alt_once;

static int __init bootonly(char *str)
{
	smp_alt_once = 1;
	return 1;
}
__setup("smp-alt-boot", bootonly);
#else
#define smp_alt_once 1
#endif

static int debug_alternative;

static int __init debug_alt(char *str)
{
	debug_alternative = 1;
	return 1;
}
__setup("debug-alternative", debug_alt);

static int noreplace_smp;

static int __init setup_noreplace_smp(char *str)
{
	noreplace_smp = 1;
	return 1;
}
__setup("noreplace-smp", setup_noreplace_smp);

#ifdef CONFIG_PARAVIRT
static int noreplace_paravirt = 0;

static int __init setup_noreplace_paravirt(char *str)
{
	noreplace_paravirt = 1;
	return 1;
}
__setup("noreplace-paravirt", setup_noreplace_paravirt);
#endif

#define DPRINTK(fmt, args...) if (debug_alternative) \
	printk(KERN_DEBUG fmt, args)

#ifdef GENERIC_NOP1
/* Use inline assembly to define this because the nops are defined
   as inline assembly strings in the include files and we cannot
   get them easily into strings. */
asm("\t.data\nintelnops: "
	GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
	GENERIC_NOP7 GENERIC_NOP8);
extern unsigned char intelnops[];
static unsigned char *intel_nops[ASM_NOP_MAX+1] = {
	NULL,
	intelnops,
	intelnops + 1,
	intelnops + 1 + 2,
	intelnops + 1 + 2 + 3,
	intelnops + 1 + 2 + 3 + 4,
	intelnops + 1 + 2 + 3 + 4 + 5,
	intelnops + 1 + 2 + 3 + 4 + 5 + 6,
	intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
};
#endif

#ifdef K8_NOP1
asm("\t.data\nk8nops: "
	K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
	K8_NOP7 K8_NOP8);
extern unsigned char k8nops[];
static unsigned char *k8_nops[ASM_NOP_MAX+1] = {
	NULL,
	k8nops,
	k8nops + 1,
	k8nops + 1 + 2,
	k8nops + 1 + 2 + 3,
	k8nops + 1 + 2 + 3 + 4,
	k8nops + 1 + 2 + 3 + 4 + 5,
	k8nops + 1 + 2 + 3 + 4 + 5 + 6,
	k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
};
#endif

#ifdef K7_NOP1
asm("\t.data\nk7nops: "
	K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
	K7_NOP7 K7_NOP8);
extern unsigned char k7nops[];
static unsigned char *k7_nops[ASM_NOP_MAX+1] = {
	NULL,
	k7nops,
	k7nops + 1,
	k7nops + 1 + 2,
	k7nops + 1 + 2 + 3,
	k7nops + 1 + 2 + 3 + 4,
	k7nops + 1 + 2 + 3 + 4 + 5,
	k7nops + 1 + 2 + 3 + 4 + 5 + 6,
	k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
};
#endif

#ifdef CONFIG_X86_64

extern char __vsyscall_0;
static inline unsigned char** find_nop_table(void)
{
	return k8_nops;
}

#else /* CONFIG_X86_64 */

static struct nop {
	int cpuid;
	unsigned char **noptable;
} noptypes[] = {
	{ X86_FEATURE_K8, k8_nops },
	{ X86_FEATURE_K7, k7_nops },
	{ -1, NULL }
};

static unsigned char** find_nop_table(void)
{
	unsigned char **noptable = intel_nops;
	int i;

	for (i = 0; noptypes[i].cpuid >= 0; i++) {
		if (boot_cpu_has(noptypes[i].cpuid)) {
			noptable = noptypes[i].noptable;
			break;
		}
	}
	return noptable;
}

#endif /* CONFIG_X86_64 */

/* Use this to add nops to a buffer, then text_poke the whole buffer. */
static void add_nops(void *insns, unsigned int len)
{
	unsigned char **noptable = find_nop_table();

	while (len > 0) {
		unsigned int noplen = len;
		if (noplen > ASM_NOP_MAX)
			noplen = ASM_NOP_MAX;
		memcpy(insns, noptable[noplen], noplen);
		insns += noplen;
		len -= noplen;
	}
}

extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
extern u8 *__smp_locks[], *__smp_locks_end[];

/* Replace instructions with better alternatives for this CPU type.
   This runs before SMP is initialized to avoid SMP problems with
   self modifying code. This implies that assymetric systems where
   APs have less capabilities than the boot processor are not handled.
   Tough. Make sure you disable such features by hand. */

void apply_alternatives(struct alt_instr *start, struct alt_instr *end)
{
	struct alt_instr *a;
	char insnbuf[MAX_PATCH_LEN];

	DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end);
	for (a = start; a < end; a++) {
		u8 *instr = a->instr;
		BUG_ON(a->replacementlen > a->instrlen);
		BUG_ON(a->instrlen > sizeof(insnbuf));
		if (!boot_cpu_has(a->cpuid))
			continue;
#ifdef CONFIG_X86_64
		/* vsyscall code is not mapped yet. resolve it manually. */
		if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END) {
			instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0));
			DPRINTK("%s: vsyscall fixup: %p => %p\n",
				__FUNCTION__, a->instr, instr);
		}
#endif
		memcpy(insnbuf, a->replacement, a->replacementlen);
		add_nops(insnbuf + a->replacementlen,
			 a->instrlen - a->replacementlen);
		text_poke(instr, insnbuf, a->instrlen);
	}
}

#ifdef CONFIG_SMP

static void alternatives_smp_lock(u8 **start, u8 **end, u8 *text, u8 *text_end)
{
	u8 **ptr;

	for (ptr = start; ptr < end; ptr++) {
		if (*ptr < text)
			continue;
		if (*ptr > text_end)
			continue;
		text_poke(*ptr, ((unsigned char []){0xf0}), 1); /* add lock prefix */
	};
}

static void alternatives_smp_unlock(u8 **start, u8 **end, u8 *text, u8 *text_end)
{
	u8 **ptr;
	char insn[1];

	if (noreplace_smp)
		return;

	add_nops(insn, 1);
	for (ptr = start; ptr < end; ptr++) {
		if (*ptr < text)
			continue;
		if (*ptr > text_end)
			continue;
		text_poke(*ptr, insn, 1);
	};
}

struct smp_alt_module {
	/* what is this ??? */
	struct module	*mod;
	char		*name;

	/* ptrs to lock prefixes */
	u8		**locks;
	u8		**locks_end;

	/* .text segment, needed to avoid patching init code ;) */
	u8		*text;
	u8		*text_end;

	struct list_head next;
};
static LIST_HEAD(smp_alt_modules);
static DEFINE_SPINLOCK(smp_alt);

void alternatives_smp_module_add(struct module *mod, char *name,
				 void *locks, void *locks_end,
				 void *text,  void *text_end)
{
	struct smp_alt_module *smp;
	unsigned long flags;

	if (noreplace_smp)
		return;

	if (smp_alt_once) {
		if (boot_cpu_has(X86_FEATURE_UP))
			alternatives_smp_unlock(locks, locks_end,
						text, text_end);
		return;
	}

	smp = kzalloc(sizeof(*smp), GFP_KERNEL);
	if (NULL == smp)
		return; /* we'll run the (safe but slow) SMP code then ... */

	smp->mod	= mod;
	smp->name	= name;
	smp->locks	= locks;
	smp->locks_end	= locks_end;
	smp->text	= text;
	smp->text_end	= text_end;
	DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
		__FUNCTION__, smp->locks, smp->locks_end,
		smp->text, smp->text_end, smp->name);

	spin_lock_irqsave(&smp_alt, flags);
	list_add_tail(&smp->next, &smp_alt_modules);
	if (boot_cpu_has(X86_FEATURE_UP))
		alternatives_smp_unlock(smp->locks, smp->locks_end,
					smp->text, smp->text_end);
	spin_unlock_irqrestore(&smp_alt, flags);
}

void alternatives_smp_module_del(struct module *mod)
{
	struct smp_alt_module *item;
	unsigned long flags;

	if (smp_alt_once || noreplace_smp)
		return;

	spin_lock_irqsave(&smp_alt, flags);
	list_for_each_entry(item, &smp_alt_modules, next) {
		if (mod != item->mod)
			continue;
		list_del(&item->next);
		spin_unlock_irqrestore(&smp_alt, flags);
		DPRINTK("%s: %s\n", __FUNCTION__, item->name);
		kfree(item);
		return;
	}
	spin_unlock_irqrestore(&smp_alt, flags);
}

void alternatives_smp_switch(int smp)
{
	struct smp_alt_module *mod;
	unsigned long flags;

#ifdef CONFIG_LOCKDEP
	/*
	 * A not yet fixed binutils section handling bug prevents
	 * alternatives-replacement from working reliably, so turn
	 * it off:
	 */
	printk("lockdep: not fixing up alternatives.\n");
	return;
#endif

	if (noreplace_smp || smp_alt_once)
		return;
	BUG_ON(!smp && (num_online_cpus() > 1));

	spin_lock_irqsave(&smp_alt, flags);
	if (smp) {
		printk(KERN_INFO "SMP alternatives: switching to SMP code\n");
		clear_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
		clear_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
		list_for_each_entry(mod, &smp_alt_modules, next)
			alternatives_smp_lock(mod->locks, mod->locks_end,
					      mod->text, mod->text_end);
	} else {
		printk(KERN_INFO "SMP alternatives: switching to UP code\n");
		set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
		set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
		list_for_each_entry(mod, &smp_alt_modules, next)
			alternatives_smp_unlock(mod->locks, mod->locks_end,
						mod->text, mod->text_end);
	}
	spin_unlock_irqrestore(&smp_alt, flags);
}

#endif

#ifdef CONFIG_PARAVIRT
void apply_paravirt(struct paravirt_patch_site *start,
		    struct paravirt_patch_site *end)
{
	struct paravirt_patch_site *p;
	char insnbuf[MAX_PATCH_LEN];

	if (noreplace_paravirt)
		return;

	for (p = start; p < end; p++) {
		unsigned int used;

		BUG_ON(p->len > MAX_PATCH_LEN);
		/* prep the buffer with the original instructions */
		memcpy(insnbuf, p->instr, p->len);
		used = paravirt_ops.patch(p->instrtype, p->clobbers, insnbuf,
					  (unsigned long)p->instr, p->len);

		BUG_ON(used > p->len);

		/* Pad the rest with nops */
		add_nops(insnbuf + used, p->len - used);
		text_poke(p->instr, insnbuf, p->len);
	}
}
extern struct paravirt_patch_site __start_parainstructions[],
	__stop_parainstructions[];
#endif	/* CONFIG_PARAVIRT */

void __init alternative_instructions(void)
{
	unsigned long flags;

	/* The patching is not fully atomic, so try to avoid local interruptions
	   that might execute the to be patched code.
	   Other CPUs are not running. */
	stop_nmi();
#ifdef CONFIG_X86_MCE
	stop_mce();
#endif

	local_irq_save(flags);
	apply_alternatives(__alt_instructions, __alt_instructions_end);

	/* switch to patch-once-at-boottime-only mode and free the
	 * tables in case we know the number of CPUs will never ever
	 * change */
#ifdef CONFIG_HOTPLUG_CPU
	if (num_possible_cpus() < 2)
		smp_alt_once = 1;
#endif

#ifdef CONFIG_SMP
	if (smp_alt_once) {
		if (1 == num_possible_cpus()) {
			printk(KERN_INFO "SMP alternatives: switching to UP code\n");
			set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
			set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
			alternatives_smp_unlock(__smp_locks, __smp_locks_end,
						_text, _etext);
		}
		free_init_pages("SMP alternatives",
				(unsigned long)__smp_locks,
				(unsigned long)__smp_locks_end);
	} else {
		alternatives_smp_module_add(NULL, "core kernel",
					    __smp_locks, __smp_locks_end,
					    _text, _etext);
		alternatives_smp_switch(0);
	}
#endif
 	apply_paravirt(__parainstructions, __parainstructions_end);
	local_irq_restore(flags);

	restart_nmi();
#ifdef CONFIG_X86_MCE
	restart_mce();
#endif
}

/*
 * Warning:
 * When you use this code to patch more than one byte of an instruction
 * you need to make sure that other CPUs cannot execute this code in parallel.
 * Also no thread must be currently preempted in the middle of these instructions.
 * And on the local CPU you need to be protected again NMI or MCE handlers
 * seeing an inconsistent instruction while you patch.
 */
void __kprobes text_poke(void *addr, unsigned char *opcode, int len)
{
	memcpy(addr, opcode, len);
	sync_core();
	/* Not strictly needed, but can speed CPU recovery up. Ignore cross cacheline
	   case. */
	if (cpu_has_clflush)
		asm("clflush (%0) " :: "r" (addr) : "memory");
}
Commit	Line	Data
	1	#include <linux/module.h>
	2	#include <linux/sched.h>
	3	#include <linux/spinlock.h>
	4	#include <linux/list.h>
	5	#include <linux/kprobes.h>
	6	#include <linux/mm.h>
	7	#include <linux/vmalloc.h>
	8	#include <asm/alternative.h>
	9	#include <asm/sections.h>
	10	#include <asm/pgtable.h>
	11	#include <asm/mce.h>
	12	#include <asm/nmi.h>
	13
	14	#define MAX_PATCH_LEN (255-1)
	15
	16	#ifdef CONFIG_HOTPLUG_CPU
	17	static int smp_alt_once;
	18
	19	static int __init bootonly(char *str)
	20	{
	21	smp_alt_once = 1;
	22	return 1;
	23	}
	24	__setup("smp-alt-boot", bootonly);
	25	#else
	26	#define smp_alt_once 1
	27	#endif
	28
	29	static int debug_alternative;
	30
	31	static int __init debug_alt(char *str)
	32	{
	33	debug_alternative = 1;
	34	return 1;
	35	}
	36	__setup("debug-alternative", debug_alt);
	37
	38	static int noreplace_smp;
	39
	40	static int __init setup_noreplace_smp(char *str)
	41	{
	42	noreplace_smp = 1;
	43	return 1;
	44	}
	45	__setup("noreplace-smp", setup_noreplace_smp);
	46
	47	#ifdef CONFIG_PARAVIRT
	48	static int noreplace_paravirt = 0;
	49
	50	static int __init setup_noreplace_paravirt(char *str)
	51	{
	52	noreplace_paravirt = 1;
	53	return 1;
	54	}
	55	__setup("noreplace-paravirt", setup_noreplace_paravirt);
	56	#endif
	57
	58	#define DPRINTK(fmt, args...) if (debug_alternative) \
	59	printk(KERN_DEBUG fmt, args)
	60
	61	#ifdef GENERIC_NOP1
	62	/* Use inline assembly to define this because the nops are defined
	63	as inline assembly strings in the include files and we cannot
	64	get them easily into strings. */
	65	asm("\t.data\nintelnops: "
	66	GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
	67	GENERIC_NOP7 GENERIC_NOP8);
	68	extern unsigned char intelnops[];
	69	static unsigned char *intel_nops[ASM_NOP_MAX+1] = {
	70	NULL,
	71	intelnops,
	72	intelnops + 1,
	73	intelnops + 1 + 2,
	74	intelnops + 1 + 2 + 3,
	75	intelnops + 1 + 2 + 3 + 4,
	76	intelnops + 1 + 2 + 3 + 4 + 5,
	77	intelnops + 1 + 2 + 3 + 4 + 5 + 6,
	78	intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
	79	};
	80	#endif
	81
	82	#ifdef K8_NOP1
	83	asm("\t.data\nk8nops: "
	84	K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
	85	K8_NOP7 K8_NOP8);
	86	extern unsigned char k8nops[];
	87	static unsigned char *k8_nops[ASM_NOP_MAX+1] = {
	88	NULL,
	89	k8nops,
	90	k8nops + 1,
	91	k8nops + 1 + 2,
	92	k8nops + 1 + 2 + 3,
	93	k8nops + 1 + 2 + 3 + 4,
	94	k8nops + 1 + 2 + 3 + 4 + 5,
	95	k8nops + 1 + 2 + 3 + 4 + 5 + 6,
	96	k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
	97	};
	98	#endif
	99
	100	#ifdef K7_NOP1
	101	asm("\t.data\nk7nops: "
	102	K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
	103	K7_NOP7 K7_NOP8);
	104	extern unsigned char k7nops[];
	105	static unsigned char *k7_nops[ASM_NOP_MAX+1] = {
	106	NULL,
	107	k7nops,
	108	k7nops + 1,
	109	k7nops + 1 + 2,
	110	k7nops + 1 + 2 + 3,
	111	k7nops + 1 + 2 + 3 + 4,
	112	k7nops + 1 + 2 + 3 + 4 + 5,
	113	k7nops + 1 + 2 + 3 + 4 + 5 + 6,
	114	k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
	115	};
	116	#endif
	117
	118	#ifdef CONFIG_X86_64
	119
	120	extern char __vsyscall_0;
	121	static inline unsigned char** find_nop_table(void)
	122	{
	123	return k8_nops;
	124	}
	125
	126	#else /* CONFIG_X86_64 */
	127
	128	static struct nop {
	129	int cpuid;
	130	unsigned char **noptable;
	131	} noptypes[] = {
	132	{ X86_FEATURE_K8, k8_nops },
	133	{ X86_FEATURE_K7, k7_nops },
	134	{ -1, NULL }
	135	};
	136
	137	static unsigned char** find_nop_table(void)
	138	{
	139	unsigned char **noptable = intel_nops;
	140	int i;
	141
	142	for (i = 0; noptypes[i].cpuid >= 0; i++) {
	143	if (boot_cpu_has(noptypes[i].cpuid)) {
	144	noptable = noptypes[i].noptable;
	145	break;
	146	}
	147	}
	148	return noptable;
	149	}
	150
	151	#endif /* CONFIG_X86_64 */
	152
	153	/* Use this to add nops to a buffer, then text_poke the whole buffer. */
	154	static void add_nops(void *insns, unsigned int len)
	155	{
	156	unsigned char **noptable = find_nop_table();
	157
	158	while (len > 0) {
	159	unsigned int noplen = len;
	160	if (noplen > ASM_NOP_MAX)
	161	noplen = ASM_NOP_MAX;
	162	memcpy(insns, noptable[noplen], noplen);
	163	insns += noplen;
	164	len -= noplen;
	165	}
	166	}
	167
	168	extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
	169	extern u8 __smp_locks[], __smp_locks_end[];
	170
	171	/* Replace instructions with better alternatives for this CPU type.
	172	This runs before SMP is initialized to avoid SMP problems with
	173	self modifying code. This implies that assymetric systems where
	174	APs have less capabilities than the boot processor are not handled.
	175	Tough. Make sure you disable such features by hand. */
	176
	177	void apply_alternatives(struct alt_instr start, struct alt_instr end)
	178	{
	179	struct alt_instr *a;
	180	char insnbuf[MAX_PATCH_LEN];
	181
	182	DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end);
	183	for (a = start; a < end; a++) {
	184	u8 *instr = a->instr;
	185	BUG_ON(a->replacementlen > a->instrlen);
	186	BUG_ON(a->instrlen > sizeof(insnbuf));
	187	if (!boot_cpu_has(a->cpuid))
	188	continue;
	189	#ifdef CONFIG_X86_64
	190	/* vsyscall code is not mapped yet. resolve it manually. */
	191	if (instr >= (u8 )VSYSCALL_START && instr < (u8)VSYSCALL_END) {
	192	instr = __va(instr - (u8)VSYSCALL_START + (u8)__pa_symbol(&__vsyscall_0));
	193	DPRINTK("%s: vsyscall fixup: %p => %p\n",
	194	__FUNCTION__, a->instr, instr);
	195	}
	196	#endif
	197	memcpy(insnbuf, a->replacement, a->replacementlen);
	198	add_nops(insnbuf + a->replacementlen,
	199	a->instrlen - a->replacementlen);
	200	text_poke(instr, insnbuf, a->instrlen);
	201	}
	202	}
	203
	204	#ifdef CONFIG_SMP
	205
	206	static void alternatives_smp_lock(u8 start, u8 end, u8 text, u8 text_end)
	207	{
	208	u8 **ptr;
	209
	210	for (ptr = start; ptr < end; ptr++) {
	211	if (*ptr < text)
	212	continue;
	213	if (*ptr > text_end)
	214	continue;
	215	text_poke(ptr, ((unsigned char []){0xf0}), 1); / add lock prefix */
	216	};
	217	}
	218
	219	static void alternatives_smp_unlock(u8 start, u8 end, u8 text, u8 text_end)
	220	{
	221	u8 **ptr;
	222	char insn[1];
	223
	224	if (noreplace_smp)
	225	return;
	226
	227	add_nops(insn, 1);
	228	for (ptr = start; ptr < end; ptr++) {
	229	if (*ptr < text)
	230	continue;
	231	if (*ptr > text_end)
	232	continue;
	233	text_poke(*ptr, insn, 1);
	234	};
	235	}
	236
	237	struct smp_alt_module {
	238	/* what is this ??? */
	239	struct module *mod;
	240	char *name;
	241
	242	/* ptrs to lock prefixes */
	243	u8 **locks;
	244	u8 **locks_end;
	245
	246	/* .text segment, needed to avoid patching init code ;) */
	247	u8 *text;
	248	u8 *text_end;
	249
	250	struct list_head next;
	251	};
	252	static LIST_HEAD(smp_alt_modules);
	253	static DEFINE_SPINLOCK(smp_alt);
	254
	255	void alternatives_smp_module_add(struct module mod, char name,
	256	void locks, void locks_end,
	257	void text, void text_end)
	258	{
	259	struct smp_alt_module *smp;
	260	unsigned long flags;
	261
	262	if (noreplace_smp)
	263	return;
	264
	265	if (smp_alt_once) {
	266	if (boot_cpu_has(X86_FEATURE_UP))
	267	alternatives_smp_unlock(locks, locks_end,
	268	text, text_end);
	269	return;
	270	}
	271
	272	smp = kzalloc(sizeof(*smp), GFP_KERNEL);
	273	if (NULL == smp)
	274	return; /* we'll run the (safe but slow) SMP code then ... */
	275
	276	smp->mod = mod;
	277	smp->name = name;
	278	smp->locks = locks;
	279	smp->locks_end = locks_end;
	280	smp->text = text;
	281	smp->text_end = text_end;
	282	DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
	283	__FUNCTION__, smp->locks, smp->locks_end,
	284	smp->text, smp->text_end, smp->name);
	285
	286	spin_lock_irqsave(&smp_alt, flags);
	287	list_add_tail(&smp->next, &smp_alt_modules);
	288	if (boot_cpu_has(X86_FEATURE_UP))
	289	alternatives_smp_unlock(smp->locks, smp->locks_end,
	290	smp->text, smp->text_end);
	291	spin_unlock_irqrestore(&smp_alt, flags);
	292	}
	293
	294	void alternatives_smp_module_del(struct module *mod)
	295	{
	296	struct smp_alt_module *item;
	297	unsigned long flags;
	298
	299	if (smp_alt_once \|\| noreplace_smp)
	300	return;
	301
	302	spin_lock_irqsave(&smp_alt, flags);
	303	list_for_each_entry(item, &smp_alt_modules, next) {
	304	if (mod != item->mod)
	305	continue;
	306	list_del(&item->next);
	307	spin_unlock_irqrestore(&smp_alt, flags);
	308	DPRINTK("%s: %s\n", __FUNCTION__, item->name);
	309	kfree(item);
	310	return;
	311	}
	312	spin_unlock_irqrestore(&smp_alt, flags);
	313	}
	314
	315	void alternatives_smp_switch(int smp)
	316	{
	317	struct smp_alt_module *mod;
	318	unsigned long flags;
	319
	320	#ifdef CONFIG_LOCKDEP
	321	/*
	322	* A not yet fixed binutils section handling bug prevents
	323	* alternatives-replacement from working reliably, so turn
	324	* it off:
	325	*/
	326	printk("lockdep: not fixing up alternatives.\n");
	327	return;
	328	#endif
	329
	330	if (noreplace_smp \|\| smp_alt_once)
	331	return;
	332	BUG_ON(!smp && (num_online_cpus() > 1));
	333
	334	spin_lock_irqsave(&smp_alt, flags);
	335	if (smp) {
	336	printk(KERN_INFO "SMP alternatives: switching to SMP code\n");
	337	clear_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
	338	clear_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
	339	list_for_each_entry(mod, &smp_alt_modules, next)
	340	alternatives_smp_lock(mod->locks, mod->locks_end,
	341	mod->text, mod->text_end);
	342	} else {
	343	printk(KERN_INFO "SMP alternatives: switching to UP code\n");
	344	set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
	345	set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
	346	list_for_each_entry(mod, &smp_alt_modules, next)
	347	alternatives_smp_unlock(mod->locks, mod->locks_end,
	348	mod->text, mod->text_end);
	349	}
	350	spin_unlock_irqrestore(&smp_alt, flags);
	351	}
	352
	353	#endif
	354
	355	#ifdef CONFIG_PARAVIRT
	356	void apply_paravirt(struct paravirt_patch_site *start,
	357	struct paravirt_patch_site *end)
	358	{
	359	struct paravirt_patch_site *p;
	360	char insnbuf[MAX_PATCH_LEN];
	361
	362	if (noreplace_paravirt)
	363	return;
	364
	365	for (p = start; p < end; p++) {
	366	unsigned int used;
	367
	368	BUG_ON(p->len > MAX_PATCH_LEN);
	369	/* prep the buffer with the original instructions */
	370	memcpy(insnbuf, p->instr, p->len);
	371	used = paravirt_ops.patch(p->instrtype, p->clobbers, insnbuf,
	372	(unsigned long)p->instr, p->len);
	373
	374	BUG_ON(used > p->len);
	375
	376	/* Pad the rest with nops */
	377	add_nops(insnbuf + used, p->len - used);
	378	text_poke(p->instr, insnbuf, p->len);
	379	}
	380	}
	381	extern struct paravirt_patch_site __start_parainstructions[],
	382	__stop_parainstructions[];
	383	#endif /* CONFIG_PARAVIRT */
	384
	385	void __init alternative_instructions(void)
	386	{
	387	unsigned long flags;
	388
	389	/* The patching is not fully atomic, so try to avoid local interruptions
	390	that might execute the to be patched code.
	391	Other CPUs are not running. */
	392	stop_nmi();
	393	#ifdef CONFIG_X86_MCE
	394	stop_mce();
	395	#endif
	396
	397	local_irq_save(flags);
	398	apply_alternatives(__alt_instructions, __alt_instructions_end);
	399
	400	/* switch to patch-once-at-boottime-only mode and free the
	401	* tables in case we know the number of CPUs will never ever
	402	* change */
	403	#ifdef CONFIG_HOTPLUG_CPU
	404	if (num_possible_cpus() < 2)
	405	smp_alt_once = 1;
	406	#endif
	407
	408	#ifdef CONFIG_SMP
	409	if (smp_alt_once) {
	410	if (1 == num_possible_cpus()) {
	411	printk(KERN_INFO "SMP alternatives: switching to UP code\n");
	412	set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
	413	set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
	414	alternatives_smp_unlock(__smp_locks, __smp_locks_end,
	415	_text, _etext);
	416	}
	417	free_init_pages("SMP alternatives",
	418	(unsigned long)__smp_locks,
	419	(unsigned long)__smp_locks_end);
	420	} else {
	421	alternatives_smp_module_add(NULL, "core kernel",
	422	__smp_locks, __smp_locks_end,
	423	_text, _etext);
	424	alternatives_smp_switch(0);
	425	}
	426	#endif
	427	apply_paravirt(__parainstructions, __parainstructions_end);
	428	local_irq_restore(flags);
	429
	430	restart_nmi();
	431	#ifdef CONFIG_X86_MCE
	432	restart_mce();
	433	#endif
	434	}
	435
	436	/*
	437	* Warning:
	438	* When you use this code to patch more than one byte of an instruction
	439	* you need to make sure that other CPUs cannot execute this code in parallel.
	440	* Also no thread must be currently preempted in the middle of these instructions.
	441	* And on the local CPU you need to be protected again NMI or MCE handlers
	442	* seeing an inconsistent instruction while you patch.
	443	*/
	444	void __kprobes text_poke(void addr, unsigned char opcode, int len)
	445	{
	446	memcpy(addr, opcode, len);
	447	sync_core();
	448	/* Not strictly needed, but can speed CPU recovery up. Ignore cross cacheline
	449	case. */
	450	if (cpu_has_clflush)
	451	asm("clflush (%0) " :: "r" (addr) : "memory");
	452	}