[net-next-2.6.git] / arch / i386 / kernel / alternative.c

#include <linux/module.h>
#include <linux/sched.h>
#include <linux/spinlock.h>
#include <linux/list.h>
#include <asm/alternative.h>
#include <asm/sections.h>

static int noreplace_smp     = 0;
static int smp_alt_once      = 0;
static int debug_alternative = 0;

static int __init bootonly(char *str)
{
	smp_alt_once = 1;
	return 1;
}
__setup("smp-alt-boot", bootonly);

static int __init debug_alt(char *str)
{
	debug_alternative = 1;
	return 1;
}
__setup("debug-alternative", debug_alt);

static int __init setup_noreplace_smp(char *str)
{
	noreplace_smp = 1;
	return 1;
}
__setup("noreplace-smp", setup_noreplace_smp);


#define DPRINTK(fmt, args...) if (debug_alternative) \
	printk(KERN_DEBUG fmt, args)

#ifdef GENERIC_NOP1
/* Use inline assembly to define this because the nops are defined
   as inline assembly strings in the include files and we cannot
   get them easily into strings. */
asm("\t.data\nintelnops: "
	GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
	GENERIC_NOP7 GENERIC_NOP8);
extern unsigned char intelnops[];
static unsigned char *intel_nops[ASM_NOP_MAX+1] = {
	NULL,
	intelnops,
	intelnops + 1,
	intelnops + 1 + 2,
	intelnops + 1 + 2 + 3,
	intelnops + 1 + 2 + 3 + 4,
	intelnops + 1 + 2 + 3 + 4 + 5,
	intelnops + 1 + 2 + 3 + 4 + 5 + 6,
	intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
};
#endif

#ifdef K8_NOP1
asm("\t.data\nk8nops: "
	K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
	K8_NOP7 K8_NOP8);
extern unsigned char k8nops[];
static unsigned char *k8_nops[ASM_NOP_MAX+1] = {
	NULL,
	k8nops,
	k8nops + 1,
	k8nops + 1 + 2,
	k8nops + 1 + 2 + 3,
	k8nops + 1 + 2 + 3 + 4,
	k8nops + 1 + 2 + 3 + 4 + 5,
	k8nops + 1 + 2 + 3 + 4 + 5 + 6,
	k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
};
#endif

#ifdef K7_NOP1
asm("\t.data\nk7nops: "
	K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
	K7_NOP7 K7_NOP8);
extern unsigned char k7nops[];
static unsigned char *k7_nops[ASM_NOP_MAX+1] = {
	NULL,
	k7nops,
	k7nops + 1,
	k7nops + 1 + 2,
	k7nops + 1 + 2 + 3,
	k7nops + 1 + 2 + 3 + 4,
	k7nops + 1 + 2 + 3 + 4 + 5,
	k7nops + 1 + 2 + 3 + 4 + 5 + 6,
	k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
};
#endif

#ifdef CONFIG_X86_64

extern char __vsyscall_0;
static inline unsigned char** find_nop_table(void)
{
	return k8_nops;
}

#else /* CONFIG_X86_64 */

static struct nop {
	int cpuid;
	unsigned char **noptable;
} noptypes[] = {
	{ X86_FEATURE_K8, k8_nops },
	{ X86_FEATURE_K7, k7_nops },
	{ -1, NULL }
};

static unsigned char** find_nop_table(void)
{
	unsigned char **noptable = intel_nops;
	int i;

	for (i = 0; noptypes[i].cpuid >= 0; i++) {
		if (boot_cpu_has(noptypes[i].cpuid)) {
			noptable = noptypes[i].noptable;
			break;
		}
	}
	return noptable;
}

#endif /* CONFIG_X86_64 */

static void nop_out(void *insns, unsigned int len)
{
	unsigned char **noptable = find_nop_table();

	while (len > 0) {
		unsigned int noplen = len;
		if (noplen > ASM_NOP_MAX)
			noplen = ASM_NOP_MAX;
		memcpy(insns, noptable[noplen], noplen);
		insns += noplen;
		len -= noplen;
	}
}

extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
extern u8 *__smp_locks[], *__smp_locks_end[];

/* Replace instructions with better alternatives for this CPU type.
   This runs before SMP is initialized to avoid SMP problems with
   self modifying code. This implies that assymetric systems where
   APs have less capabilities than the boot processor are not handled.
   Tough. Make sure you disable such features by hand. */

void apply_alternatives(struct alt_instr *start, struct alt_instr *end)
{
	struct alt_instr *a;
	u8 *instr;
	int diff;

	DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end);
	for (a = start; a < end; a++) {
		BUG_ON(a->replacementlen > a->instrlen);
		if (!boot_cpu_has(a->cpuid))
			continue;
		instr = a->instr;
#ifdef CONFIG_X86_64
		/* vsyscall code is not mapped yet. resolve it manually. */
		if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END) {
			instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0));
			DPRINTK("%s: vsyscall fixup: %p => %p\n",
				__FUNCTION__, a->instr, instr);
		}
#endif
		memcpy(instr, a->replacement, a->replacementlen);
		diff = a->instrlen - a->replacementlen;
		nop_out(instr + a->replacementlen, diff);
	}
}

#ifdef CONFIG_SMP

static void alternatives_smp_lock(u8 **start, u8 **end, u8 *text, u8 *text_end)
{
	u8 **ptr;

	for (ptr = start; ptr < end; ptr++) {
		if (*ptr < text)
			continue;
		if (*ptr > text_end)
			continue;
		**ptr = 0xf0; /* lock prefix */
	};
}

static void alternatives_smp_unlock(u8 **start, u8 **end, u8 *text, u8 *text_end)
{
	u8 **ptr;

	if (noreplace_smp)
		return;

	for (ptr = start; ptr < end; ptr++) {
		if (*ptr < text)
			continue;
		if (*ptr > text_end)
			continue;
		nop_out(*ptr, 1);
	};
}

struct smp_alt_module {
	/* what is this ??? */
	struct module	*mod;
	char		*name;

	/* ptrs to lock prefixes */
	u8		**locks;
	u8		**locks_end;

	/* .text segment, needed to avoid patching init code ;) */
	u8		*text;
	u8		*text_end;

	struct list_head next;
};
static LIST_HEAD(smp_alt_modules);
static DEFINE_SPINLOCK(smp_alt);

void alternatives_smp_module_add(struct module *mod, char *name,
				 void *locks, void *locks_end,
				 void *text,  void *text_end)
{
	struct smp_alt_module *smp;
	unsigned long flags;

	if (noreplace_smp)
		return;

	if (smp_alt_once) {
		if (boot_cpu_has(X86_FEATURE_UP))
			alternatives_smp_unlock(locks, locks_end,
						text, text_end);
		return;
	}

	smp = kzalloc(sizeof(*smp), GFP_KERNEL);
	if (NULL == smp)
		return; /* we'll run the (safe but slow) SMP code then ... */

	smp->mod	= mod;
	smp->name	= name;
	smp->locks	= locks;
	smp->locks_end	= locks_end;
	smp->text	= text;
	smp->text_end	= text_end;
	DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
		__FUNCTION__, smp->locks, smp->locks_end,
		smp->text, smp->text_end, smp->name);

	spin_lock_irqsave(&smp_alt, flags);
	list_add_tail(&smp->next, &smp_alt_modules);
	if (boot_cpu_has(X86_FEATURE_UP))
		alternatives_smp_unlock(smp->locks, smp->locks_end,
					smp->text, smp->text_end);
	spin_unlock_irqrestore(&smp_alt, flags);
}

void alternatives_smp_module_del(struct module *mod)
{
	struct smp_alt_module *item;
	unsigned long flags;

	if (smp_alt_once || noreplace_smp)
		return;

	spin_lock_irqsave(&smp_alt, flags);
	list_for_each_entry(item, &smp_alt_modules, next) {
		if (mod != item->mod)
			continue;
		list_del(&item->next);
		spin_unlock_irqrestore(&smp_alt, flags);
		DPRINTK("%s: %s\n", __FUNCTION__, item->name);
		kfree(item);
		return;
	}
	spin_unlock_irqrestore(&smp_alt, flags);
}

void alternatives_smp_switch(int smp)
{
	struct smp_alt_module *mod;
	unsigned long flags;

#ifdef CONFIG_LOCKDEP
	/*
	 * A not yet fixed binutils section handling bug prevents
	 * alternatives-replacement from working reliably, so turn
	 * it off:
	 */
	printk("lockdep: not fixing up alternatives.\n");
	return;
#endif

	if (noreplace_smp || smp_alt_once)
		return;
	BUG_ON(!smp && (num_online_cpus() > 1));

	spin_lock_irqsave(&smp_alt, flags);
	if (smp) {
		printk(KERN_INFO "SMP alternatives: switching to SMP code\n");
		clear_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
		clear_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
		list_for_each_entry(mod, &smp_alt_modules, next)
			alternatives_smp_lock(mod->locks, mod->locks_end,
					      mod->text, mod->text_end);
	} else {
		printk(KERN_INFO "SMP alternatives: switching to UP code\n");
		set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
		set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
		list_for_each_entry(mod, &smp_alt_modules, next)
			alternatives_smp_unlock(mod->locks, mod->locks_end,
						mod->text, mod->text_end);
	}
	spin_unlock_irqrestore(&smp_alt, flags);
}

#endif

#ifdef CONFIG_PARAVIRT
void apply_paravirt(struct paravirt_patch *start, struct paravirt_patch *end)
{
	struct paravirt_patch *p;

	for (p = start; p < end; p++) {
		unsigned int used;

		used = paravirt_ops.patch(p->instrtype, p->clobbers, p->instr,
					  p->len);

		/* Pad the rest with nops */
		nop_out(p->instr + used, p->len - used);
	}

	/* Sync to be conservative, in case we patched following instructions */
	sync_core();
}
extern struct paravirt_patch __start_parainstructions[],
	__stop_parainstructions[];
#endif	/* CONFIG_PARAVIRT */

void __init alternative_instructions(void)
{
	unsigned long flags;

	local_irq_save(flags);
	apply_alternatives(__alt_instructions, __alt_instructions_end);

	/* switch to patch-once-at-boottime-only mode and free the
	 * tables in case we know the number of CPUs will never ever
	 * change */
#ifdef CONFIG_HOTPLUG_CPU
	if (num_possible_cpus() < 2)
		smp_alt_once = 1;
#else
	smp_alt_once = 1;
#endif

#ifdef CONFIG_SMP
	if (smp_alt_once) {
		if (1 == num_possible_cpus()) {
			printk(KERN_INFO "SMP alternatives: switching to UP code\n");
			set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
			set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
			alternatives_smp_unlock(__smp_locks, __smp_locks_end,
						_text, _etext);
		}
		free_init_pages("SMP alternatives",
				__pa_symbol(&__smp_locks),
				__pa_symbol(&__smp_locks_end));
	} else {
		alternatives_smp_module_add(NULL, "core kernel",
					    __smp_locks, __smp_locks_end,
					    _text, _etext);
		alternatives_smp_switch(0);
	}
#endif
 	apply_paravirt(__start_parainstructions, __stop_parainstructions);
	local_irq_restore(flags);
}
Commit	Line	Data
9a0b5817	1	#include <linux/module.h>
f6a57033	2	#include <linux/sched.h>
9a0b5817 GH	3	#include <linux/spinlock.h>
	4	#include <linux/list.h>
	5	#include <asm/alternative.h>
	6	#include <asm/sections.h>
	7
b7fb4af0	8	static int noreplace_smp = 0;
d167a518 GH	9	static int smp_alt_once = 0;
d167a518 GH	10	static int debug_alternative = 0;
9a0b5817	11
d167a518 GH	12	static int __init bootonly(char *str)
	13	{
	14	smp_alt_once = 1;
	15	return 1;
	16	}
b7fb4af0 JF	17	__setup("smp-alt-boot", bootonly);
b7fb4af0 JF	18
d167a518 GH	19	static int __init debug_alt(char *str)
	20	{
	21	debug_alternative = 1;
	22	return 1;
	23	}
d167a518 GH	24	__setup("debug-alternative", debug_alt);
d167a518 GH	25
b7fb4af0 JF	26	static int __init setup_noreplace_smp(char *str)
	27	{
	28	noreplace_smp = 1;
	29	return 1;
	30	}
	31	__setup("noreplace-smp", setup_noreplace_smp);
	32
	33
d167a518 GH	34	#define DPRINTK(fmt, args...) if (debug_alternative) \
	35	printk(KERN_DEBUG fmt, args)
	36
	37	#ifdef GENERIC_NOP1
9a0b5817 GH	38	/* Use inline assembly to define this because the nops are defined
	39	as inline assembly strings in the include files and we cannot
	40	get them easily into strings. */
	41	asm("\t.data\nintelnops: "
	42	GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
	43	GENERIC_NOP7 GENERIC_NOP8);
d167a518	44	extern unsigned char intelnops[];
9a0b5817 GH	45	static unsigned char *intel_nops[ASM_NOP_MAX+1] = {
	46	NULL,
	47	intelnops,
	48	intelnops + 1,
	49	intelnops + 1 + 2,
	50	intelnops + 1 + 2 + 3,
	51	intelnops + 1 + 2 + 3 + 4,
	52	intelnops + 1 + 2 + 3 + 4 + 5,
	53	intelnops + 1 + 2 + 3 + 4 + 5 + 6,
	54	intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
	55	};
d167a518 GH	56	#endif
	57
	58	#ifdef K8_NOP1
	59	asm("\t.data\nk8nops: "
	60	K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
	61	K8_NOP7 K8_NOP8);
	62	extern unsigned char k8nops[];
9a0b5817 GH	63	static unsigned char *k8_nops[ASM_NOP_MAX+1] = {
	64	NULL,
	65	k8nops,
	66	k8nops + 1,
	67	k8nops + 1 + 2,
	68	k8nops + 1 + 2 + 3,
	69	k8nops + 1 + 2 + 3 + 4,
	70	k8nops + 1 + 2 + 3 + 4 + 5,
	71	k8nops + 1 + 2 + 3 + 4 + 5 + 6,
	72	k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
	73	};
d167a518 GH	74	#endif
	75
	76	#ifdef K7_NOP1
	77	asm("\t.data\nk7nops: "
	78	K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
	79	K7_NOP7 K7_NOP8);
	80	extern unsigned char k7nops[];
9a0b5817 GH	81	static unsigned char *k7_nops[ASM_NOP_MAX+1] = {
	82	NULL,
	83	k7nops,
	84	k7nops + 1,
	85	k7nops + 1 + 2,
	86	k7nops + 1 + 2 + 3,
	87	k7nops + 1 + 2 + 3 + 4,
	88	k7nops + 1 + 2 + 3 + 4 + 5,
	89	k7nops + 1 + 2 + 3 + 4 + 5 + 6,
	90	k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
	91	};
d167a518 GH	92	#endif
	93
	94	#ifdef CONFIG_X86_64
	95
	96	extern char __vsyscall_0;
	97	static inline unsigned char** find_nop_table(void)
	98	{
	99	return k8_nops;
	100	}
	101
	102	#else /* CONFIG_X86_64 */
	103
9a0b5817 GH	104	static struct nop {
	105	int cpuid;
	106	unsigned char **noptable;
	107	} noptypes[] = {
	108	{ X86_FEATURE_K8, k8_nops },
	109	{ X86_FEATURE_K7, k7_nops },
	110	{ -1, NULL }
	111	};
	112
9a0b5817 GH	113	static unsigned char** find_nop_table(void)
	114	{
	115	unsigned char **noptable = intel_nops;
	116	int i;
	117
	118	for (i = 0; noptypes[i].cpuid >= 0; i++) {
	119	if (boot_cpu_has(noptypes[i].cpuid)) {
	120	noptable = noptypes[i].noptable;
	121	break;
	122	}
	123	}
	124	return noptable;
	125	}
	126
d167a518 GH	127	#endif /* CONFIG_X86_64 */
d167a518 GH	128
139ec7c4 RR	129	static void nop_out(void *insns, unsigned int len)
	130	{
	131	unsigned char **noptable = find_nop_table();
	132
	133	while (len > 0) {
	134	unsigned int noplen = len;
	135	if (noplen > ASM_NOP_MAX)
	136	noplen = ASM_NOP_MAX;
	137	memcpy(insns, noptable[noplen], noplen);
	138	insns += noplen;
	139	len -= noplen;
	140	}
	141	}
	142
d167a518	143	extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
d167a518 GH	144	extern u8 __smp_locks[], __smp_locks_end[];
d167a518 GH	145
9a0b5817 GH	146	/* Replace instructions with better alternatives for this CPU type.
	147	This runs before SMP is initialized to avoid SMP problems with
	148	self modifying code. This implies that assymetric systems where
	149	APs have less capabilities than the boot processor are not handled.
	150	Tough. Make sure you disable such features by hand. */
	151
	152	void apply_alternatives(struct alt_instr start, struct alt_instr end)
	153	{
9a0b5817	154	struct alt_instr *a;
d167a518	155	u8 *instr;
139ec7c4	156	int diff;
9a0b5817 GH	157
	158	DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end);
	159	for (a = start; a < end; a++) {
	160	BUG_ON(a->replacementlen > a->instrlen);
	161	if (!boot_cpu_has(a->cpuid))
	162	continue;
d167a518 GH	163	instr = a->instr;
	164	#ifdef CONFIG_X86_64
	165	/* vsyscall code is not mapped yet. resolve it manually. */
	166	if (instr >= (u8 )VSYSCALL_START && instr < (u8)VSYSCALL_END) {
	167	instr = __va(instr - (u8)VSYSCALL_START + (u8)__pa_symbol(&__vsyscall_0));
	168	DPRINTK("%s: vsyscall fixup: %p => %p\n",
	169	__FUNCTION__, a->instr, instr);
	170	}
	171	#endif
	172	memcpy(instr, a->replacement, a->replacementlen);
9a0b5817	173	diff = a->instrlen - a->replacementlen;
139ec7c4	174	nop_out(instr + a->replacementlen, diff);
9a0b5817 GH	175	}
	176	}
	177
8ec4d41f GH	178	#ifdef CONFIG_SMP
8ec4d41f GH	179
9a0b5817 GH	180	static void alternatives_smp_lock(u8 start, u8 end, u8 text, u8 text_end)
	181	{
	182	u8 **ptr;
	183
	184	for (ptr = start; ptr < end; ptr++) {
	185	if (*ptr < text)
	186	continue;
	187	if (*ptr > text_end)
	188	continue;
	189	*ptr = 0xf0; / lock prefix */
	190	};
	191	}
	192
	193	static void alternatives_smp_unlock(u8 start, u8 end, u8 text, u8 text_end)
	194	{
9a0b5817 GH	195	u8 **ptr;
9a0b5817 GH	196
b7fb4af0 JF	197	if (noreplace_smp)
	198	return;
	199
9a0b5817 GH	200	for (ptr = start; ptr < end; ptr++) {
	201	if (*ptr < text)
	202	continue;
	203	if (*ptr > text_end)
	204	continue;
139ec7c4	205	nop_out(*ptr, 1);
9a0b5817 GH	206	};
	207	}
	208
	209	struct smp_alt_module {
	210	/* what is this ??? */
	211	struct module *mod;
	212	char *name;
	213
	214	/* ptrs to lock prefixes */
	215	u8 **locks;
	216	u8 **locks_end;
	217
	218	/* .text segment, needed to avoid patching init code ;) */
	219	u8 *text;
	220	u8 *text_end;
	221
	222	struct list_head next;
	223	};
	224	static LIST_HEAD(smp_alt_modules);
	225	static DEFINE_SPINLOCK(smp_alt);
	226
9a0b5817 GH	227	void alternatives_smp_module_add(struct module mod, char name,
	228	void locks, void locks_end,
	229	void text, void text_end)
	230	{
	231	struct smp_alt_module *smp;
	232	unsigned long flags;
	233
b7fb4af0 JF	234	if (noreplace_smp)
	235	return;
	236
9a0b5817 GH	237	if (smp_alt_once) {
	238	if (boot_cpu_has(X86_FEATURE_UP))
	239	alternatives_smp_unlock(locks, locks_end,
	240	text, text_end);
	241	return;
	242	}
	243
	244	smp = kzalloc(sizeof(*smp), GFP_KERNEL);
	245	if (NULL == smp)
	246	return; /* we'll run the (safe but slow) SMP code then ... */
	247
	248	smp->mod = mod;
	249	smp->name = name;
	250	smp->locks = locks;
	251	smp->locks_end = locks_end;
	252	smp->text = text;
	253	smp->text_end = text_end;
	254	DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
	255	__FUNCTION__, smp->locks, smp->locks_end,
	256	smp->text, smp->text_end, smp->name);
	257
	258	spin_lock_irqsave(&smp_alt, flags);
	259	list_add_tail(&smp->next, &smp_alt_modules);
	260	if (boot_cpu_has(X86_FEATURE_UP))
	261	alternatives_smp_unlock(smp->locks, smp->locks_end,
	262	smp->text, smp->text_end);
	263	spin_unlock_irqrestore(&smp_alt, flags);
	264	}
	265
	266	void alternatives_smp_module_del(struct module *mod)
	267	{
	268	struct smp_alt_module *item;
	269	unsigned long flags;
	270
b7fb4af0	271	if (smp_alt_once \|\| noreplace_smp)
9a0b5817 GH	272	return;
	273
	274	spin_lock_irqsave(&smp_alt, flags);
	275	list_for_each_entry(item, &smp_alt_modules, next) {
	276	if (mod != item->mod)
	277	continue;
	278	list_del(&item->next);
	279	spin_unlock_irqrestore(&smp_alt, flags);
	280	DPRINTK("%s: %s\n", __FUNCTION__, item->name);
	281	kfree(item);
	282	return;
	283	}
	284	spin_unlock_irqrestore(&smp_alt, flags);
	285	}
	286
	287	void alternatives_smp_switch(int smp)
	288	{
	289	struct smp_alt_module *mod;
	290	unsigned long flags;
	291
3047e99e IM	292	#ifdef CONFIG_LOCKDEP
	293	/*
	294	* A not yet fixed binutils section handling bug prevents
	295	* alternatives-replacement from working reliably, so turn
	296	* it off:
	297	*/
	298	printk("lockdep: not fixing up alternatives.\n");
	299	return;
	300	#endif
	301
b7fb4af0	302	if (noreplace_smp \|\| smp_alt_once)
9a0b5817 GH	303	return;
	304	BUG_ON(!smp && (num_online_cpus() > 1));
	305
	306	spin_lock_irqsave(&smp_alt, flags);
	307	if (smp) {
	308	printk(KERN_INFO "SMP alternatives: switching to SMP code\n");
	309	clear_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
	310	clear_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
9a0b5817 GH	311	list_for_each_entry(mod, &smp_alt_modules, next)
	312	alternatives_smp_lock(mod->locks, mod->locks_end,
	313	mod->text, mod->text_end);
	314	} else {
	315	printk(KERN_INFO "SMP alternatives: switching to UP code\n");
	316	set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
	317	set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
9a0b5817 GH	318	list_for_each_entry(mod, &smp_alt_modules, next)
	319	alternatives_smp_unlock(mod->locks, mod->locks_end,
	320	mod->text, mod->text_end);
	321	}
	322	spin_unlock_irqrestore(&smp_alt, flags);
	323	}
	324
8ec4d41f GH	325	#endif
8ec4d41f GH	326
139ec7c4 RR	327	#ifdef CONFIG_PARAVIRT
	328	void apply_paravirt(struct paravirt_patch start, struct paravirt_patch end)
	329	{
	330	struct paravirt_patch *p;
	331
	332	for (p = start; p < end; p++) {
	333	unsigned int used;
	334
	335	used = paravirt_ops.patch(p->instrtype, p->clobbers, p->instr,
	336	p->len);
7f63c41c	337
139ec7c4 RR	338	/* Pad the rest with nops */
	339	nop_out(p->instr + used, p->len - used);
	340	}
	341
	342	/* Sync to be conservative, in case we patched following instructions */
	343	sync_core();
	344	}
	345	extern struct paravirt_patch __start_parainstructions[],
	346	__stop_parainstructions[];
	347	#endif /* CONFIG_PARAVIRT */
	348
9a0b5817 GH	349	void __init alternative_instructions(void)
9a0b5817 GH	350	{
e51959fa	351	unsigned long flags;
e51959fa ZA	352
e51959fa ZA	353	local_irq_save(flags);
9a0b5817 GH	354	apply_alternatives(__alt_instructions, __alt_instructions_end);
	355
	356	/* switch to patch-once-at-boottime-only mode and free the
	357	* tables in case we know the number of CPUs will never ever
	358	* change */
	359	#ifdef CONFIG_HOTPLUG_CPU
	360	if (num_possible_cpus() < 2)
	361	smp_alt_once = 1;
	362	#else
	363	smp_alt_once = 1;
	364	#endif
	365
8ec4d41f	366	#ifdef CONFIG_SMP
9a0b5817 GH	367	if (smp_alt_once) {
	368	if (1 == num_possible_cpus()) {
	369	printk(KERN_INFO "SMP alternatives: switching to UP code\n");
	370	set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
	371	set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
9a0b5817 GH	372	alternatives_smp_unlock(__smp_locks, __smp_locks_end,
	373	_text, _etext);
	374	}
	375	free_init_pages("SMP alternatives",
d0175ab6 JF	376	__pa_symbol(&__smp_locks),
d0175ab6 JF	377	__pa_symbol(&__smp_locks_end));
9a0b5817	378	} else {
9a0b5817 GH	379	alternatives_smp_module_add(NULL, "core kernel",
	380	__smp_locks, __smp_locks_end,
	381	_text, _etext);
	382	alternatives_smp_switch(0);
	383	}
8ec4d41f	384	#endif
139ec7c4	385	apply_paravirt(__start_parainstructions, __stop_parainstructions);
e51959fa	386	local_irq_restore(flags);
9a0b5817	387	}