]>
Commit | Line | Data |
---|---|---|
1965aae3 PA |
1 | #ifndef _ASM_X86_SYSTEM_H |
2 | #define _ASM_X86_SYSTEM_H | |
d8954222 GOC |
3 | |
4 | #include <asm/asm.h> | |
d46d7d75 GOC |
5 | #include <asm/segment.h> |
6 | #include <asm/cpufeature.h> | |
7 | #include <asm/cmpxchg.h> | |
fde1b3fa | 8 | #include <asm/nops.h> |
d8954222 | 9 | |
d3ca901f | 10 | #include <linux/kernel.h> |
d46d7d75 | 11 | #include <linux/irqflags.h> |
d3ca901f | 12 | |
ded9aa0d | 13 | /* entries in ARCH_DLINFO: */ |
cf9db6c4 | 14 | #if defined(CONFIG_IA32_EMULATION) || !defined(CONFIG_X86_64) |
ded9aa0d | 15 | # define AT_VECTOR_SIZE_ARCH 2 |
cf9db6c4 | 16 | #else /* else it's non-compat x86-64 */ |
ded9aa0d JB |
17 | # define AT_VECTOR_SIZE_ARCH 1 |
18 | #endif | |
19 | ||
0a3b4d15 | 20 | struct task_struct; /* one of the stranger aspects of C forward declarations */ |
599db4fe HH |
21 | struct task_struct *__switch_to(struct task_struct *prev, |
22 | struct task_struct *next); | |
2fb6b2a0 | 23 | struct tss_struct; |
389d1fb1 JF |
24 | void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p, |
25 | struct tss_struct *tss); | |
814e2c84 | 26 | extern void show_regs_common(void); |
0a3b4d15 | 27 | |
aab02f0a JS |
28 | #ifdef CONFIG_X86_32 |
29 | ||
60a5317f TH |
30 | #ifdef CONFIG_CC_STACKPROTECTOR |
31 | #define __switch_canary \ | |
5c79d2a5 TH |
32 | "movl %P[task_canary](%[next]), %%ebx\n\t" \ |
33 | "movl %%ebx, "__percpu_arg([stack_canary])"\n\t" | |
60a5317f | 34 | #define __switch_canary_oparam \ |
dd17c8f7 | 35 | , [stack_canary] "=m" (stack_canary.canary) |
60a5317f | 36 | #define __switch_canary_iparam \ |
60a5317f TH |
37 | , [task_canary] "i" (offsetof(struct task_struct, stack_canary)) |
38 | #else /* CC_STACKPROTECTOR */ | |
39 | #define __switch_canary | |
40 | #define __switch_canary_oparam | |
41 | #define __switch_canary_iparam | |
42 | #endif /* CC_STACKPROTECTOR */ | |
43 | ||
0a3b4d15 GOC |
44 | /* |
45 | * Saving eflags is important. It switches not only IOPL between tasks, | |
46 | * it also protects other tasks from NT leaking through sysenter etc. | |
47 | */ | |
23b55bd9 IM |
48 | #define switch_to(prev, next, last) \ |
49 | do { \ | |
8b6451fe IM |
50 | /* \ |
51 | * Context-switching clobbers all registers, so we clobber \ | |
52 | * them explicitly, via unused output variables. \ | |
53 | * (EAX and EBP is not listed because EBP is saved/restored \ | |
54 | * explicitly for wchan access and EAX is the return value of \ | |
55 | * __switch_to()) \ | |
56 | */ \ | |
57 | unsigned long ebx, ecx, edx, esi, edi; \ | |
23b55bd9 | 58 | \ |
c5386c20 JP |
59 | asm volatile("pushfl\n\t" /* save flags */ \ |
60 | "pushl %%ebp\n\t" /* save EBP */ \ | |
61 | "movl %%esp,%[prev_sp]\n\t" /* save ESP */ \ | |
62 | "movl %[next_sp],%%esp\n\t" /* restore ESP */ \ | |
63 | "movl $1f,%[prev_ip]\n\t" /* save EIP */ \ | |
64 | "pushl %[next_ip]\n\t" /* restore EIP */ \ | |
5c79d2a5 | 65 | __switch_canary \ |
c5386c20 JP |
66 | "jmp __switch_to\n" /* regparm call */ \ |
67 | "1:\t" \ | |
68 | "popl %%ebp\n\t" /* restore EBP */ \ | |
69 | "popfl\n" /* restore flags */ \ | |
23b55bd9 | 70 | \ |
c5386c20 JP |
71 | /* output parameters */ \ |
72 | : [prev_sp] "=m" (prev->thread.sp), \ | |
73 | [prev_ip] "=m" (prev->thread.ip), \ | |
74 | "=a" (last), \ | |
23b55bd9 | 75 | \ |
c5386c20 JP |
76 | /* clobbered output registers: */ \ |
77 | "=b" (ebx), "=c" (ecx), "=d" (edx), \ | |
78 | "=S" (esi), "=D" (edi) \ | |
79 | \ | |
60a5317f TH |
80 | __switch_canary_oparam \ |
81 | \ | |
c5386c20 JP |
82 | /* input parameters: */ \ |
83 | : [next_sp] "m" (next->thread.sp), \ | |
84 | [next_ip] "m" (next->thread.ip), \ | |
85 | \ | |
86 | /* regparm parameters for __switch_to(): */ \ | |
87 | [prev] "a" (prev), \ | |
33f8c40a VN |
88 | [next] "d" (next) \ |
89 | \ | |
60a5317f TH |
90 | __switch_canary_iparam \ |
91 | \ | |
33f8c40a VN |
92 | : /* reloaded segment registers */ \ |
93 | "memory"); \ | |
0a3b4d15 GOC |
94 | } while (0) |
95 | ||
d46d7d75 GOC |
96 | /* |
97 | * disable hlt during certain critical i/o operations | |
98 | */ | |
99 | #define HAVE_DISABLE_HLT | |
96a388de | 100 | #else |
0a3b4d15 GOC |
101 | #define __SAVE(reg, offset) "movq %%" #reg ",(14-" #offset ")*8(%%rsp)\n\t" |
102 | #define __RESTORE(reg, offset) "movq (14-" #offset ")*8(%%rsp),%%" #reg "\n\t" | |
103 | ||
104 | /* frame pointer must be last for get_wchan */ | |
105 | #define SAVE_CONTEXT "pushf ; pushq %%rbp ; movq %%rsi,%%rbp\n\t" | |
106 | #define RESTORE_CONTEXT "movq %%rbp,%%rsi ; popq %%rbp ; popf\t" | |
107 | ||
108 | #define __EXTRA_CLOBBER \ | |
109 | , "rcx", "rbx", "rdx", "r8", "r9", "r10", "r11", \ | |
110 | "r12", "r13", "r14", "r15" | |
111 | ||
b4a8f7a2 TH |
112 | #ifdef CONFIG_CC_STACKPROTECTOR |
113 | #define __switch_canary \ | |
114 | "movq %P[task_canary](%%rsi),%%r8\n\t" \ | |
67e68bde TH |
115 | "movq %%r8,"__percpu_arg([gs_canary])"\n\t" |
116 | #define __switch_canary_oparam \ | |
dd17c8f7 | 117 | , [gs_canary] "=m" (irq_stack_union.stack_canary) |
67e68bde TH |
118 | #define __switch_canary_iparam \ |
119 | , [task_canary] "i" (offsetof(struct task_struct, stack_canary)) | |
b4a8f7a2 TH |
120 | #else /* CC_STACKPROTECTOR */ |
121 | #define __switch_canary | |
67e68bde TH |
122 | #define __switch_canary_oparam |
123 | #define __switch_canary_iparam | |
b4a8f7a2 TH |
124 | #endif /* CC_STACKPROTECTOR */ |
125 | ||
0a3b4d15 GOC |
126 | /* Save restore flags to clear handle leaking NT */ |
127 | #define switch_to(prev, next, last) \ | |
b4a8f7a2 | 128 | asm volatile(SAVE_CONTEXT \ |
0a3b4d15 GOC |
129 | "movq %%rsp,%P[threadrsp](%[prev])\n\t" /* save RSP */ \ |
130 | "movq %P[threadrsp](%[next]),%%rsp\n\t" /* restore RSP */ \ | |
131 | "call __switch_to\n\t" \ | |
87b26406 | 132 | "movq "__percpu_arg([current_task])",%%rsi\n\t" \ |
b4a8f7a2 | 133 | __switch_canary \ |
0a3b4d15 | 134 | "movq %P[thread_info](%%rsi),%%r8\n\t" \ |
0a3b4d15 | 135 | "movq %%rax,%%rdi\n\t" \ |
dd17c8f7 | 136 | "testl %[_tif_fork],%P[ti_flags](%%r8)\n\t" \ |
7106a5ab | 137 | "jnz ret_from_fork\n\t" \ |
0a3b4d15 GOC |
138 | RESTORE_CONTEXT \ |
139 | : "=a" (last) \ | |
67e68bde | 140 | __switch_canary_oparam \ |
0a3b4d15 GOC |
141 | : [next] "S" (next), [prev] "D" (prev), \ |
142 | [threadrsp] "i" (offsetof(struct task_struct, thread.sp)), \ | |
143 | [ti_flags] "i" (offsetof(struct thread_info, flags)), \ | |
7106a5ab | 144 | [_tif_fork] "i" (_TIF_FORK), \ |
0a3b4d15 | 145 | [thread_info] "i" (offsetof(struct task_struct, stack)), \ |
dd17c8f7 | 146 | [current_task] "m" (current_task) \ |
67e68bde | 147 | __switch_canary_iparam \ |
0a3b4d15 | 148 | : "memory", "cc" __EXTRA_CLOBBER) |
96a388de | 149 | #endif |
d8954222 GOC |
150 | |
151 | #ifdef __KERNEL__ | |
d8954222 | 152 | |
9f9d489a | 153 | extern void native_load_gs_index(unsigned); |
d3ca901f | 154 | |
a6b46552 GOC |
155 | /* |
156 | * Load a segment. Fall back on loading the zero | |
157 | * segment if something goes wrong.. | |
158 | */ | |
64b028b2 IM |
159 | #define loadsegment(seg, value) \ |
160 | do { \ | |
161 | unsigned short __val = (value); \ | |
162 | \ | |
163 | asm volatile(" \n" \ | |
164 | "1: movl %k0,%%" #seg " \n" \ | |
165 | \ | |
166 | ".section .fixup,\"ax\" \n" \ | |
167 | "2: xorl %k0,%k0 \n" \ | |
168 | " jmp 1b \n" \ | |
169 | ".previous \n" \ | |
170 | \ | |
171 | _ASM_EXTABLE(1b, 2b) \ | |
172 | \ | |
173 | : "+r" (__val) : : "memory"); \ | |
79b0379c | 174 | } while (0) |
a6b46552 | 175 | |
d8954222 GOC |
176 | /* |
177 | * Save a segment register away | |
178 | */ | |
c5386c20 | 179 | #define savesegment(seg, value) \ |
d9fc3fd3 | 180 | asm("mov %%" #seg ",%0":"=r" (value) : : "memory") |
d8954222 | 181 | |
d9a89a26 TH |
182 | /* |
183 | * x86_32 user gs accessors. | |
184 | */ | |
185 | #ifdef CONFIG_X86_32 | |
ccbeed3a | 186 | #ifdef CONFIG_X86_32_LAZY_GS |
d9a89a26 TH |
187 | #define get_user_gs(regs) (u16)({unsigned long v; savesegment(gs, v); v;}) |
188 | #define set_user_gs(regs, v) loadsegment(gs, (unsigned long)(v)) | |
189 | #define task_user_gs(tsk) ((tsk)->thread.gs) | |
ccbeed3a TH |
190 | #define lazy_save_gs(v) savesegment(gs, (v)) |
191 | #define lazy_load_gs(v) loadsegment(gs, (v)) | |
192 | #else /* X86_32_LAZY_GS */ | |
193 | #define get_user_gs(regs) (u16)((regs)->gs) | |
194 | #define set_user_gs(regs, v) do { (regs)->gs = (v); } while (0) | |
195 | #define task_user_gs(tsk) (task_pt_regs(tsk)->gs) | |
196 | #define lazy_save_gs(v) do { } while (0) | |
197 | #define lazy_load_gs(v) do { } while (0) | |
198 | #endif /* X86_32_LAZY_GS */ | |
199 | #endif /* X86_32 */ | |
d9a89a26 | 200 | |
d8954222 GOC |
201 | static inline unsigned long get_limit(unsigned long segment) |
202 | { | |
203 | unsigned long __limit; | |
c5386c20 JP |
204 | asm("lsll %1,%0" : "=r" (__limit) : "r" (segment)); |
205 | return __limit + 1; | |
d8954222 | 206 | } |
d3ca901f GOC |
207 | |
208 | static inline void native_clts(void) | |
209 | { | |
c5386c20 | 210 | asm volatile("clts"); |
d3ca901f GOC |
211 | } |
212 | ||
213 | /* | |
214 | * Volatile isn't enough to prevent the compiler from reordering the | |
215 | * read/write functions for the control registers and messing everything up. | |
216 | * A memory clobber would solve the problem, but would prevent reordering of | |
217 | * all loads stores around it, which can hurt performance. Solution is to | |
218 | * use a variable and mimic reads and writes to it to enforce serialization | |
219 | */ | |
220 | static unsigned long __force_order; | |
221 | ||
222 | static inline unsigned long native_read_cr0(void) | |
223 | { | |
224 | unsigned long val; | |
c5386c20 | 225 | asm volatile("mov %%cr0,%0\n\t" : "=r" (val), "=m" (__force_order)); |
d3ca901f GOC |
226 | return val; |
227 | } | |
228 | ||
229 | static inline void native_write_cr0(unsigned long val) | |
230 | { | |
c5386c20 | 231 | asm volatile("mov %0,%%cr0": : "r" (val), "m" (__force_order)); |
d3ca901f GOC |
232 | } |
233 | ||
234 | static inline unsigned long native_read_cr2(void) | |
235 | { | |
236 | unsigned long val; | |
c5386c20 | 237 | asm volatile("mov %%cr2,%0\n\t" : "=r" (val), "=m" (__force_order)); |
d3ca901f GOC |
238 | return val; |
239 | } | |
240 | ||
241 | static inline void native_write_cr2(unsigned long val) | |
242 | { | |
c5386c20 | 243 | asm volatile("mov %0,%%cr2": : "r" (val), "m" (__force_order)); |
d3ca901f GOC |
244 | } |
245 | ||
246 | static inline unsigned long native_read_cr3(void) | |
247 | { | |
248 | unsigned long val; | |
c5386c20 | 249 | asm volatile("mov %%cr3,%0\n\t" : "=r" (val), "=m" (__force_order)); |
d3ca901f GOC |
250 | return val; |
251 | } | |
252 | ||
253 | static inline void native_write_cr3(unsigned long val) | |
254 | { | |
c5386c20 | 255 | asm volatile("mov %0,%%cr3": : "r" (val), "m" (__force_order)); |
d3ca901f GOC |
256 | } |
257 | ||
258 | static inline unsigned long native_read_cr4(void) | |
259 | { | |
260 | unsigned long val; | |
c5386c20 | 261 | asm volatile("mov %%cr4,%0\n\t" : "=r" (val), "=m" (__force_order)); |
d3ca901f GOC |
262 | return val; |
263 | } | |
264 | ||
265 | static inline unsigned long native_read_cr4_safe(void) | |
266 | { | |
267 | unsigned long val; | |
268 | /* This could fault if %cr4 does not exist. In x86_64, a cr4 always | |
269 | * exists, so it will never fail. */ | |
270 | #ifdef CONFIG_X86_32 | |
88976ee1 PA |
271 | asm volatile("1: mov %%cr4, %0\n" |
272 | "2:\n" | |
c5386c20 | 273 | _ASM_EXTABLE(1b, 2b) |
88976ee1 | 274 | : "=r" (val), "=m" (__force_order) : "0" (0)); |
d3ca901f GOC |
275 | #else |
276 | val = native_read_cr4(); | |
277 | #endif | |
278 | return val; | |
279 | } | |
280 | ||
281 | static inline void native_write_cr4(unsigned long val) | |
282 | { | |
c5386c20 | 283 | asm volatile("mov %0,%%cr4": : "r" (val), "m" (__force_order)); |
d3ca901f GOC |
284 | } |
285 | ||
94ea03cd GOC |
286 | #ifdef CONFIG_X86_64 |
287 | static inline unsigned long native_read_cr8(void) | |
288 | { | |
289 | unsigned long cr8; | |
290 | asm volatile("movq %%cr8,%0" : "=r" (cr8)); | |
291 | return cr8; | |
292 | } | |
293 | ||
294 | static inline void native_write_cr8(unsigned long val) | |
295 | { | |
296 | asm volatile("movq %0,%%cr8" :: "r" (val) : "memory"); | |
297 | } | |
298 | #endif | |
299 | ||
d3ca901f GOC |
300 | static inline void native_wbinvd(void) |
301 | { | |
302 | asm volatile("wbinvd": : :"memory"); | |
303 | } | |
c5386c20 | 304 | |
d3ca901f GOC |
305 | #ifdef CONFIG_PARAVIRT |
306 | #include <asm/paravirt.h> | |
307 | #else | |
308 | #define read_cr0() (native_read_cr0()) | |
309 | #define write_cr0(x) (native_write_cr0(x)) | |
310 | #define read_cr2() (native_read_cr2()) | |
311 | #define write_cr2(x) (native_write_cr2(x)) | |
312 | #define read_cr3() (native_read_cr3()) | |
313 | #define write_cr3(x) (native_write_cr3(x)) | |
314 | #define read_cr4() (native_read_cr4()) | |
315 | #define read_cr4_safe() (native_read_cr4_safe()) | |
316 | #define write_cr4(x) (native_write_cr4(x)) | |
317 | #define wbinvd() (native_wbinvd()) | |
d46d7d75 | 318 | #ifdef CONFIG_X86_64 |
94ea03cd GOC |
319 | #define read_cr8() (native_read_cr8()) |
320 | #define write_cr8(x) (native_write_cr8(x)) | |
9f9d489a | 321 | #define load_gs_index native_load_gs_index |
d46d7d75 GOC |
322 | #endif |
323 | ||
d3ca901f GOC |
324 | /* Clear the 'TS' bit */ |
325 | #define clts() (native_clts()) | |
326 | ||
327 | #endif/* CONFIG_PARAVIRT */ | |
328 | ||
4e09e21c | 329 | #define stts() write_cr0(read_cr0() | X86_CR0_TS) |
d3ca901f | 330 | |
d8954222 GOC |
331 | #endif /* __KERNEL__ */ |
332 | ||
84fb144b | 333 | static inline void clflush(volatile void *__p) |
d8954222 | 334 | { |
84fb144b | 335 | asm volatile("clflush %0" : "+m" (*(volatile char __force *)__p)); |
d8954222 GOC |
336 | } |
337 | ||
c5386c20 | 338 | #define nop() asm volatile ("nop") |
d8954222 GOC |
339 | |
340 | void disable_hlt(void); | |
341 | void enable_hlt(void); | |
342 | ||
d8954222 GOC |
343 | void cpu_idle_wait(void); |
344 | ||
345 | extern unsigned long arch_align_stack(unsigned long sp); | |
346 | extern void free_init_pages(char *what, unsigned long begin, unsigned long end); | |
347 | ||
348 | void default_idle(void); | |
349 | ||
d3ec5cae IV |
350 | void stop_this_cpu(void *dummy); |
351 | ||
833d8469 GOC |
352 | /* |
353 | * Force strict CPU ordering. | |
354 | * And yes, this is required on UP too when we're talking | |
355 | * to devices. | |
356 | */ | |
357 | #ifdef CONFIG_X86_32 | |
358 | /* | |
0d7a1819 | 359 | * Some non-Intel clones support out of order store. wmb() ceases to be a |
833d8469 GOC |
360 | * nop for these. |
361 | */ | |
362 | #define mb() alternative("lock; addl $0,0(%%esp)", "mfence", X86_FEATURE_XMM2) | |
363 | #define rmb() alternative("lock; addl $0,0(%%esp)", "lfence", X86_FEATURE_XMM2) | |
364 | #define wmb() alternative("lock; addl $0,0(%%esp)", "sfence", X86_FEATURE_XMM) | |
365 | #else | |
366 | #define mb() asm volatile("mfence":::"memory") | |
367 | #define rmb() asm volatile("lfence":::"memory") | |
368 | #define wmb() asm volatile("sfence" ::: "memory") | |
369 | #endif | |
370 | ||
371 | /** | |
372 | * read_barrier_depends - Flush all pending reads that subsequents reads | |
373 | * depend on. | |
374 | * | |
375 | * No data-dependent reads from memory-like regions are ever reordered | |
376 | * over this barrier. All reads preceding this primitive are guaranteed | |
377 | * to access memory (but not necessarily other CPUs' caches) before any | |
378 | * reads following this primitive that depend on the data return by | |
379 | * any of the preceding reads. This primitive is much lighter weight than | |
380 | * rmb() on most CPUs, and is never heavier weight than is | |
381 | * rmb(). | |
382 | * | |
383 | * These ordering constraints are respected by both the local CPU | |
384 | * and the compiler. | |
385 | * | |
386 | * Ordering is not guaranteed by anything other than these primitives, | |
387 | * not even by data dependencies. See the documentation for | |
388 | * memory_barrier() for examples and URLs to more information. | |
389 | * | |
390 | * For example, the following code would force ordering (the initial | |
391 | * value of "a" is zero, "b" is one, and "p" is "&a"): | |
392 | * | |
393 | * <programlisting> | |
394 | * CPU 0 CPU 1 | |
395 | * | |
396 | * b = 2; | |
397 | * memory_barrier(); | |
398 | * p = &b; q = p; | |
399 | * read_barrier_depends(); | |
400 | * d = *q; | |
401 | * </programlisting> | |
402 | * | |
403 | * because the read of "*q" depends on the read of "p" and these | |
404 | * two reads are separated by a read_barrier_depends(). However, | |
405 | * the following code, with the same initial values for "a" and "b": | |
406 | * | |
407 | * <programlisting> | |
408 | * CPU 0 CPU 1 | |
409 | * | |
410 | * a = 2; | |
411 | * memory_barrier(); | |
412 | * b = 3; y = b; | |
413 | * read_barrier_depends(); | |
414 | * x = a; | |
415 | * </programlisting> | |
416 | * | |
417 | * does not enforce ordering, since there is no data dependency between | |
418 | * the read of "a" and the read of "b". Therefore, on some CPUs, such | |
419 | * as Alpha, "y" could be set to 3 and "x" to 0. Use rmb() | |
420 | * in cases like this where there are no data dependencies. | |
421 | **/ | |
422 | ||
423 | #define read_barrier_depends() do { } while (0) | |
424 | ||
425 | #ifdef CONFIG_SMP | |
426 | #define smp_mb() mb() | |
427 | #ifdef CONFIG_X86_PPRO_FENCE | |
428 | # define smp_rmb() rmb() | |
429 | #else | |
430 | # define smp_rmb() barrier() | |
431 | #endif | |
432 | #ifdef CONFIG_X86_OOSTORE | |
433 | # define smp_wmb() wmb() | |
434 | #else | |
435 | # define smp_wmb() barrier() | |
436 | #endif | |
437 | #define smp_read_barrier_depends() read_barrier_depends() | |
c5386c20 | 438 | #define set_mb(var, value) do { (void)xchg(&var, value); } while (0) |
833d8469 GOC |
439 | #else |
440 | #define smp_mb() barrier() | |
441 | #define smp_rmb() barrier() | |
442 | #define smp_wmb() barrier() | |
443 | #define smp_read_barrier_depends() do { } while (0) | |
444 | #define set_mb(var, value) do { var = value; barrier(); } while (0) | |
445 | #endif | |
446 | ||
fde1b3fa AK |
447 | /* |
448 | * Stop RDTSC speculation. This is needed when you need to use RDTSC | |
449 | * (or get_cycles or vread that possibly accesses the TSC) in a defined | |
450 | * code region. | |
451 | * | |
452 | * (Could use an alternative three way for this if there was one.) | |
453 | */ | |
12448293 | 454 | static __always_inline void rdtsc_barrier(void) |
fde1b3fa AK |
455 | { |
456 | alternative(ASM_NOP3, "mfence", X86_FEATURE_MFENCE_RDTSC); | |
457 | alternative(ASM_NOP3, "lfence", X86_FEATURE_LFENCE_RDTSC); | |
458 | } | |
833d8469 | 459 | |
1965aae3 | 460 | #endif /* _ASM_X86_SYSTEM_H */ |