]>
Commit | Line | Data |
---|---|---|
1eeaed76 RM |
1 | /* |
2 | * Copyright (C) 1994 Linus Torvalds | |
3 | * | |
4 | * Pentium III FXSR, SSE support | |
5 | * General FPU state handling cleanups | |
6 | * Gareth Hughes <gareth@valinux.com>, May 2000 | |
7 | * x86-64 work by Andi Kleen 2002 | |
8 | */ | |
9 | ||
1965aae3 PA |
10 | #ifndef _ASM_X86_I387_H |
11 | #define _ASM_X86_I387_H | |
1eeaed76 | 12 | |
3b0d6596 HX |
13 | #ifndef __ASSEMBLY__ |
14 | ||
1eeaed76 RM |
15 | #include <linux/sched.h> |
16 | #include <linux/kernel_stat.h> | |
17 | #include <linux/regset.h> | |
e4914012 | 18 | #include <linux/hardirq.h> |
86603283 | 19 | #include <linux/slab.h> |
92c37fa3 | 20 | #include <asm/asm.h> |
c9775b4c | 21 | #include <asm/cpufeature.h> |
1eeaed76 RM |
22 | #include <asm/processor.h> |
23 | #include <asm/sigcontext.h> | |
24 | #include <asm/user.h> | |
25 | #include <asm/uaccess.h> | |
dc1e35c6 | 26 | #include <asm/xsave.h> |
1eeaed76 | 27 | |
3c1c7f10 | 28 | extern unsigned int sig_xstate_size; |
1eeaed76 | 29 | extern void fpu_init(void); |
1eeaed76 | 30 | extern void mxcsr_feature_mask_init(void); |
aa283f49 | 31 | extern int init_fpu(struct task_struct *child); |
1eeaed76 | 32 | extern asmlinkage void math_state_restore(void); |
e6e9cac8 | 33 | extern void __math_state_restore(void); |
61c4628b | 34 | extern void init_thread_xstate(void); |
36454936 | 35 | extern int dump_fpu(struct pt_regs *, struct user_i387_struct *); |
1eeaed76 RM |
36 | |
37 | extern user_regset_active_fn fpregs_active, xfpregs_active; | |
5b3efd50 SS |
38 | extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get, |
39 | xstateregs_get; | |
40 | extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set, | |
41 | xstateregs_set; | |
42 | ||
43 | /* | |
44 | * xstateregs_active == fpregs_active. Please refer to the comment | |
45 | * at the definition of fpregs_active. | |
46 | */ | |
47 | #define xstateregs_active fpregs_active | |
1eeaed76 | 48 | |
c37b5efe | 49 | extern struct _fpx_sw_bytes fx_sw_reserved; |
1eeaed76 | 50 | #ifdef CONFIG_IA32_EMULATION |
3c1c7f10 | 51 | extern unsigned int sig_xstate_ia32_size; |
c37b5efe | 52 | extern struct _fpx_sw_bytes fx_sw_reserved_ia32; |
1eeaed76 | 53 | struct _fpstate_ia32; |
ab513701 SS |
54 | struct _xstate_ia32; |
55 | extern int save_i387_xstate_ia32(void __user *buf); | |
56 | extern int restore_i387_xstate_ia32(void __user *buf); | |
1eeaed76 RM |
57 | #endif |
58 | ||
b359e8a4 SS |
59 | #define X87_FSW_ES (1 << 7) /* Exception Summary */ |
60 | ||
29104e10 SS |
61 | static __always_inline __pure bool use_xsaveopt(void) |
62 | { | |
63 | return 0; | |
64 | } | |
65 | ||
c9775b4c | 66 | static __always_inline __pure bool use_xsave(void) |
c9ad4882 | 67 | { |
c9775b4c | 68 | return static_cpu_has(X86_FEATURE_XSAVE); |
c9ad4882 AK |
69 | } |
70 | ||
29104e10 SS |
71 | extern void __sanitize_i387_state(struct task_struct *); |
72 | ||
73 | static inline void sanitize_i387_state(struct task_struct *tsk) | |
74 | { | |
75 | if (!use_xsaveopt()) | |
76 | return; | |
77 | __sanitize_i387_state(tsk); | |
78 | } | |
79 | ||
1eeaed76 RM |
80 | #ifdef CONFIG_X86_64 |
81 | ||
82 | /* Ignore delayed exceptions from user space */ | |
83 | static inline void tolerant_fwait(void) | |
84 | { | |
85 | asm volatile("1: fwait\n" | |
86 | "2:\n" | |
affe6637 | 87 | _ASM_EXTABLE(1b, 2b)); |
1eeaed76 RM |
88 | } |
89 | ||
b359e8a4 | 90 | static inline int fxrstor_checking(struct i387_fxsave_struct *fx) |
1eeaed76 RM |
91 | { |
92 | int err; | |
93 | ||
94 | asm volatile("1: rex64/fxrstor (%[fx])\n\t" | |
95 | "2:\n" | |
96 | ".section .fixup,\"ax\"\n" | |
97 | "3: movl $-1,%[err]\n" | |
98 | " jmp 2b\n" | |
99 | ".previous\n" | |
affe6637 | 100 | _ASM_EXTABLE(1b, 3b) |
1eeaed76 | 101 | : [err] "=r" (err) |
4ecf4584 | 102 | #if 0 /* See comment in fxsave() below. */ |
1eeaed76 RM |
103 | : [fx] "r" (fx), "m" (*fx), "0" (0)); |
104 | #else | |
105 | : [fx] "cdaSDb" (fx), "m" (*fx), "0" (0)); | |
106 | #endif | |
1eeaed76 RM |
107 | return err; |
108 | } | |
109 | ||
1eeaed76 RM |
110 | /* AMD CPUs don't save/restore FDP/FIP/FOP unless an exception |
111 | is pending. Clear the x87 state here by setting it to fixed | |
112 | values. The kernel data segment can be sometimes 0 and sometimes | |
113 | new user value. Both should be ok. | |
114 | Use the PDA as safe address because it should be already in L1. */ | |
86603283 | 115 | static inline void fpu_clear(struct fpu *fpu) |
1eeaed76 | 116 | { |
86603283 AK |
117 | struct xsave_struct *xstate = &fpu->state->xsave; |
118 | struct i387_fxsave_struct *fx = &fpu->state->fxsave; | |
b359e8a4 SS |
119 | |
120 | /* | |
121 | * xsave header may indicate the init state of the FP. | |
122 | */ | |
c9ad4882 | 123 | if (use_xsave() && |
b359e8a4 SS |
124 | !(xstate->xsave_hdr.xstate_bv & XSTATE_FP)) |
125 | return; | |
126 | ||
1eeaed76 | 127 | if (unlikely(fx->swd & X87_FSW_ES)) |
affe6637 | 128 | asm volatile("fnclex"); |
1eeaed76 | 129 | alternative_input(ASM_NOP8 ASM_NOP2, |
affe6637 JP |
130 | " emms\n" /* clear stack tags */ |
131 | " fildl %%gs:0", /* load to clear state */ | |
132 | X86_FEATURE_FXSAVE_LEAK); | |
1eeaed76 RM |
133 | } |
134 | ||
86603283 AK |
135 | static inline void clear_fpu_state(struct task_struct *tsk) |
136 | { | |
137 | fpu_clear(&tsk->thread.fpu); | |
138 | } | |
139 | ||
c37b5efe | 140 | static inline int fxsave_user(struct i387_fxsave_struct __user *fx) |
1eeaed76 RM |
141 | { |
142 | int err; | |
143 | ||
144 | asm volatile("1: rex64/fxsave (%[fx])\n\t" | |
145 | "2:\n" | |
146 | ".section .fixup,\"ax\"\n" | |
147 | "3: movl $-1,%[err]\n" | |
148 | " jmp 2b\n" | |
149 | ".previous\n" | |
affe6637 | 150 | _ASM_EXTABLE(1b, 3b) |
1eeaed76 | 151 | : [err] "=r" (err), "=m" (*fx) |
4ecf4584 | 152 | #if 0 /* See comment in fxsave() below. */ |
1eeaed76 RM |
153 | : [fx] "r" (fx), "0" (0)); |
154 | #else | |
155 | : [fx] "cdaSDb" (fx), "0" (0)); | |
156 | #endif | |
affe6637 JP |
157 | if (unlikely(err) && |
158 | __clear_user(fx, sizeof(struct i387_fxsave_struct))) | |
1eeaed76 RM |
159 | err = -EFAULT; |
160 | /* No need to clear here because the caller clears USED_MATH */ | |
161 | return err; | |
162 | } | |
163 | ||
86603283 | 164 | static inline void fpu_fxsave(struct fpu *fpu) |
1eeaed76 RM |
165 | { |
166 | /* Using "rex64; fxsave %0" is broken because, if the memory operand | |
167 | uses any extended registers for addressing, a second REX prefix | |
168 | will be generated (to the assembler, rex64 followed by semicolon | |
169 | is a separate instruction), and hence the 64-bitness is lost. */ | |
170 | #if 0 | |
171 | /* Using "fxsaveq %0" would be the ideal choice, but is only supported | |
172 | starting with gas 2.16. */ | |
173 | __asm__ __volatile__("fxsaveq %0" | |
86603283 | 174 | : "=m" (fpu->state->fxsave)); |
1eeaed76 RM |
175 | #elif 0 |
176 | /* Using, as a workaround, the properly prefixed form below isn't | |
177 | accepted by any binutils version so far released, complaining that | |
178 | the same type of prefix is used twice if an extended register is | |
179 | needed for addressing (fix submitted to mainline 2005-11-21). */ | |
180 | __asm__ __volatile__("rex64/fxsave %0" | |
86603283 | 181 | : "=m" (fpu->state->fxsave)); |
1eeaed76 RM |
182 | #else |
183 | /* This, however, we can work around by forcing the compiler to select | |
184 | an addressing mode that doesn't require extended registers. */ | |
61c4628b | 185 | __asm__ __volatile__("rex64/fxsave (%1)" |
86603283 AK |
186 | : "=m" (fpu->state->fxsave) |
187 | : "cdaSDb" (&fpu->state->fxsave)); | |
1eeaed76 | 188 | #endif |
b359e8a4 SS |
189 | } |
190 | ||
86603283 | 191 | static inline void fpu_save_init(struct fpu *fpu) |
b359e8a4 | 192 | { |
c9ad4882 | 193 | if (use_xsave()) |
86603283 | 194 | fpu_xsave(fpu); |
b359e8a4 | 195 | else |
86603283 | 196 | fpu_fxsave(fpu); |
b359e8a4 | 197 | |
86603283 AK |
198 | fpu_clear(fpu); |
199 | } | |
200 | ||
201 | static inline void __save_init_fpu(struct task_struct *tsk) | |
202 | { | |
203 | fpu_save_init(&tsk->thread.fpu); | |
1eeaed76 RM |
204 | task_thread_info(tsk)->status &= ~TS_USEDFPU; |
205 | } | |
206 | ||
1eeaed76 RM |
207 | #else /* CONFIG_X86_32 */ |
208 | ||
ab9e1858 | 209 | #ifdef CONFIG_MATH_EMULATION |
86603283 | 210 | extern void finit_soft_fpu(struct i387_soft_struct *soft); |
ab9e1858 | 211 | #else |
86603283 | 212 | static inline void finit_soft_fpu(struct i387_soft_struct *soft) {} |
ab9e1858 | 213 | #endif |
e8a496ac | 214 | |
1eeaed76 RM |
215 | static inline void tolerant_fwait(void) |
216 | { | |
217 | asm volatile("fnclex ; fwait"); | |
218 | } | |
219 | ||
34ba476a JS |
220 | /* perform fxrstor iff the processor has extended states, otherwise frstor */ |
221 | static inline int fxrstor_checking(struct i387_fxsave_struct *fx) | |
1eeaed76 RM |
222 | { |
223 | /* | |
224 | * The "nop" is needed to make the instructions the same | |
225 | * length. | |
226 | */ | |
227 | alternative_input( | |
228 | "nop ; frstor %1", | |
229 | "fxrstor %1", | |
230 | X86_FEATURE_FXSR, | |
34ba476a JS |
231 | "m" (*fx)); |
232 | ||
fcb2ac5b | 233 | return 0; |
1eeaed76 RM |
234 | } |
235 | ||
236 | /* We need a safe address that is cheap to find and that is already | |
237 | in L1 during context switch. The best choices are unfortunately | |
238 | different for UP and SMP */ | |
239 | #ifdef CONFIG_SMP | |
240 | #define safe_address (__per_cpu_offset[0]) | |
241 | #else | |
242 | #define safe_address (kstat_cpu(0).cpustat.user) | |
243 | #endif | |
244 | ||
245 | /* | |
246 | * These must be called with preempt disabled | |
247 | */ | |
86603283 | 248 | static inline void fpu_save_init(struct fpu *fpu) |
1eeaed76 | 249 | { |
c9ad4882 | 250 | if (use_xsave()) { |
86603283 AK |
251 | struct xsave_struct *xstate = &fpu->state->xsave; |
252 | struct i387_fxsave_struct *fx = &fpu->state->fxsave; | |
b359e8a4 | 253 | |
86603283 | 254 | fpu_xsave(fpu); |
b359e8a4 SS |
255 | |
256 | /* | |
257 | * xsave header may indicate the init state of the FP. | |
258 | */ | |
259 | if (!(xstate->xsave_hdr.xstate_bv & XSTATE_FP)) | |
260 | goto end; | |
261 | ||
262 | if (unlikely(fx->swd & X87_FSW_ES)) | |
263 | asm volatile("fnclex"); | |
264 | ||
265 | /* | |
266 | * we can do a simple return here or be paranoid :) | |
267 | */ | |
268 | goto clear_state; | |
269 | } | |
270 | ||
1eeaed76 RM |
271 | /* Use more nops than strictly needed in case the compiler |
272 | varies code */ | |
273 | alternative_input( | |
274 | "fnsave %[fx] ;fwait;" GENERIC_NOP8 GENERIC_NOP4, | |
275 | "fxsave %[fx]\n" | |
276 | "bt $7,%[fsw] ; jnc 1f ; fnclex\n1:", | |
277 | X86_FEATURE_FXSR, | |
86603283 AK |
278 | [fx] "m" (fpu->state->fxsave), |
279 | [fsw] "m" (fpu->state->fxsave.swd) : "memory"); | |
b359e8a4 | 280 | clear_state: |
1eeaed76 RM |
281 | /* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception |
282 | is pending. Clear the x87 state here by setting it to fixed | |
283 | values. safe_address is a random variable that should be in L1 */ | |
284 | alternative_input( | |
285 | GENERIC_NOP8 GENERIC_NOP2, | |
286 | "emms\n\t" /* clear stack tags */ | |
287 | "fildl %[addr]", /* set F?P to defined value */ | |
288 | X86_FEATURE_FXSAVE_LEAK, | |
289 | [addr] "m" (safe_address)); | |
b359e8a4 | 290 | end: |
86603283 AK |
291 | ; |
292 | } | |
293 | ||
294 | static inline void __save_init_fpu(struct task_struct *tsk) | |
295 | { | |
296 | fpu_save_init(&tsk->thread.fpu); | |
1eeaed76 RM |
297 | task_thread_info(tsk)->status &= ~TS_USEDFPU; |
298 | } | |
299 | ||
86603283 | 300 | |
ab513701 SS |
301 | #endif /* CONFIG_X86_64 */ |
302 | ||
86603283 AK |
303 | static inline int fpu_fxrstor_checking(struct fpu *fpu) |
304 | { | |
305 | return fxrstor_checking(&fpu->state->fxsave); | |
306 | } | |
307 | ||
308 | static inline int fpu_restore_checking(struct fpu *fpu) | |
34ba476a | 309 | { |
c9ad4882 | 310 | if (use_xsave()) |
86603283 | 311 | return fpu_xrstor_checking(fpu); |
34ba476a | 312 | else |
86603283 AK |
313 | return fpu_fxrstor_checking(fpu); |
314 | } | |
315 | ||
316 | static inline int restore_fpu_checking(struct task_struct *tsk) | |
317 | { | |
318 | return fpu_restore_checking(&tsk->thread.fpu); | |
34ba476a JS |
319 | } |
320 | ||
1eeaed76 RM |
321 | /* |
322 | * Signal frame handlers... | |
323 | */ | |
ab513701 SS |
324 | extern int save_i387_xstate(void __user *buf); |
325 | extern int restore_i387_xstate(void __user *buf); | |
1eeaed76 RM |
326 | |
327 | static inline void __unlazy_fpu(struct task_struct *tsk) | |
328 | { | |
329 | if (task_thread_info(tsk)->status & TS_USEDFPU) { | |
330 | __save_init_fpu(tsk); | |
331 | stts(); | |
332 | } else | |
333 | tsk->fpu_counter = 0; | |
334 | } | |
335 | ||
336 | static inline void __clear_fpu(struct task_struct *tsk) | |
337 | { | |
338 | if (task_thread_info(tsk)->status & TS_USEDFPU) { | |
339 | tolerant_fwait(); | |
340 | task_thread_info(tsk)->status &= ~TS_USEDFPU; | |
341 | stts(); | |
342 | } | |
343 | } | |
344 | ||
345 | static inline void kernel_fpu_begin(void) | |
346 | { | |
347 | struct thread_info *me = current_thread_info(); | |
348 | preempt_disable(); | |
349 | if (me->status & TS_USEDFPU) | |
350 | __save_init_fpu(me->task); | |
351 | else | |
352 | clts(); | |
353 | } | |
354 | ||
355 | static inline void kernel_fpu_end(void) | |
356 | { | |
357 | stts(); | |
358 | preempt_enable(); | |
359 | } | |
360 | ||
ae4b688d HY |
361 | static inline bool irq_fpu_usable(void) |
362 | { | |
363 | struct pt_regs *regs; | |
364 | ||
365 | return !in_interrupt() || !(regs = get_irq_regs()) || \ | |
366 | user_mode(regs) || (read_cr0() & X86_CR0_TS); | |
367 | } | |
368 | ||
e4914012 SS |
369 | /* |
370 | * Some instructions like VIA's padlock instructions generate a spurious | |
371 | * DNA fault but don't modify SSE registers. And these instructions | |
0b8c3d5a CE |
372 | * get used from interrupt context as well. To prevent these kernel instructions |
373 | * in interrupt context interacting wrongly with other user/kernel fpu usage, we | |
e4914012 SS |
374 | * should use them only in the context of irq_ts_save/restore() |
375 | */ | |
376 | static inline int irq_ts_save(void) | |
377 | { | |
378 | /* | |
0b8c3d5a CE |
379 | * If in process context and not atomic, we can take a spurious DNA fault. |
380 | * Otherwise, doing clts() in process context requires disabling preemption | |
381 | * or some heavy lifting like kernel_fpu_begin() | |
e4914012 | 382 | */ |
0b8c3d5a | 383 | if (!in_atomic()) |
e4914012 SS |
384 | return 0; |
385 | ||
386 | if (read_cr0() & X86_CR0_TS) { | |
387 | clts(); | |
388 | return 1; | |
389 | } | |
390 | ||
391 | return 0; | |
392 | } | |
393 | ||
394 | static inline void irq_ts_restore(int TS_state) | |
395 | { | |
396 | if (TS_state) | |
397 | stts(); | |
398 | } | |
399 | ||
1eeaed76 RM |
400 | #ifdef CONFIG_X86_64 |
401 | ||
402 | static inline void save_init_fpu(struct task_struct *tsk) | |
403 | { | |
404 | __save_init_fpu(tsk); | |
405 | stts(); | |
406 | } | |
407 | ||
408 | #define unlazy_fpu __unlazy_fpu | |
409 | #define clear_fpu __clear_fpu | |
410 | ||
411 | #else /* CONFIG_X86_32 */ | |
412 | ||
413 | /* | |
414 | * These disable preemption on their own and are safe | |
415 | */ | |
416 | static inline void save_init_fpu(struct task_struct *tsk) | |
417 | { | |
418 | preempt_disable(); | |
419 | __save_init_fpu(tsk); | |
420 | stts(); | |
421 | preempt_enable(); | |
422 | } | |
423 | ||
424 | static inline void unlazy_fpu(struct task_struct *tsk) | |
425 | { | |
426 | preempt_disable(); | |
427 | __unlazy_fpu(tsk); | |
428 | preempt_enable(); | |
429 | } | |
430 | ||
431 | static inline void clear_fpu(struct task_struct *tsk) | |
432 | { | |
433 | preempt_disable(); | |
434 | __clear_fpu(tsk); | |
435 | preempt_enable(); | |
436 | } | |
437 | ||
438 | #endif /* CONFIG_X86_64 */ | |
439 | ||
1eeaed76 RM |
440 | /* |
441 | * i387 state interaction | |
442 | */ | |
443 | static inline unsigned short get_fpu_cwd(struct task_struct *tsk) | |
444 | { | |
445 | if (cpu_has_fxsr) { | |
86603283 | 446 | return tsk->thread.fpu.state->fxsave.cwd; |
1eeaed76 | 447 | } else { |
86603283 | 448 | return (unsigned short)tsk->thread.fpu.state->fsave.cwd; |
1eeaed76 RM |
449 | } |
450 | } | |
451 | ||
452 | static inline unsigned short get_fpu_swd(struct task_struct *tsk) | |
453 | { | |
454 | if (cpu_has_fxsr) { | |
86603283 | 455 | return tsk->thread.fpu.state->fxsave.swd; |
1eeaed76 | 456 | } else { |
86603283 | 457 | return (unsigned short)tsk->thread.fpu.state->fsave.swd; |
1eeaed76 RM |
458 | } |
459 | } | |
460 | ||
461 | static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk) | |
462 | { | |
463 | if (cpu_has_xmm) { | |
86603283 | 464 | return tsk->thread.fpu.state->fxsave.mxcsr; |
1eeaed76 RM |
465 | } else { |
466 | return MXCSR_DEFAULT; | |
467 | } | |
468 | } | |
469 | ||
86603283 AK |
470 | static bool fpu_allocated(struct fpu *fpu) |
471 | { | |
472 | return fpu->state != NULL; | |
473 | } | |
474 | ||
475 | static inline int fpu_alloc(struct fpu *fpu) | |
476 | { | |
477 | if (fpu_allocated(fpu)) | |
478 | return 0; | |
479 | fpu->state = kmem_cache_alloc(task_xstate_cachep, GFP_KERNEL); | |
480 | if (!fpu->state) | |
481 | return -ENOMEM; | |
482 | WARN_ON((unsigned long)fpu->state & 15); | |
483 | return 0; | |
484 | } | |
485 | ||
486 | static inline void fpu_free(struct fpu *fpu) | |
487 | { | |
488 | if (fpu->state) { | |
489 | kmem_cache_free(task_xstate_cachep, fpu->state); | |
490 | fpu->state = NULL; | |
491 | } | |
492 | } | |
493 | ||
494 | static inline void fpu_copy(struct fpu *dst, struct fpu *src) | |
495 | { | |
496 | memcpy(dst->state, src->state, xstate_size); | |
497 | } | |
498 | ||
3b0d6596 HX |
499 | #endif /* __ASSEMBLY__ */ |
500 | ||
501 | #define PSHUFB_XMM5_XMM0 .byte 0x66, 0x0f, 0x38, 0x00, 0xc5 | |
502 | #define PSHUFB_XMM5_XMM6 .byte 0x66, 0x0f, 0x38, 0x00, 0xf5 | |
503 | ||
1965aae3 | 504 | #endif /* _ASM_X86_I387_H */ |