arch/x86/kernel/xsave.c

   1 /*
   2  * xsave/xrstor support.
   3  *
   4  * Author: Suresh Siddha <suresh.b.siddha@intel.com>
   5  */
   6 #include <linux/bootmem.h>
   7 #include <linux/compat.h>
   8 #include <asm/i387.h>
   9 #ifdef CONFIG_IA32_EMULATION
  10 #include <asm/sigcontext32.h>
  11 #endif
  12 #include <asm/xcr.h>
  13
  14 /*
  15  * Supported feature mask by the CPU and the kernel.
  16  */
  17 u64 pcntxt_mask;
  18
  19 struct _fpx_sw_bytes fx_sw_reserved;
  20 #ifdef CONFIG_IA32_EMULATION
  21 struct _fpx_sw_bytes fx_sw_reserved_ia32;
  22 #endif
  23
  24 static unsigned int *xstate_offsets, *xstate_sizes, xstate_features;
  25
  26 /*
  27  * Check for the presence of extended state information in the
  28  * user fpstate pointer in the sigcontext.
  29  */
  30 int check_for_xstate(struct i387_fxsave_struct __user *buf,
  31                      void __user *fpstate,
  32                      struct _fpx_sw_bytes *fx_sw_user)
  33 {
  34         int min_xstate_size = sizeof(struct i387_fxsave_struct) +
  35                               sizeof(struct xsave_hdr_struct);
  36         unsigned int magic2;
  37         int err;
  38
  39         err = __copy_from_user(fx_sw_user, &buf->sw_reserved[0],
  40                                sizeof(struct _fpx_sw_bytes));
  41         if (err)
  42                 return -EFAULT;
  43
  44         /*
  45          * First Magic check failed.
  46          */
  47         if (fx_sw_user->magic1 != FP_XSTATE_MAGIC1)
  48                 return -EINVAL;
  49
  50         /*
  51          * Check for error scenarios.
  52          */
  53         if (fx_sw_user->xstate_size < min_xstate_size ||
  54             fx_sw_user->xstate_size > xstate_size ||
  55             fx_sw_user->xstate_size > fx_sw_user->extended_size)
  56                 return -EINVAL;
  57
  58         err = __get_user(magic2, (__u32 *) (((void *)fpstate) +
  59                                             fx_sw_user->extended_size -
  60                                             FP_XSTATE_MAGIC2_SIZE));
  61         if (err)
  62                 return err;
  63         /*
  64          * Check for the presence of second magic word at the end of memory
  65          * layout. This detects the case where the user just copied the legacy
  66          * fpstate layout with out copying the extended state information
  67          * in the memory layout.
  68          */
  69         if (magic2 != FP_XSTATE_MAGIC2)
  70                 return -EFAULT;
  71
  72         return 0;
  73 }
  74
  75 #ifdef CONFIG_X86_64
  76 /*
  77  * Signal frame handlers.
  78  */
  79
  80 int save_i387_xstate(void __user *buf)
  81 {
  82         struct task_struct *tsk = current;
  83         int err = 0;
  84
  85         if (!access_ok(VERIFY_WRITE, buf, sig_xstate_size))
  86                 return -EACCES;
  87
  88         BUG_ON(sig_xstate_size < xstate_size);
  89
  90         if ((unsigned long)buf % 64)
  91                 printk("save_i387_xstate: bad fpstate %p\n", buf);
  92
  93         if (!used_math())
  94                 return 0;
  95
  96         if (task_thread_info(tsk)->status & TS_USEDFPU) {
  97                 /*
  98                  * Start with clearing the user buffer. This will present a
  99                  * clean context for the bytes not touched by the fxsave/xsave.
 100                  */
 101                 err = __clear_user(buf, sig_xstate_size);
 102                 if (err)
 103                         return err;
 104
 105                 if (use_xsave())
 106                         err = xsave_user(buf);
 107                 else
 108                         err = fxsave_user(buf);
 109
 110                 if (err)
 111                         return err;
 112                 task_thread_info(tsk)->status &= ~TS_USEDFPU;
 113                 stts();
 114         } else {
 115                 if (__copy_to_user(buf, &tsk->thread.fpu.state->fxsave,
 116                                    xstate_size))
 117                         return -1;
 118         }
 119
 120         clear_used_math(); /* trigger finit */
 121
 122         if (use_xsave()) {
 123                 struct _fpstate __user *fx = buf;
 124                 struct _xstate __user *x = buf;
 125                 u64 xstate_bv;
 126
 127                 err = __copy_to_user(&fx->sw_reserved, &fx_sw_reserved,
 128                                      sizeof(struct _fpx_sw_bytes));
 129
 130                 err |= __put_user(FP_XSTATE_MAGIC2,
 131                                   (__u32 __user *) (buf + sig_xstate_size
 132                                                     - FP_XSTATE_MAGIC2_SIZE));
 133
 134                 /*
 135                  * Read the xstate_bv which we copied (directly from the cpu or
 136                  * from the state in task struct) to the user buffers and
 137                  * set the FP/SSE bits.
 138                  */
 139                 err |= __get_user(xstate_bv, &x->xstate_hdr.xstate_bv);
 140
 141                 /*
 142                  * For legacy compatible, we always set FP/SSE bits in the bit
 143                  * vector while saving the state to the user context. This will
 144                  * enable us capturing any changes(during sigreturn) to
 145                  * the FP/SSE bits by the legacy applications which don't touch
 146                  * xstate_bv in the xsave header.
 147                  *
 148                  * xsave aware apps can change the xstate_bv in the xsave
 149                  * header as well as change any contents in the memory layout.
 150                  * xrestore as part of sigreturn will capture all the changes.
 151                  */
 152                 xstate_bv |= XSTATE_FPSSE;
 153
 154                 err |= __put_user(xstate_bv, &x->xstate_hdr.xstate_bv);
 155
 156                 if (err)
 157                         return err;
 158         }
 159
 160         return 1;
 161 }
 162
 163 /*
 164  * Restore the extended state if present. Otherwise, restore the FP/SSE
 165  * state.
 166  */
 167 static int restore_user_xstate(void __user *buf)
 168 {
 169         struct _fpx_sw_bytes fx_sw_user;
 170         u64 mask;
 171         int err;
 172
 173         if (((unsigned long)buf % 64) ||
 174              check_for_xstate(buf, buf, &fx_sw_user))
 175                 goto fx_only;
 176
 177         mask = fx_sw_user.xstate_bv;
 178
 179         /*
 180          * restore the state passed by the user.
 181          */
 182         err = xrestore_user(buf, mask);
 183         if (err)
 184                 return err;
 185
 186         /*
 187          * init the state skipped by the user.
 188          */
 189         mask = pcntxt_mask & ~mask;
 190
 191         xrstor_state(init_xstate_buf, mask);
 192
 193         return 0;
 194
 195 fx_only:
 196         /*
 197          * couldn't find the extended state information in the
 198          * memory layout. Restore just the FP/SSE and init all
 199          * the other extended state.
 200          */
 201         xrstor_state(init_xstate_buf, pcntxt_mask & ~XSTATE_FPSSE);
 202         return fxrstor_checking((__force struct i387_fxsave_struct *)buf);
 203 }
 204
 205 /*
 206  * This restores directly out of user space. Exceptions are handled.
 207  */
 208 int restore_i387_xstate(void __user *buf)
 209 {
 210         struct task_struct *tsk = current;
 211         int err = 0;
 212
 213         if (!buf) {
 214                 if (used_math())
 215                         goto clear;
 216                 return 0;
 217         } else
 218                 if (!access_ok(VERIFY_READ, buf, sig_xstate_size))
 219                         return -EACCES;
 220
 221         if (!used_math()) {
 222                 err = init_fpu(tsk);
 223                 if (err)
 224                         return err;
 225         }
 226
 227         if (!(task_thread_info(current)->status & TS_USEDFPU)) {
 228                 clts();
 229                 task_thread_info(current)->status |= TS_USEDFPU;
 230         }
 231         if (use_xsave())
 232                 err = restore_user_xstate(buf);
 233         else
 234                 err = fxrstor_checking((__force struct i387_fxsave_struct *)
 235                                        buf);
 236         if (unlikely(err)) {
 237                 /*
 238                  * Encountered an error while doing the restore from the
 239                  * user buffer, clear the fpu state.
 240                  */
 241 clear:
 242                 clear_fpu(tsk);
 243                 clear_used_math();
 244         }
 245         return err;
 246 }
 247 #endif
 248
 249 /*
 250  * Prepare the SW reserved portion of the fxsave memory layout, indicating
 251  * the presence of the extended state information in the memory layout
 252  * pointed by the fpstate pointer in the sigcontext.
 253  * This will be saved when ever the FP and extended state context is
 254  * saved on the user stack during the signal handler delivery to the user.
 255  */
 256 static void prepare_fx_sw_frame(void)
 257 {
 258         int size_extended = (xstate_size - sizeof(struct i387_fxsave_struct)) +
 259                              FP_XSTATE_MAGIC2_SIZE;
 260
 261         sig_xstate_size = sizeof(struct _fpstate) + size_extended;
 262
 263 #ifdef CONFIG_IA32_EMULATION
 264         sig_xstate_ia32_size = sizeof(struct _fpstate_ia32) + size_extended;
 265 #endif
 266
 267         memset(&fx_sw_reserved, 0, sizeof(fx_sw_reserved));
 268
 269         fx_sw_reserved.magic1 = FP_XSTATE_MAGIC1;
 270         fx_sw_reserved.extended_size = sig_xstate_size;
 271         fx_sw_reserved.xstate_bv = pcntxt_mask;
 272         fx_sw_reserved.xstate_size = xstate_size;
 273 #ifdef CONFIG_IA32_EMULATION
 274         memcpy(&fx_sw_reserved_ia32, &fx_sw_reserved,
 275                sizeof(struct _fpx_sw_bytes));
 276         fx_sw_reserved_ia32.extended_size = sig_xstate_ia32_size;
 277 #endif
 278 }
 279
 280 /*
 281  * Represents init state for the supported extended state.
 282  */
 283 struct xsave_struct *init_xstate_buf;
 284
 285 #ifdef CONFIG_X86_64
 286 unsigned int sig_xstate_size = sizeof(struct _fpstate);
 287 #endif
 288
 289 /*
 290  * Enable the extended processor state save/restore feature
 291  */
 292 void __cpuinit xsave_init(void)
 293 {
 294         if (!cpu_has_xsave)
 295                 return;
 296
 297         set_in_cr4(X86_CR4_OSXSAVE);
 298
 299         /*
 300          * Enable all the features that the HW is capable of
 301          * and the Linux kernel is aware of.
 302          */
 303         xsetbv(XCR_XFEATURE_ENABLED_MASK, pcntxt_mask);
 304 }
 305
 306 /*
 307  * Record the offsets and sizes of different state managed by the xsave
 308  * memory layout.
 309  */
 310 static void setup_xstate_features(void)
 311 {
 312         int eax, ebx, ecx, edx, leaf = 0x2;
 313
 314         xstate_features = fls64(pcntxt_mask);
 315         xstate_offsets = alloc_bootmem(xstate_features * sizeof(int));
 316         xstate_sizes = alloc_bootmem(xstate_features * sizeof(int));
 317
 318         do {
 319                 cpuid_count(0xd, leaf, &eax, &ebx, &ecx, &edx);
 320
 321                 if (eax == 0)
 322                         break;
 323
 324                 xstate_offsets[leaf] = ebx;
 325                 xstate_sizes[leaf] = eax;
 326
 327                 leaf++;
 328         } while (1);
 329 }
 330
 331 /*
 332  * setup the xstate image representing the init state
 333  */
 334 static void __init setup_xstate_init(void)
 335 {
 336         init_xstate_buf = alloc_bootmem(xstate_size);
 337         init_xstate_buf->i387.mxcsr = MXCSR_DEFAULT;
 338
 339         setup_xstate_features();
 340 }
 341
 342 /*
 343  * Enable and initialize the xsave feature.
 344  */
 345 void __ref xsave_cntxt_init(void)
 346 {
 347         unsigned int eax, ebx, ecx, edx;
 348
 349         cpuid_count(0xd, 0, &eax, &ebx, &ecx, &edx);
 350         pcntxt_mask = eax + ((u64)edx << 32);
 351
 352         if ((pcntxt_mask & XSTATE_FPSSE) != XSTATE_FPSSE) {
 353                 printk(KERN_ERR "FP/SSE not shown under xsave features 0x%llx\n",
 354                        pcntxt_mask);
 355                 BUG();
 356         }
 357
 358         /*
 359          * Support only the state known to OS.
 360          */
 361         pcntxt_mask = pcntxt_mask & XCNTXT_MASK;
 362         xsave_init();
 363
 364         /*
 365          * Recompute the context size for enabled features
 366          */
 367         cpuid_count(0xd, 0, &eax, &ebx, &ecx, &edx);
 368         xstate_size = ebx;
 369
 370         update_regset_xstate_info(xstate_size, pcntxt_mask);
 371         prepare_fx_sw_frame();
 372
 373         setup_xstate_init();
 374
 375         printk(KERN_INFO "xsave/xrstor: enabled xstate_bv 0x%llx, "
 376                "cntxt size 0x%x\n",
 377                pcntxt_mask, xstate_size);
 378 }