arch/x86/kernel/cpu/perf_counter.c

   1 /*
   2  * Performance counter x86 architecture code
   3  *
   4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
   5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
   6  *  Copyright (C) 2009 Jaswinder Singh Rajput
   7  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
   8  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
   9  *
  10  *  For licencing details see kernel-base/COPYING
  11  */
  12
  13 #include <linux/perf_counter.h>
  14 #include <linux/capability.h>
  15 #include <linux/notifier.h>
  16 #include <linux/hardirq.h>
  17 #include <linux/kprobes.h>
  18 #include <linux/module.h>
  19 #include <linux/kdebug.h>
  20 #include <linux/sched.h>
  21 #include <linux/uaccess.h>
  22
  23 #include <asm/apic.h>
  24 #include <asm/stacktrace.h>
  25 #include <asm/nmi.h>
  26
  27 static u64 perf_counter_mask __read_mostly;
  28
  29 struct cpu_hw_counters {
  30         struct perf_counter     *counters[X86_PMC_IDX_MAX];
  31         unsigned long           used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  32         unsigned long           active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  33         unsigned long           interrupts;
  34         int                     enabled;
  35 };
  36
  37 /*
  38  * struct x86_pmu - generic x86 pmu
  39  */
  40 struct x86_pmu {
  41         const char      *name;
  42         int             version;
  43         int             (*handle_irq)(struct pt_regs *);
  44         void            (*disable_all)(void);
  45         void            (*enable_all)(void);
  46         void            (*enable)(struct hw_perf_counter *, int);
  47         void            (*disable)(struct hw_perf_counter *, int);
  48         unsigned        eventsel;
  49         unsigned        perfctr;
  50         u64             (*event_map)(int);
  51         u64             (*raw_event)(u64);
  52         int             max_events;
  53         int             num_counters;
  54         int             num_counters_fixed;
  55         int             counter_bits;
  56         u64             counter_mask;
  57         u64             max_period;
  58         u64             intel_ctrl;
  59 };
  60
  61 static struct x86_pmu x86_pmu __read_mostly;
  62
  63 static DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters) = {
  64         .enabled = 1,
  65 };
  66
  67 /*
  68  * Intel PerfMon v3. Used on Core2 and later.
  69  */
  70 static const u64 intel_perfmon_event_map[] =
  71 {
  72   [PERF_COUNT_HW_CPU_CYCLES]            = 0x003c,
  73   [PERF_COUNT_HW_INSTRUCTIONS]          = 0x00c0,
  74   [PERF_COUNT_HW_CACHE_REFERENCES]      = 0x4f2e,
  75   [PERF_COUNT_HW_CACHE_MISSES]          = 0x412e,
  76   [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]   = 0x00c4,
  77   [PERF_COUNT_HW_BRANCH_MISSES]         = 0x00c5,
  78   [PERF_COUNT_HW_BUS_CYCLES]            = 0x013c,
  79 };
  80
  81 static u64 intel_pmu_event_map(int event)
  82 {
  83         return intel_perfmon_event_map[event];
  84 }
  85
  86 /*
  87  * Generalized hw caching related event table, filled
  88  * in on a per model basis. A value of 0 means
  89  * 'not supported', -1 means 'event makes no sense on
  90  * this CPU', any other value means the raw event
  91  * ID.
  92  */
  93
  94 #define C(x) PERF_COUNT_HW_CACHE_##x
  95
  96 static u64 __read_mostly hw_cache_event_ids
  97                                 [PERF_COUNT_HW_CACHE_MAX]
  98                                 [PERF_COUNT_HW_CACHE_OP_MAX]
  99                                 [PERF_COUNT_HW_CACHE_RESULT_MAX];
 100
 101 static const u64 nehalem_hw_cache_event_ids
 102                                 [PERF_COUNT_HW_CACHE_MAX]
 103                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 104                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 105 {
 106  [ C(L1D) ] = {
 107         [ C(OP_READ) ] = {
 108                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI            */
 109                 [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE         */
 110         },
 111         [ C(OP_WRITE) ] = {
 112                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI            */
 113                 [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE         */
 114         },
 115         [ C(OP_PREFETCH) ] = {
 116                 [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS        */
 117                 [ C(RESULT_MISS)   ] = 0x024e, /* L1D_PREFETCH.MISS            */
 118         },
 119  },
 120  [ C(L1I ) ] = {
 121         [ C(OP_READ) ] = {
 122                 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                    */
 123                 [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                   */
 124         },
 125         [ C(OP_WRITE) ] = {
 126                 [ C(RESULT_ACCESS) ] = -1,
 127                 [ C(RESULT_MISS)   ] = -1,
 128         },
 129         [ C(OP_PREFETCH) ] = {
 130                 [ C(RESULT_ACCESS) ] = 0x0,
 131                 [ C(RESULT_MISS)   ] = 0x0,
 132         },
 133  },
 134  [ C(LL  ) ] = {
 135         [ C(OP_READ) ] = {
 136                 [ C(RESULT_ACCESS) ] = 0x0324, /* L2_RQSTS.LOADS               */
 137                 [ C(RESULT_MISS)   ] = 0x0224, /* L2_RQSTS.LD_MISS             */
 138         },
 139         [ C(OP_WRITE) ] = {
 140                 [ C(RESULT_ACCESS) ] = 0x0c24, /* L2_RQSTS.RFOS                */
 141                 [ C(RESULT_MISS)   ] = 0x0824, /* L2_RQSTS.RFO_MISS            */
 142         },
 143         [ C(OP_PREFETCH) ] = {
 144                 [ C(RESULT_ACCESS) ] = 0x4f2e, /* LLC Reference                */
 145                 [ C(RESULT_MISS)   ] = 0x412e, /* LLC Misses                   */
 146         },
 147  },
 148  [ C(DTLB) ] = {
 149         [ C(OP_READ) ] = {
 150                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI   (alias)  */
 151                 [ C(RESULT_MISS)   ] = 0x0108, /* DTLB_LOAD_MISSES.ANY         */
 152         },
 153         [ C(OP_WRITE) ] = {
 154                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI   (alias)  */
 155                 [ C(RESULT_MISS)   ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS  */
 156         },
 157         [ C(OP_PREFETCH) ] = {
 158                 [ C(RESULT_ACCESS) ] = 0x0,
 159                 [ C(RESULT_MISS)   ] = 0x0,
 160         },
 161  },
 162  [ C(ITLB) ] = {
 163         [ C(OP_READ) ] = {
 164                 [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P           */
 165                 [ C(RESULT_MISS)   ] = 0x20c8, /* ITLB_MISS_RETIRED            */
 166         },
 167         [ C(OP_WRITE) ] = {
 168                 [ C(RESULT_ACCESS) ] = -1,
 169                 [ C(RESULT_MISS)   ] = -1,
 170         },
 171         [ C(OP_PREFETCH) ] = {
 172                 [ C(RESULT_ACCESS) ] = -1,
 173                 [ C(RESULT_MISS)   ] = -1,
 174         },
 175  },
 176  [ C(BPU ) ] = {
 177         [ C(OP_READ) ] = {
 178                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
 179                 [ C(RESULT_MISS)   ] = 0x03e8, /* BPU_CLEARS.ANY               */
 180         },
 181         [ C(OP_WRITE) ] = {
 182                 [ C(RESULT_ACCESS) ] = -1,
 183                 [ C(RESULT_MISS)   ] = -1,
 184         },
 185         [ C(OP_PREFETCH) ] = {
 186                 [ C(RESULT_ACCESS) ] = -1,
 187                 [ C(RESULT_MISS)   ] = -1,
 188         },
 189  },
 190 };
 191
 192 static const u64 core2_hw_cache_event_ids
 193                                 [PERF_COUNT_HW_CACHE_MAX]
 194                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 195                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 196 {
 197  [ C(L1D) ] = {
 198         [ C(OP_READ) ] = {
 199                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI          */
 200                 [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE       */
 201         },
 202         [ C(OP_WRITE) ] = {
 203                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI          */
 204                 [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE       */
 205         },
 206         [ C(OP_PREFETCH) ] = {
 207                 [ C(RESULT_ACCESS) ] = 0x104e, /* L1D_PREFETCH.REQUESTS      */
 208                 [ C(RESULT_MISS)   ] = 0,
 209         },
 210  },
 211  [ C(L1I ) ] = {
 212         [ C(OP_READ) ] = {
 213                 [ C(RESULT_ACCESS) ] = 0x0080, /* L1I.READS                  */
 214                 [ C(RESULT_MISS)   ] = 0x0081, /* L1I.MISSES                 */
 215         },
 216         [ C(OP_WRITE) ] = {
 217                 [ C(RESULT_ACCESS) ] = -1,
 218                 [ C(RESULT_MISS)   ] = -1,
 219         },
 220         [ C(OP_PREFETCH) ] = {
 221                 [ C(RESULT_ACCESS) ] = 0,
 222                 [ C(RESULT_MISS)   ] = 0,
 223         },
 224  },
 225  [ C(LL  ) ] = {
 226         [ C(OP_READ) ] = {
 227                 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
 228                 [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
 229         },
 230         [ C(OP_WRITE) ] = {
 231                 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
 232                 [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
 233         },
 234         [ C(OP_PREFETCH) ] = {
 235                 [ C(RESULT_ACCESS) ] = 0,
 236                 [ C(RESULT_MISS)   ] = 0,
 237         },
 238  },
 239  [ C(DTLB) ] = {
 240         [ C(OP_READ) ] = {
 241                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI  (alias) */
 242                 [ C(RESULT_MISS)   ] = 0x0208, /* DTLB_MISSES.MISS_LD        */
 243         },
 244         [ C(OP_WRITE) ] = {
 245                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI  (alias) */
 246                 [ C(RESULT_MISS)   ] = 0x0808, /* DTLB_MISSES.MISS_ST        */
 247         },
 248         [ C(OP_PREFETCH) ] = {
 249                 [ C(RESULT_ACCESS) ] = 0,
 250                 [ C(RESULT_MISS)   ] = 0,
 251         },
 252  },
 253  [ C(ITLB) ] = {
 254         [ C(OP_READ) ] = {
 255                 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
 256                 [ C(RESULT_MISS)   ] = 0x1282, /* ITLBMISSES                 */
 257         },
 258         [ C(OP_WRITE) ] = {
 259                 [ C(RESULT_ACCESS) ] = -1,
 260                 [ C(RESULT_MISS)   ] = -1,
 261         },
 262         [ C(OP_PREFETCH) ] = {
 263                 [ C(RESULT_ACCESS) ] = -1,
 264                 [ C(RESULT_MISS)   ] = -1,
 265         },
 266  },
 267  [ C(BPU ) ] = {
 268         [ C(OP_READ) ] = {
 269                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
 270                 [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
 271         },
 272         [ C(OP_WRITE) ] = {
 273                 [ C(RESULT_ACCESS) ] = -1,
 274                 [ C(RESULT_MISS)   ] = -1,
 275         },
 276         [ C(OP_PREFETCH) ] = {
 277                 [ C(RESULT_ACCESS) ] = -1,
 278                 [ C(RESULT_MISS)   ] = -1,
 279         },
 280  },
 281 };
 282
 283 static const u64 atom_hw_cache_event_ids
 284                                 [PERF_COUNT_HW_CACHE_MAX]
 285                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 286                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 287 {
 288  [ C(L1D) ] = {
 289         [ C(OP_READ) ] = {
 290                 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE.LD               */
 291                 [ C(RESULT_MISS)   ] = 0,
 292         },
 293         [ C(OP_WRITE) ] = {
 294                 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE.ST               */
 295                 [ C(RESULT_MISS)   ] = 0,
 296         },
 297         [ C(OP_PREFETCH) ] = {
 298                 [ C(RESULT_ACCESS) ] = 0x0,
 299                 [ C(RESULT_MISS)   ] = 0,
 300         },
 301  },
 302  [ C(L1I ) ] = {
 303         [ C(OP_READ) ] = {
 304                 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                  */
 305                 [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                 */
 306         },
 307         [ C(OP_WRITE) ] = {
 308                 [ C(RESULT_ACCESS) ] = -1,
 309                 [ C(RESULT_MISS)   ] = -1,
 310         },
 311         [ C(OP_PREFETCH) ] = {
 312                 [ C(RESULT_ACCESS) ] = 0,
 313                 [ C(RESULT_MISS)   ] = 0,
 314         },
 315  },
 316  [ C(LL  ) ] = {
 317         [ C(OP_READ) ] = {
 318                 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
 319                 [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
 320         },
 321         [ C(OP_WRITE) ] = {
 322                 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
 323                 [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
 324         },
 325         [ C(OP_PREFETCH) ] = {
 326                 [ C(RESULT_ACCESS) ] = 0,
 327                 [ C(RESULT_MISS)   ] = 0,
 328         },
 329  },
 330  [ C(DTLB) ] = {
 331         [ C(OP_READ) ] = {
 332                 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE_LD.MESI  (alias) */
 333                 [ C(RESULT_MISS)   ] = 0x0508, /* DTLB_MISSES.MISS_LD        */
 334         },
 335         [ C(OP_WRITE) ] = {
 336                 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE_ST.MESI  (alias) */
 337                 [ C(RESULT_MISS)   ] = 0x0608, /* DTLB_MISSES.MISS_ST        */
 338         },
 339         [ C(OP_PREFETCH) ] = {
 340                 [ C(RESULT_ACCESS) ] = 0,
 341                 [ C(RESULT_MISS)   ] = 0,
 342         },
 343  },
 344  [ C(ITLB) ] = {
 345         [ C(OP_READ) ] = {
 346                 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
 347                 [ C(RESULT_MISS)   ] = 0x0282, /* ITLB.MISSES                */
 348         },
 349         [ C(OP_WRITE) ] = {
 350                 [ C(RESULT_ACCESS) ] = -1,
 351                 [ C(RESULT_MISS)   ] = -1,
 352         },
 353         [ C(OP_PREFETCH) ] = {
 354                 [ C(RESULT_ACCESS) ] = -1,
 355                 [ C(RESULT_MISS)   ] = -1,
 356         },
 357  },
 358  [ C(BPU ) ] = {
 359         [ C(OP_READ) ] = {
 360                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
 361                 [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
 362         },
 363         [ C(OP_WRITE) ] = {
 364                 [ C(RESULT_ACCESS) ] = -1,
 365                 [ C(RESULT_MISS)   ] = -1,
 366         },
 367         [ C(OP_PREFETCH) ] = {
 368                 [ C(RESULT_ACCESS) ] = -1,
 369                 [ C(RESULT_MISS)   ] = -1,
 370         },
 371  },
 372 };
 373
 374 static u64 intel_pmu_raw_event(u64 event)
 375 {
 376 #define CORE_EVNTSEL_EVENT_MASK         0x000000FFULL
 377 #define CORE_EVNTSEL_UNIT_MASK          0x0000FF00ULL
 378 #define CORE_EVNTSEL_EDGE_MASK          0x00040000ULL
 379 #define CORE_EVNTSEL_INV_MASK           0x00800000ULL
 380 #define CORE_EVNTSEL_COUNTER_MASK       0xFF000000ULL
 381
 382 #define CORE_EVNTSEL_MASK               \
 383         (CORE_EVNTSEL_EVENT_MASK |      \
 384          CORE_EVNTSEL_UNIT_MASK  |      \
 385          CORE_EVNTSEL_EDGE_MASK  |      \
 386          CORE_EVNTSEL_INV_MASK  |       \
 387          CORE_EVNTSEL_COUNTER_MASK)
 388
 389         return event & CORE_EVNTSEL_MASK;
 390 }
 391
 392 static const u64 amd_0f_hw_cache_event_ids
 393                                 [PERF_COUNT_HW_CACHE_MAX]
 394                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 395                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 396 {
 397  [ C(L1D) ] = {
 398         [ C(OP_READ) ] = {
 399                 [ C(RESULT_ACCESS) ] = 0,
 400                 [ C(RESULT_MISS)   ] = 0,
 401         },
 402         [ C(OP_WRITE) ] = {
 403                 [ C(RESULT_ACCESS) ] = 0,
 404                 [ C(RESULT_MISS)   ] = 0,
 405         },
 406         [ C(OP_PREFETCH) ] = {
 407                 [ C(RESULT_ACCESS) ] = 0,
 408                 [ C(RESULT_MISS)   ] = 0,
 409         },
 410  },
 411  [ C(L1I ) ] = {
 412         [ C(OP_READ) ] = {
 413                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction cache fetches  */
 414                 [ C(RESULT_MISS)   ] = 0x0081, /* Instruction cache misses   */
 415         },
 416         [ C(OP_WRITE) ] = {
 417                 [ C(RESULT_ACCESS) ] = -1,
 418                 [ C(RESULT_MISS)   ] = -1,
 419         },
 420         [ C(OP_PREFETCH) ] = {
 421                 [ C(RESULT_ACCESS) ] = 0,
 422                 [ C(RESULT_MISS)   ] = 0,
 423         },
 424  },
 425  [ C(LL  ) ] = {
 426         [ C(OP_READ) ] = {
 427                 [ C(RESULT_ACCESS) ] = 0,
 428                 [ C(RESULT_MISS)   ] = 0,
 429         },
 430         [ C(OP_WRITE) ] = {
 431                 [ C(RESULT_ACCESS) ] = 0,
 432                 [ C(RESULT_MISS)   ] = 0,
 433         },
 434         [ C(OP_PREFETCH) ] = {
 435                 [ C(RESULT_ACCESS) ] = 0,
 436                 [ C(RESULT_MISS)   ] = 0,
 437         },
 438  },
 439  [ C(DTLB) ] = {
 440         [ C(OP_READ) ] = {
 441                 [ C(RESULT_ACCESS) ] = 0,
 442                 [ C(RESULT_MISS)   ] = 0,
 443         },
 444         [ C(OP_WRITE) ] = {
 445                 [ C(RESULT_ACCESS) ] = 0,
 446                 [ C(RESULT_MISS)   ] = 0,
 447         },
 448         [ C(OP_PREFETCH) ] = {
 449                 [ C(RESULT_ACCESS) ] = 0,
 450                 [ C(RESULT_MISS)   ] = 0,
 451         },
 452  },
 453  [ C(ITLB) ] = {
 454         [ C(OP_READ) ] = {
 455                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes        */
 456                 [ C(RESULT_MISS)   ] = 0x0085, /* Instr. fetch ITLB misses   */
 457         },
 458         [ C(OP_WRITE) ] = {
 459                 [ C(RESULT_ACCESS) ] = -1,
 460                 [ C(RESULT_MISS)   ] = -1,
 461         },
 462         [ C(OP_PREFETCH) ] = {
 463                 [ C(RESULT_ACCESS) ] = -1,
 464                 [ C(RESULT_MISS)   ] = -1,
 465         },
 466  },
 467  [ C(BPU ) ] = {
 468         [ C(OP_READ) ] = {
 469                 [ C(RESULT_ACCESS) ] = 0x00c2, /* Retired Branch Instr.      */
 470                 [ C(RESULT_MISS)   ] = 0x00c3, /* Retired Mispredicted BI    */
 471         },
 472         [ C(OP_WRITE) ] = {
 473                 [ C(RESULT_ACCESS) ] = -1,
 474                 [ C(RESULT_MISS)   ] = -1,
 475         },
 476         [ C(OP_PREFETCH) ] = {
 477                 [ C(RESULT_ACCESS) ] = -1,
 478                 [ C(RESULT_MISS)   ] = -1,
 479         },
 480  },
 481 };
 482
 483 /*
 484  * AMD Performance Monitor K7 and later.
 485  */
 486 static const u64 amd_perfmon_event_map[] =
 487 {
 488   [PERF_COUNT_HW_CPU_CYCLES]            = 0x0076,
 489   [PERF_COUNT_HW_INSTRUCTIONS]          = 0x00c0,
 490   [PERF_COUNT_HW_CACHE_REFERENCES]      = 0x0080,
 491   [PERF_COUNT_HW_CACHE_MISSES]          = 0x0081,
 492   [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]   = 0x00c4,
 493   [PERF_COUNT_HW_BRANCH_MISSES]         = 0x00c5,
 494 };
 495
 496 static u64 amd_pmu_event_map(int event)
 497 {
 498         return amd_perfmon_event_map[event];
 499 }
 500
 501 static u64 amd_pmu_raw_event(u64 event)
 502 {
 503 #define K7_EVNTSEL_EVENT_MASK   0x7000000FFULL
 504 #define K7_EVNTSEL_UNIT_MASK    0x00000FF00ULL
 505 #define K7_EVNTSEL_EDGE_MASK    0x000040000ULL
 506 #define K7_EVNTSEL_INV_MASK     0x000800000ULL
 507 #define K7_EVNTSEL_COUNTER_MASK 0x0FF000000ULL
 508
 509 #define K7_EVNTSEL_MASK                 \
 510         (K7_EVNTSEL_EVENT_MASK |        \
 511          K7_EVNTSEL_UNIT_MASK  |        \
 512          K7_EVNTSEL_EDGE_MASK  |        \
 513          K7_EVNTSEL_INV_MASK   |        \
 514          K7_EVNTSEL_COUNTER_MASK)
 515
 516         return event & K7_EVNTSEL_MASK;
 517 }
 518
 519 /*
 520  * Propagate counter elapsed time into the generic counter.
 521  * Can only be executed on the CPU where the counter is active.
 522  * Returns the delta events processed.
 523  */
 524 static u64
 525 x86_perf_counter_update(struct perf_counter *counter,
 526                         struct hw_perf_counter *hwc, int idx)
 527 {
 528         int shift = 64 - x86_pmu.counter_bits;
 529         u64 prev_raw_count, new_raw_count;
 530         s64 delta;
 531
 532         /*
 533          * Careful: an NMI might modify the previous counter value.
 534          *
 535          * Our tactic to handle this is to first atomically read and
 536          * exchange a new raw count - then add that new-prev delta
 537          * count to the generic counter atomically:
 538          */
 539 again:
 540         prev_raw_count = atomic64_read(&hwc->prev_count);
 541         rdmsrl(hwc->counter_base + idx, new_raw_count);
 542
 543         if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
 544                                         new_raw_count) != prev_raw_count)
 545                 goto again;
 546
 547         /*
 548          * Now we have the new raw value and have updated the prev
 549          * timestamp already. We can now calculate the elapsed delta
 550          * (counter-)time and add that to the generic counter.
 551          *
 552          * Careful, not all hw sign-extends above the physical width
 553          * of the count.
 554          */
 555         delta = (new_raw_count << shift) - (prev_raw_count << shift);
 556         delta >>= shift;
 557
 558         atomic64_add(delta, &counter->count);
 559         atomic64_sub(delta, &hwc->period_left);
 560
 561         return new_raw_count;
 562 }
 563
 564 static atomic_t active_counters;
 565 static DEFINE_MUTEX(pmc_reserve_mutex);
 566
 567 static bool reserve_pmc_hardware(void)
 568 {
 569         int i;
 570
 571         if (nmi_watchdog == NMI_LOCAL_APIC)
 572                 disable_lapic_nmi_watchdog();
 573
 574         for (i = 0; i < x86_pmu.num_counters; i++) {
 575                 if (!reserve_perfctr_nmi(x86_pmu.perfctr + i))
 576                         goto perfctr_fail;
 577         }
 578
 579         for (i = 0; i < x86_pmu.num_counters; i++) {
 580                 if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
 581                         goto eventsel_fail;
 582         }
 583
 584         return true;
 585
 586 eventsel_fail:
 587         for (i--; i >= 0; i--)
 588                 release_evntsel_nmi(x86_pmu.eventsel + i);
 589
 590         i = x86_pmu.num_counters;
 591
 592 perfctr_fail:
 593         for (i--; i >= 0; i--)
 594                 release_perfctr_nmi(x86_pmu.perfctr + i);
 595
 596         if (nmi_watchdog == NMI_LOCAL_APIC)
 597                 enable_lapic_nmi_watchdog();
 598
 599         return false;
 600 }
 601
 602 static void release_pmc_hardware(void)
 603 {
 604         int i;
 605
 606         for (i = 0; i < x86_pmu.num_counters; i++) {
 607                 release_perfctr_nmi(x86_pmu.perfctr + i);
 608                 release_evntsel_nmi(x86_pmu.eventsel + i);
 609         }
 610
 611         if (nmi_watchdog == NMI_LOCAL_APIC)
 612                 enable_lapic_nmi_watchdog();
 613 }
 614
 615 static void hw_perf_counter_destroy(struct perf_counter *counter)
 616 {
 617         if (atomic_dec_and_mutex_lock(&active_counters, &pmc_reserve_mutex)) {
 618                 release_pmc_hardware();
 619                 mutex_unlock(&pmc_reserve_mutex);
 620         }
 621 }
 622
 623 static inline int x86_pmu_initialized(void)
 624 {
 625         return x86_pmu.handle_irq != NULL;
 626 }
 627
 628 static inline int
 629 set_ext_hw_attr(struct hw_perf_counter *hwc, struct perf_counter_attr *attr)
 630 {
 631         unsigned int cache_type, cache_op, cache_result;
 632         u64 config, val;
 633
 634         config = attr->config;
 635
 636         cache_type = (config >>  0) & 0xff;
 637         if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
 638                 return -EINVAL;
 639
 640         cache_op = (config >>  8) & 0xff;
 641         if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
 642                 return -EINVAL;
 643
 644         cache_result = (config >> 16) & 0xff;
 645         if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
 646                 return -EINVAL;
 647
 648         val = hw_cache_event_ids[cache_type][cache_op][cache_result];
 649
 650         if (val == 0)
 651                 return -ENOENT;
 652
 653         if (val == -1)
 654                 return -EINVAL;
 655
 656         hwc->config |= val;
 657
 658         return 0;
 659 }
 660
 661 /*
 662  * Setup the hardware configuration for a given attr_type
 663  */
 664 static int __hw_perf_counter_init(struct perf_counter *counter)
 665 {
 666         struct perf_counter_attr *attr = &counter->attr;
 667         struct hw_perf_counter *hwc = &counter->hw;
 668         int err;
 669
 670         if (!x86_pmu_initialized())
 671                 return -ENODEV;
 672
 673         err = 0;
 674         if (!atomic_inc_not_zero(&active_counters)) {
 675                 mutex_lock(&pmc_reserve_mutex);
 676                 if (atomic_read(&active_counters) == 0 && !reserve_pmc_hardware())
 677                         err = -EBUSY;
 678                 else
 679                         atomic_inc(&active_counters);
 680                 mutex_unlock(&pmc_reserve_mutex);
 681         }
 682         if (err)
 683                 return err;
 684
 685         /*
 686          * Generate PMC IRQs:
 687          * (keep 'enabled' bit clear for now)
 688          */
 689         hwc->config = ARCH_PERFMON_EVENTSEL_INT;
 690
 691         /*
 692          * Count user and OS events unless requested not to.
 693          */
 694         if (!attr->exclude_user)
 695                 hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
 696         if (!attr->exclude_kernel)
 697                 hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
 698
 699         if (!hwc->sample_period) {
 700                 hwc->sample_period = x86_pmu.max_period;
 701                 hwc->last_period = hwc->sample_period;
 702                 atomic64_set(&hwc->period_left, hwc->sample_period);
 703         }
 704
 705         counter->destroy = hw_perf_counter_destroy;
 706
 707         /*
 708          * Raw event type provide the config in the event structure
 709          */
 710         if (attr->type == PERF_TYPE_RAW) {
 711                 hwc->config |= x86_pmu.raw_event(attr->config);
 712                 return 0;
 713         }
 714
 715         if (attr->type == PERF_TYPE_HW_CACHE)
 716                 return set_ext_hw_attr(hwc, attr);
 717
 718         if (attr->config >= x86_pmu.max_events)
 719                 return -EINVAL;
 720         /*
 721          * The generic map:
 722          */
 723         hwc->config |= x86_pmu.event_map(attr->config);
 724
 725         return 0;
 726 }
 727
 728 static void intel_pmu_disable_all(void)
 729 {
 730         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
 731 }
 732
 733 static void amd_pmu_disable_all(void)
 734 {
 735         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 736         int idx;
 737
 738         if (!cpuc->enabled)
 739                 return;
 740
 741         cpuc->enabled = 0;
 742         /*
 743          * ensure we write the disable before we start disabling the
 744          * counters proper, so that amd_pmu_enable_counter() does the
 745          * right thing.
 746          */
 747         barrier();
 748
 749         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 750                 u64 val;
 751
 752                 if (!test_bit(idx, cpuc->active_mask))
 753                         continue;
 754                 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
 755                 if (!(val & ARCH_PERFMON_EVENTSEL0_ENABLE))
 756                         continue;
 757                 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
 758                 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
 759         }
 760 }
 761
 762 void hw_perf_disable(void)
 763 {
 764         if (!x86_pmu_initialized())
 765                 return;
 766         return x86_pmu.disable_all();
 767 }
 768
 769 static void intel_pmu_enable_all(void)
 770 {
 771         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
 772 }
 773
 774 static void amd_pmu_enable_all(void)
 775 {
 776         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 777         int idx;
 778
 779         if (cpuc->enabled)
 780                 return;
 781
 782         cpuc->enabled = 1;
 783         barrier();
 784
 785         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 786                 u64 val;
 787
 788                 if (!test_bit(idx, cpuc->active_mask))
 789                         continue;
 790                 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
 791                 if (val & ARCH_PERFMON_EVENTSEL0_ENABLE)
 792                         continue;
 793                 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
 794                 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
 795         }
 796 }
 797
 798 void hw_perf_enable(void)
 799 {
 800         if (!x86_pmu_initialized())
 801                 return;
 802         x86_pmu.enable_all();
 803 }
 804
 805 static inline u64 intel_pmu_get_status(void)
 806 {
 807         u64 status;
 808
 809         rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
 810
 811         return status;
 812 }
 813
 814 static inline void intel_pmu_ack_status(u64 ack)
 815 {
 816         wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
 817 }
 818
 819 static inline void x86_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
 820 {
 821         int err;
 822         err = checking_wrmsrl(hwc->config_base + idx,
 823                               hwc->config | ARCH_PERFMON_EVENTSEL0_ENABLE);
 824 }
 825
 826 static inline void x86_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
 827 {
 828         int err;
 829         err = checking_wrmsrl(hwc->config_base + idx,
 830                               hwc->config);
 831 }
 832
 833 static inline void
 834 intel_pmu_disable_fixed(struct hw_perf_counter *hwc, int __idx)
 835 {
 836         int idx = __idx - X86_PMC_IDX_FIXED;
 837         u64 ctrl_val, mask;
 838         int err;
 839
 840         mask = 0xfULL << (idx * 4);
 841
 842         rdmsrl(hwc->config_base, ctrl_val);
 843         ctrl_val &= ~mask;
 844         err = checking_wrmsrl(hwc->config_base, ctrl_val);
 845 }
 846
 847 static inline void
 848 intel_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
 849 {
 850         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
 851                 intel_pmu_disable_fixed(hwc, idx);
 852                 return;
 853         }
 854
 855         x86_pmu_disable_counter(hwc, idx);
 856 }
 857
 858 static inline void
 859 amd_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
 860 {
 861         x86_pmu_disable_counter(hwc, idx);
 862 }
 863
 864 static DEFINE_PER_CPU(u64, prev_left[X86_PMC_IDX_MAX]);
 865
 866 /*
 867  * Set the next IRQ period, based on the hwc->period_left value.
 868  * To be called with the counter disabled in hw:
 869  */
 870 static int
 871 x86_perf_counter_set_period(struct perf_counter *counter,
 872                              struct hw_perf_counter *hwc, int idx)
 873 {
 874         s64 left = atomic64_read(&hwc->period_left);
 875         s64 period = hwc->sample_period;
 876         int err, ret = 0;
 877
 878         /*
 879          * If we are way outside a reasoable range then just skip forward:
 880          */
 881         if (unlikely(left <= -period)) {
 882                 left = period;
 883                 atomic64_set(&hwc->period_left, left);
 884                 hwc->last_period = period;
 885                 ret = 1;
 886         }
 887
 888         if (unlikely(left <= 0)) {
 889                 left += period;
 890                 atomic64_set(&hwc->period_left, left);
 891                 hwc->last_period = period;
 892                 ret = 1;
 893         }
 894         /*
 895          * Quirk: certain CPUs dont like it if just 1 event is left:
 896          */
 897         if (unlikely(left < 2))
 898                 left = 2;
 899
 900         if (left > x86_pmu.max_period)
 901                 left = x86_pmu.max_period;
 902
 903         per_cpu(prev_left[idx], smp_processor_id()) = left;
 904
 905         /*
 906          * The hw counter starts counting from this counter offset,
 907          * mark it to be able to extra future deltas:
 908          */
 909         atomic64_set(&hwc->prev_count, (u64)-left);
 910
 911         err = checking_wrmsrl(hwc->counter_base + idx,
 912                              (u64)(-left) & x86_pmu.counter_mask);
 913
 914         return ret;
 915 }
 916
 917 static inline void
 918 intel_pmu_enable_fixed(struct hw_perf_counter *hwc, int __idx)
 919 {
 920         int idx = __idx - X86_PMC_IDX_FIXED;
 921         u64 ctrl_val, bits, mask;
 922         int err;
 923
 924         /*
 925          * Enable IRQ generation (0x8),
 926          * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
 927          * if requested:
 928          */
 929         bits = 0x8ULL;
 930         if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
 931                 bits |= 0x2;
 932         if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
 933                 bits |= 0x1;
 934         bits <<= (idx * 4);
 935         mask = 0xfULL << (idx * 4);
 936
 937         rdmsrl(hwc->config_base, ctrl_val);
 938         ctrl_val &= ~mask;
 939         ctrl_val |= bits;
 940         err = checking_wrmsrl(hwc->config_base, ctrl_val);
 941 }
 942
 943 static void intel_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
 944 {
 945         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
 946                 intel_pmu_enable_fixed(hwc, idx);
 947                 return;
 948         }
 949
 950         x86_pmu_enable_counter(hwc, idx);
 951 }
 952
 953 static void amd_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
 954 {
 955         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 956
 957         if (cpuc->enabled)
 958                 x86_pmu_enable_counter(hwc, idx);
 959         else
 960                 x86_pmu_disable_counter(hwc, idx);
 961 }
 962
 963 static int
 964 fixed_mode_idx(struct perf_counter *counter, struct hw_perf_counter *hwc)
 965 {
 966         unsigned int event;
 967
 968         if (!x86_pmu.num_counters_fixed)
 969                 return -1;
 970
 971         /*
 972          * Quirk, IA32_FIXED_CTRs do not work on current Atom processors:
 973          */
 974         if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
 975                                         boot_cpu_data.x86_model == 28)
 976                 return -1;
 977
 978         event = hwc->config & ARCH_PERFMON_EVENT_MASK;
 979
 980         if (unlikely(event == x86_pmu.event_map(PERF_COUNT_HW_INSTRUCTIONS)))
 981                 return X86_PMC_IDX_FIXED_INSTRUCTIONS;
 982         if (unlikely(event == x86_pmu.event_map(PERF_COUNT_HW_CPU_CYCLES)))
 983                 return X86_PMC_IDX_FIXED_CPU_CYCLES;
 984         if (unlikely(event == x86_pmu.event_map(PERF_COUNT_HW_BUS_CYCLES)))
 985                 return X86_PMC_IDX_FIXED_BUS_CYCLES;
 986
 987         return -1;
 988 }
 989
 990 /*
 991  * Find a PMC slot for the freshly enabled / scheduled in counter:
 992  */
 993 static int x86_pmu_enable(struct perf_counter *counter)
 994 {
 995         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 996         struct hw_perf_counter *hwc = &counter->hw;
 997         int idx;
 998
 999         idx = fixed_mode_idx(counter, hwc);
1000         if (idx >= 0) {
1001                 /*
1002                  * Try to get the fixed counter, if that is already taken
1003                  * then try to get a generic counter:
1004                  */
1005                 if (test_and_set_bit(idx, cpuc->used_mask))
1006                         goto try_generic;
1007
1008                 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
1009                 /*
1010                  * We set it so that counter_base + idx in wrmsr/rdmsr maps to
1011                  * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
1012                  */
1013                 hwc->counter_base =
1014                         MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
1015                 hwc->idx = idx;
1016         } else {
1017                 idx = hwc->idx;
1018                 /* Try to get the previous generic counter again */
1019                 if (test_and_set_bit(idx, cpuc->used_mask)) {
1020 try_generic:
1021                         idx = find_first_zero_bit(cpuc->used_mask,
1022                                                   x86_pmu.num_counters);
1023                         if (idx == x86_pmu.num_counters)
1024                                 return -EAGAIN;
1025
1026                         set_bit(idx, cpuc->used_mask);
1027                         hwc->idx = idx;
1028                 }
1029                 hwc->config_base  = x86_pmu.eventsel;
1030                 hwc->counter_base = x86_pmu.perfctr;
1031         }
1032
1033         perf_counters_lapic_init();
1034
1035         x86_pmu.disable(hwc, idx);
1036
1037         cpuc->counters[idx] = counter;
1038         set_bit(idx, cpuc->active_mask);
1039
1040         x86_perf_counter_set_period(counter, hwc, idx);
1041         x86_pmu.enable(hwc, idx);
1042
1043         return 0;
1044 }
1045
1046 static void x86_pmu_unthrottle(struct perf_counter *counter)
1047 {
1048         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
1049         struct hw_perf_counter *hwc = &counter->hw;
1050
1051         if (WARN_ON_ONCE(hwc->idx >= X86_PMC_IDX_MAX ||
1052                                 cpuc->counters[hwc->idx] != counter))
1053                 return;
1054
1055         x86_pmu.enable(hwc, hwc->idx);
1056 }
1057
1058 void perf_counter_print_debug(void)
1059 {
1060         u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
1061         struct cpu_hw_counters *cpuc;
1062         unsigned long flags;
1063         int cpu, idx;
1064
1065         if (!x86_pmu.num_counters)
1066                 return;
1067
1068         local_irq_save(flags);
1069
1070         cpu = smp_processor_id();
1071         cpuc = &per_cpu(cpu_hw_counters, cpu);
1072
1073         if (x86_pmu.version >= 2) {
1074                 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
1075                 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1076                 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
1077                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
1078
1079                 pr_info("\n");
1080                 pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
1081                 pr_info("CPU#%d: status:     %016llx\n", cpu, status);
1082                 pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
1083                 pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
1084         }
1085         pr_info("CPU#%d: used:       %016llx\n", cpu, *(u64 *)cpuc->used_mask);
1086
1087         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1088                 rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
1089                 rdmsrl(x86_pmu.perfctr  + idx, pmc_count);
1090
1091                 prev_left = per_cpu(prev_left[idx], cpu);
1092
1093                 pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
1094                         cpu, idx, pmc_ctrl);
1095                 pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
1096                         cpu, idx, pmc_count);
1097                 pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
1098                         cpu, idx, prev_left);
1099         }
1100         for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
1101                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
1102
1103                 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
1104                         cpu, idx, pmc_count);
1105         }
1106         local_irq_restore(flags);
1107 }
1108
1109 static void x86_pmu_disable(struct perf_counter *counter)
1110 {
1111         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
1112         struct hw_perf_counter *hwc = &counter->hw;
1113         int idx = hwc->idx;
1114
1115         /*
1116          * Must be done before we disable, otherwise the nmi handler
1117          * could reenable again:
1118          */
1119         clear_bit(idx, cpuc->active_mask);
1120         x86_pmu.disable(hwc, idx);
1121
1122         /*
1123          * Make sure the cleared pointer becomes visible before we
1124          * (potentially) free the counter:
1125          */
1126         barrier();
1127
1128         /*
1129          * Drain the remaining delta count out of a counter
1130          * that we are disabling:
1131          */
1132         x86_perf_counter_update(counter, hwc, idx);
1133         cpuc->counters[idx] = NULL;
1134         clear_bit(idx, cpuc->used_mask);
1135 }
1136
1137 /*
1138  * Save and restart an expired counter. Called by NMI contexts,
1139  * so it has to be careful about preempting normal counter ops:
1140  */
1141 static int intel_pmu_save_and_restart(struct perf_counter *counter)
1142 {
1143         struct hw_perf_counter *hwc = &counter->hw;
1144         int idx = hwc->idx;
1145         int ret;
1146
1147         x86_perf_counter_update(counter, hwc, idx);
1148         ret = x86_perf_counter_set_period(counter, hwc, idx);
1149
1150         if (counter->state == PERF_COUNTER_STATE_ACTIVE)
1151                 intel_pmu_enable_counter(hwc, idx);
1152
1153         return ret;
1154 }
1155
1156 static void intel_pmu_reset(void)
1157 {
1158         unsigned long flags;
1159         int idx;
1160
1161         if (!x86_pmu.num_counters)
1162                 return;
1163
1164         local_irq_save(flags);
1165
1166         printk("clearing PMU state on CPU#%d\n", smp_processor_id());
1167
1168         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1169                 checking_wrmsrl(x86_pmu.eventsel + idx, 0ull);
1170                 checking_wrmsrl(x86_pmu.perfctr  + idx, 0ull);
1171         }
1172         for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
1173                 checking_wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
1174         }
1175
1176         local_irq_restore(flags);
1177 }
1178
1179
1180 /*
1181  * This handler is triggered by the local APIC, so the APIC IRQ handling
1182  * rules apply:
1183  */
1184 static int intel_pmu_handle_irq(struct pt_regs *regs)
1185 {
1186         struct perf_sample_data data;
1187         struct cpu_hw_counters *cpuc;
1188         int bit, cpu, loops;
1189         u64 ack, status;
1190
1191         data.regs = regs;
1192         data.addr = 0;
1193
1194         cpu = smp_processor_id();
1195         cpuc = &per_cpu(cpu_hw_counters, cpu);
1196
1197         perf_disable();
1198         status = intel_pmu_get_status();
1199         if (!status) {
1200                 perf_enable();
1201                 return 0;
1202         }
1203
1204         loops = 0;
1205 again:
1206         if (++loops > 100) {
1207                 WARN_ONCE(1, "perfcounters: irq loop stuck!\n");
1208                 perf_counter_print_debug();
1209                 intel_pmu_reset();
1210                 perf_enable();
1211                 return 1;
1212         }
1213
1214         inc_irq_stat(apic_perf_irqs);
1215         ack = status;
1216         for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
1217                 struct perf_counter *counter = cpuc->counters[bit];
1218
1219                 clear_bit(bit, (unsigned long *) &status);
1220                 if (!test_bit(bit, cpuc->active_mask))
1221                         continue;
1222
1223                 if (!intel_pmu_save_and_restart(counter))
1224                         continue;
1225
1226                 if (perf_counter_overflow(counter, 1, &data))
1227                         intel_pmu_disable_counter(&counter->hw, bit);
1228         }
1229
1230         intel_pmu_ack_status(ack);
1231
1232         /*
1233          * Repeat if there is more work to be done:
1234          */
1235         status = intel_pmu_get_status();
1236         if (status)
1237                 goto again;
1238
1239         perf_enable();
1240
1241         return 1;
1242 }
1243
1244 static int amd_pmu_handle_irq(struct pt_regs *regs)
1245 {
1246         struct perf_sample_data data;
1247         struct cpu_hw_counters *cpuc;
1248         struct perf_counter *counter;
1249         struct hw_perf_counter *hwc;
1250         int cpu, idx, handled = 0;
1251         u64 val;
1252
1253         data.regs = regs;
1254         data.addr = 0;
1255
1256         cpu = smp_processor_id();
1257         cpuc = &per_cpu(cpu_hw_counters, cpu);
1258
1259         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1260                 if (!test_bit(idx, cpuc->active_mask))
1261                         continue;
1262
1263                 counter = cpuc->counters[idx];
1264                 hwc = &counter->hw;
1265
1266                 val = x86_perf_counter_update(counter, hwc, idx);
1267                 if (val & (1ULL << (x86_pmu.counter_bits - 1)))
1268                         continue;
1269
1270                 /*
1271                  * counter overflow
1272                  */
1273                 handled         = 1;
1274                 data.period     = counter->hw.last_period;
1275
1276                 if (!x86_perf_counter_set_period(counter, hwc, idx))
1277                         continue;
1278
1279                 if (perf_counter_overflow(counter, 1, &data))
1280                         amd_pmu_disable_counter(hwc, idx);
1281         }
1282
1283         if (handled)
1284                 inc_irq_stat(apic_perf_irqs);
1285
1286         return handled;
1287 }
1288
1289 void smp_perf_pending_interrupt(struct pt_regs *regs)
1290 {
1291         irq_enter();
1292         ack_APIC_irq();
1293         inc_irq_stat(apic_pending_irqs);
1294         perf_counter_do_pending();
1295         irq_exit();
1296 }
1297
1298 void set_perf_counter_pending(void)
1299 {
1300         apic->send_IPI_self(LOCAL_PENDING_VECTOR);
1301 }
1302
1303 void perf_counters_lapic_init(void)
1304 {
1305         if (!x86_pmu_initialized())
1306                 return;
1307
1308         /*
1309          * Always use NMI for PMU
1310          */
1311         apic_write(APIC_LVTPC, APIC_DM_NMI);
1312 }
1313
1314 static int __kprobes
1315 perf_counter_nmi_handler(struct notifier_block *self,
1316                          unsigned long cmd, void *__args)
1317 {
1318         struct die_args *args = __args;
1319         struct pt_regs *regs;
1320
1321         if (!atomic_read(&active_counters))
1322                 return NOTIFY_DONE;
1323
1324         switch (cmd) {
1325         case DIE_NMI:
1326         case DIE_NMI_IPI:
1327                 break;
1328
1329         default:
1330                 return NOTIFY_DONE;
1331         }
1332
1333         regs = args->regs;
1334
1335         apic_write(APIC_LVTPC, APIC_DM_NMI);
1336         /*
1337          * Can't rely on the handled return value to say it was our NMI, two
1338          * counters could trigger 'simultaneously' raising two back-to-back NMIs.
1339          *
1340          * If the first NMI handles both, the latter will be empty and daze
1341          * the CPU.
1342          */
1343         x86_pmu.handle_irq(regs);
1344
1345         return NOTIFY_STOP;
1346 }
1347
1348 static __read_mostly struct notifier_block perf_counter_nmi_notifier = {
1349         .notifier_call          = perf_counter_nmi_handler,
1350         .next                   = NULL,
1351         .priority               = 1
1352 };
1353
1354 static struct x86_pmu intel_pmu = {
1355         .name                   = "Intel",
1356         .handle_irq             = intel_pmu_handle_irq,
1357         .disable_all            = intel_pmu_disable_all,
1358         .enable_all             = intel_pmu_enable_all,
1359         .enable                 = intel_pmu_enable_counter,
1360         .disable                = intel_pmu_disable_counter,
1361         .eventsel               = MSR_ARCH_PERFMON_EVENTSEL0,
1362         .perfctr                = MSR_ARCH_PERFMON_PERFCTR0,
1363         .event_map              = intel_pmu_event_map,
1364         .raw_event              = intel_pmu_raw_event,
1365         .max_events             = ARRAY_SIZE(intel_perfmon_event_map),
1366         /*
1367          * Intel PMCs cannot be accessed sanely above 32 bit width,
1368          * so we install an artificial 1<<31 period regardless of
1369          * the generic counter period:
1370          */
1371         .max_period             = (1ULL << 31) - 1,
1372 };
1373
1374 static struct x86_pmu amd_pmu = {
1375         .name                   = "AMD",
1376         .handle_irq             = amd_pmu_handle_irq,
1377         .disable_all            = amd_pmu_disable_all,
1378         .enable_all             = amd_pmu_enable_all,
1379         .enable                 = amd_pmu_enable_counter,
1380         .disable                = amd_pmu_disable_counter,
1381         .eventsel               = MSR_K7_EVNTSEL0,
1382         .perfctr                = MSR_K7_PERFCTR0,
1383         .event_map              = amd_pmu_event_map,
1384         .raw_event              = amd_pmu_raw_event,
1385         .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
1386         .num_counters           = 4,
1387         .counter_bits           = 48,
1388         .counter_mask           = (1ULL << 48) - 1,
1389         /* use highest bit to detect overflow */
1390         .max_period             = (1ULL << 47) - 1,
1391 };
1392
1393 static int intel_pmu_init(void)
1394 {
1395         union cpuid10_edx edx;
1396         union cpuid10_eax eax;
1397         unsigned int unused;
1398         unsigned int ebx;
1399         int version;
1400
1401         if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
1402                 return -ENODEV;
1403
1404         /*
1405          * Check whether the Architectural PerfMon supports
1406          * Branch Misses Retired Event or not.
1407          */
1408         cpuid(10, &eax.full, &ebx, &unused, &edx.full);
1409         if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
1410                 return -ENODEV;
1411
1412         version = eax.split.version_id;
1413         if (version < 2)
1414                 return -ENODEV;
1415
1416         x86_pmu                         = intel_pmu;
1417         x86_pmu.version                 = version;
1418         x86_pmu.num_counters            = eax.split.num_counters;
1419         x86_pmu.counter_bits            = eax.split.bit_width;
1420         x86_pmu.counter_mask            = (1ULL << eax.split.bit_width) - 1;
1421
1422         /*
1423          * Quirk: v2 perfmon does not report fixed-purpose counters, so
1424          * assume at least 3 counters:
1425          */
1426         x86_pmu.num_counters_fixed      = max((int)edx.split.num_counters_fixed, 3);
1427
1428         rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
1429
1430         /*
1431          * Install the hw-cache-events table:
1432          */
1433         switch (boot_cpu_data.x86_model) {
1434         case 15: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
1435         case 22: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
1436         case 23: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
1437         case 29: /* six-core 45 nm xeon "Dunnington" */
1438                 memcpy(hw_cache_event_ids, core2_hw_cache_event_ids,
1439                        sizeof(hw_cache_event_ids));
1440
1441                 pr_cont("Core2 events, ");
1442                 break;
1443         default:
1444         case 26:
1445                 memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids,
1446                        sizeof(hw_cache_event_ids));
1447
1448                 pr_cont("Nehalem/Corei7 events, ");
1449                 break;
1450         case 28:
1451                 memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
1452                        sizeof(hw_cache_event_ids));
1453
1454                 pr_cont("Atom events, ");
1455                 break;
1456         }
1457         return 0;
1458 }
1459
1460 static int amd_pmu_init(void)
1461 {
1462         x86_pmu = amd_pmu;
1463
1464         switch (boot_cpu_data.x86) {
1465         case 0x0f:
1466         case 0x10:
1467         case 0x11:
1468                 memcpy(hw_cache_event_ids, amd_0f_hw_cache_event_ids,
1469                        sizeof(hw_cache_event_ids));
1470
1471                 pr_cont("AMD Family 0f/10/11 events, ");
1472                 break;
1473         }
1474         return 0;
1475 }
1476
1477 void __init init_hw_perf_counters(void)
1478 {
1479         int err;
1480
1481         pr_info("Performance Counters: ");
1482
1483         switch (boot_cpu_data.x86_vendor) {
1484         case X86_VENDOR_INTEL:
1485                 err = intel_pmu_init();
1486                 break;
1487         case X86_VENDOR_AMD:
1488                 err = amd_pmu_init();
1489                 break;
1490         default:
1491                 return;
1492         }
1493         if (err != 0) {
1494                 pr_cont("no PMU driver, software counters only.\n");
1495                 return;
1496         }
1497
1498         pr_cont("%s PMU driver.\n", x86_pmu.name);
1499
1500         if (x86_pmu.num_counters > X86_PMC_MAX_GENERIC) {
1501                 x86_pmu.num_counters = X86_PMC_MAX_GENERIC;
1502                 WARN(1, KERN_ERR "hw perf counters %d > max(%d), clipping!",
1503                      x86_pmu.num_counters, X86_PMC_MAX_GENERIC);
1504         }
1505         perf_counter_mask = (1 << x86_pmu.num_counters) - 1;
1506         perf_max_counters = x86_pmu.num_counters;
1507
1508         if (x86_pmu.num_counters_fixed > X86_PMC_MAX_FIXED) {
1509                 x86_pmu.num_counters_fixed = X86_PMC_MAX_FIXED;
1510                 WARN(1, KERN_ERR "hw perf counters fixed %d > max(%d), clipping!",
1511                      x86_pmu.num_counters_fixed, X86_PMC_MAX_FIXED);
1512         }
1513
1514         perf_counter_mask |=
1515                 ((1LL << x86_pmu.num_counters_fixed)-1) << X86_PMC_IDX_FIXED;
1516
1517         perf_counters_lapic_init();
1518         register_die_notifier(&perf_counter_nmi_notifier);
1519
1520         pr_info("... version:                 %d\n",     x86_pmu.version);
1521         pr_info("... bit width:               %d\n",     x86_pmu.counter_bits);
1522         pr_info("... generic counters:        %d\n",     x86_pmu.num_counters);
1523         pr_info("... value mask:              %016Lx\n", x86_pmu.counter_mask);
1524         pr_info("... max period:              %016Lx\n", x86_pmu.max_period);
1525         pr_info("... fixed-purpose counters:  %d\n",     x86_pmu.num_counters_fixed);
1526         pr_info("... counter mask:            %016Lx\n", perf_counter_mask);
1527 }
1528
1529 static inline void x86_pmu_read(struct perf_counter *counter)
1530 {
1531         x86_perf_counter_update(counter, &counter->hw, counter->hw.idx);
1532 }
1533
1534 static const struct pmu pmu = {
1535         .enable         = x86_pmu_enable,
1536         .disable        = x86_pmu_disable,
1537         .read           = x86_pmu_read,
1538         .unthrottle     = x86_pmu_unthrottle,
1539 };
1540
1541 const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
1542 {
1543         int err;
1544
1545         err = __hw_perf_counter_init(counter);
1546         if (err)
1547                 return ERR_PTR(err);
1548
1549         return &pmu;
1550 }
1551
1552 /*
1553  * callchain support
1554  */
1555
1556 static inline
1557 void callchain_store(struct perf_callchain_entry *entry, unsigned long ip)
1558 {
1559         if (entry->nr < MAX_STACK_DEPTH)
1560                 entry->ip[entry->nr++] = ip;
1561 }
1562
1563 static DEFINE_PER_CPU(struct perf_callchain_entry, irq_entry);
1564 static DEFINE_PER_CPU(struct perf_callchain_entry, nmi_entry);
1565
1566
1567 static void
1568 backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
1569 {
1570         /* Ignore warnings */
1571 }
1572
1573 static void backtrace_warning(void *data, char *msg)
1574 {
1575         /* Ignore warnings */
1576 }
1577
1578 static int backtrace_stack(void *data, char *name)
1579 {
1580         /* Don't bother with IRQ stacks for now */
1581         return -1;
1582 }
1583
1584 static void backtrace_address(void *data, unsigned long addr, int reliable)
1585 {
1586         struct perf_callchain_entry *entry = data;
1587
1588         if (reliable)
1589                 callchain_store(entry, addr);
1590 }
1591
1592 static const struct stacktrace_ops backtrace_ops = {
1593         .warning                = backtrace_warning,
1594         .warning_symbol         = backtrace_warning_symbol,
1595         .stack                  = backtrace_stack,
1596         .address                = backtrace_address,
1597 };
1598
1599 static void
1600 perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
1601 {
1602         unsigned long bp;
1603         char *stack;
1604         int nr = entry->nr;
1605
1606         callchain_store(entry, instruction_pointer(regs));
1607
1608         stack = ((char *)regs + sizeof(struct pt_regs));
1609 #ifdef CONFIG_FRAME_POINTER
1610         bp = frame_pointer(regs);
1611 #else
1612         bp = 0;
1613 #endif
1614
1615         dump_trace(NULL, regs, (void *)stack, bp, &backtrace_ops, entry);
1616
1617         entry->kernel = entry->nr - nr;
1618 }
1619
1620
1621 struct stack_frame {
1622         const void __user       *next_fp;
1623         unsigned long           return_address;
1624 };
1625
1626 static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
1627 {
1628         int ret;
1629
1630         if (!access_ok(VERIFY_READ, fp, sizeof(*frame)))
1631                 return 0;
1632
1633         ret = 1;
1634         pagefault_disable();
1635         if (__copy_from_user_inatomic(frame, fp, sizeof(*frame)))
1636                 ret = 0;
1637         pagefault_enable();
1638
1639         return ret;
1640 }
1641
1642 static void
1643 perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
1644 {
1645         struct stack_frame frame;
1646         const void __user *fp;
1647         int nr = entry->nr;
1648
1649         regs = (struct pt_regs *)current->thread.sp0 - 1;
1650         fp   = (void __user *)regs->bp;
1651
1652         callchain_store(entry, regs->ip);
1653
1654         while (entry->nr < MAX_STACK_DEPTH) {
1655                 frame.next_fp        = NULL;
1656                 frame.return_address = 0;
1657
1658                 if (!copy_stack_frame(fp, &frame))
1659                         break;
1660
1661                 if ((unsigned long)fp < user_stack_pointer(regs))
1662                         break;
1663
1664                 callchain_store(entry, frame.return_address);
1665                 fp = frame.next_fp;
1666         }
1667
1668         entry->user = entry->nr - nr;
1669 }
1670
1671 static void
1672 perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
1673 {
1674         int is_user;
1675
1676         if (!regs)
1677                 return;
1678
1679         is_user = user_mode(regs);
1680
1681         if (!current || current->pid == 0)
1682                 return;
1683
1684         if (is_user && current->state != TASK_RUNNING)
1685                 return;
1686
1687         if (!is_user)
1688                 perf_callchain_kernel(regs, entry);
1689
1690         if (current->mm)
1691                 perf_callchain_user(regs, entry);
1692 }
1693
1694 struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
1695 {
1696         struct perf_callchain_entry *entry;
1697
1698         if (in_nmi())
1699                 entry = &__get_cpu_var(nmi_entry);
1700         else
1701                 entry = &__get_cpu_var(irq_entry);
1702
1703         entry->nr = 0;
1704         entry->hv = 0;
1705         entry->kernel = 0;
1706         entry->user = 0;
1707
1708         perf_do_callchain(regs, entry);
1709
1710         return entry;
1711 }