]> bbs.cooldavid.org Git - net-next-2.6.git/blame - arch/x86/kernel/cpu/perf_event.c
git://bbs.cooldavid.org / net-next-2.6.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
perf: Fix inconsistency between IP and callchain sampling
[net-next-2.6.git] / arch / x86 / kernel / cpu / perf_event.c
CommitLineData
241771ef 1/*
cdd6c482 2 * Performance events x86 architecture code
241771ef 3 *
98144511
IM		 4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2009 Jaswinder Singh Rajput
7 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
30dd568c 9 * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
241771ef
IM		 10 *
11 * For licencing details see kernel-base/COPYING
12 */
13
cdd6c482 14#include <linux/perf_event.h>
241771ef
IM		 15#include <linux/capability.h>
16#include <linux/notifier.h>
17#include <linux/hardirq.h>
18#include <linux/kprobes.h>
4ac13294 19#include <linux/module.h>
241771ef
IM		 20#include <linux/kdebug.h>
21#include <linux/sched.h>
d7d59fb3 22#include <linux/uaccess.h>
74193ef0 23#include <linux/highmem.h>
30dd568c 24#include <linux/cpu.h>
241771ef 25
241771ef 26#include <asm/apic.h>
d7d59fb3 27#include <asm/stacktrace.h>
4e935e47 28#include <asm/nmi.h>
241771ef 29
cdd6c482 30static u64 perf_event_mask __read_mostly;
703e937c 31
cdd6c482
IM		 32/* The maximal number of PEBS events: */
33#define MAX_PEBS_EVENTS 4
30dd568c
MM		 34
35/* The size of a BTS record in bytes: */
36#define BTS_RECORD_SIZE 24
37
38/* The size of a per-cpu BTS buffer in bytes: */
5622f295 39#define BTS_BUFFER_SIZE (BTS_RECORD_SIZE * 2048)
30dd568c
MM		 40
41/* The BTS overflow threshold in bytes from the end of the buffer: */
5622f295 42#define BTS_OVFL_TH (BTS_RECORD_SIZE * 128)
30dd568c
MM		 43
44
45/*
46 * Bits in the debugctlmsr controlling branch tracing.
47 */
48#define X86_DEBUGCTL_TR (1 << 6)
49#define X86_DEBUGCTL_BTS (1 << 7)
50#define X86_DEBUGCTL_BTINT (1 << 8)
51#define X86_DEBUGCTL_BTS_OFF_OS (1 << 9)
52#define X86_DEBUGCTL_BTS_OFF_USR (1 << 10)
53
54/*
55 * A debug store configuration.
56 *
57 * We only support architectures that use 64bit fields.
58 */
59struct debug_store {
60 u64 bts_buffer_base;
61 u64 bts_index;
62 u64 bts_absolute_maximum;
63 u64 bts_interrupt_threshold;
64 u64 pebs_buffer_base;
65 u64 pebs_index;
66 u64 pebs_absolute_maximum;
67 u64 pebs_interrupt_threshold;
cdd6c482 68 u64 pebs_event_reset[MAX_PEBS_EVENTS];
30dd568c
MM		 69};
70
cdd6c482
IM		 71struct cpu_hw_events {
72 struct perf_event *events[X86_PMC_IDX_MAX];
43f6201a
RR		 73 unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
74 unsigned long active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
4b39fd96 75 unsigned long interrupts;
b0f3f28e 76 int enabled;
30dd568c 77 struct debug_store *ds;
241771ef
IM		 78};
79
b690081d
SE		 80struct event_constraint {
81 unsigned long idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
82 int code;
83};
84
85#define EVENT_CONSTRAINT(c, m) { .code = (c), .idxmsk[0] = (m) }
86#define EVENT_CONSTRAINT_END { .code = 0, .idxmsk[0] = 0 }
87
88#define for_each_event_constraint(e, c) \
89 for ((e) = (c); (e)->idxmsk[0]; (e)++)
90
91
241771ef 92/*
5f4ec28f 93 * struct x86_pmu - generic x86 pmu
241771ef 94 */
5f4ec28f 95struct x86_pmu {
faa28ae0
RR		 96 const char *name;
97 int version;
a3288106 98 int (*handle_irq)(struct pt_regs *);
9e35ad38
PZ		 99 void (*disable_all)(void);
100 void (*enable_all)(void);
cdd6c482
IM		 101 void (*enable)(struct hw_perf_event *, int);
102 void (*disable)(struct hw_perf_event *, int);
169e41eb
JSR		 103 unsigned eventsel;
104 unsigned perfctr;
b0f3f28e
PZ		 105 u64 (*event_map)(int);
106 u64 (*raw_event)(u64);
169e41eb 107 int max_events;
cdd6c482
IM		 108 int num_events;
109 int num_events_fixed;
110 int event_bits;
111 u64 event_mask;
04da8a43 112 int apic;
c619b8ff 113 u64 max_period;
9e35ad38 114 u64 intel_ctrl;
30dd568c
MM		 115 void (*enable_bts)(u64 config);
116 void (*disable_bts)(void);
fe9081cc
PZ		 117 int (*get_event_idx)(struct cpu_hw_events *cpuc,
118 struct hw_perf_event *hwc);
b56a3802
JSR		 119};
120
4a06bd85 121static struct x86_pmu x86_pmu __read_mostly;
b56a3802 122
cdd6c482 123static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
b0f3f28e
PZ		 124 .enabled = 1,
125};
241771ef 126
7a693d3f 127static const struct event_constraint *event_constraints;
b690081d 128
11d1578f
VW		 129/*
130 * Not sure about some of these
131 */
132static const u64 p6_perfmon_event_map[] =
133{
134 [PERF_COUNT_HW_CPU_CYCLES] = 0x0079,
135 [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0,
f64ccccb
IM		 136 [PERF_COUNT_HW_CACHE_REFERENCES] = 0x0f2e,
137 [PERF_COUNT_HW_CACHE_MISSES] = 0x012e,
11d1578f
VW		 138 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c4,
139 [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c5,
140 [PERF_COUNT_HW_BUS_CYCLES] = 0x0062,
141};
142
dfc65094 143static u64 p6_pmu_event_map(int hw_event)
11d1578f 144{
dfc65094 145 return p6_perfmon_event_map[hw_event];
11d1578f
VW		 146}
147
9c74fb50 148/*
cdd6c482 149 * Event setting that is specified not to count anything.
9c74fb50
PZ		 150 * We use this to effectively disable a counter.
151 *
152 * L2_RQSTS with 0 MESI unit mask.
153 */
cdd6c482 154#define P6_NOP_EVENT 0x0000002EULL
9c74fb50 155
dfc65094 156static u64 p6_pmu_raw_event(u64 hw_event)
11d1578f
VW		 157{
158#define P6_EVNTSEL_EVENT_MASK 0x000000FFULL
159#define P6_EVNTSEL_UNIT_MASK 0x0000FF00ULL
160#define P6_EVNTSEL_EDGE_MASK 0x00040000ULL
161#define P6_EVNTSEL_INV_MASK 0x00800000ULL
cdd6c482 162#define P6_EVNTSEL_REG_MASK 0xFF000000ULL
11d1578f
VW		 163
164#define P6_EVNTSEL_MASK \
165 (P6_EVNTSEL_EVENT_MASK | \
166 P6_EVNTSEL_UNIT_MASK | \
167 P6_EVNTSEL_EDGE_MASK | \
168 P6_EVNTSEL_INV_MASK | \
cdd6c482 169 P6_EVNTSEL_REG_MASK)
11d1578f 170
dfc65094 171 return hw_event & P6_EVNTSEL_MASK;
11d1578f
VW		 172}
173
b690081d
SE		 174static const struct event_constraint intel_p6_event_constraints[] =
175{
176 EVENT_CONSTRAINT(0xc1, 0x1), /* FLOPS */
177 EVENT_CONSTRAINT(0x10, 0x1), /* FP_COMP_OPS_EXE */
178 EVENT_CONSTRAINT(0x11, 0x1), /* FP_ASSIST */
179 EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
180 EVENT_CONSTRAINT(0x13, 0x2), /* DIV */
181 EVENT_CONSTRAINT(0x14, 0x1), /* CYCLES_DIV_BUSY */
182 EVENT_CONSTRAINT_END
183};
11d1578f 184
b56a3802
JSR		 185/*
186 * Intel PerfMon v3. Used on Core2 and later.
187 */
b0f3f28e 188static const u64 intel_perfmon_event_map[] =
241771ef 189{
f4dbfa8f
PZ		 190 [PERF_COUNT_HW_CPU_CYCLES] = 0x003c,
191 [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0,
192 [PERF_COUNT_HW_CACHE_REFERENCES] = 0x4f2e,
193 [PERF_COUNT_HW_CACHE_MISSES] = 0x412e,
194 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c4,
195 [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c5,
196 [PERF_COUNT_HW_BUS_CYCLES] = 0x013c,
241771ef
IM		 197};
198
b690081d
SE		 199static const struct event_constraint intel_core_event_constraints[] =
200{
201 EVENT_CONSTRAINT(0x10, 0x1), /* FP_COMP_OPS_EXE */
202 EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */
203 EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
204 EVENT_CONSTRAINT(0x13, 0x2), /* DIV */
205 EVENT_CONSTRAINT(0x14, 0x1), /* CYCLES_DIV_BUSY */
206 EVENT_CONSTRAINT(0x18, 0x1), /* IDLE_DURING_DIV */
207 EVENT_CONSTRAINT(0x19, 0x2), /* DELAYED_BYPASS */
208 EVENT_CONSTRAINT(0xa1, 0x1), /* RS_UOPS_DISPATCH_CYCLES */
209 EVENT_CONSTRAINT(0xcb, 0x1), /* MEM_LOAD_RETIRED */
210 EVENT_CONSTRAINT_END
211};
212
213static const struct event_constraint intel_nehalem_event_constraints[] =
214{
215 EVENT_CONSTRAINT(0x40, 0x3), /* L1D_CACHE_LD */
216 EVENT_CONSTRAINT(0x41, 0x3), /* L1D_CACHE_ST */
217 EVENT_CONSTRAINT(0x42, 0x3), /* L1D_CACHE_LOCK */
218 EVENT_CONSTRAINT(0x43, 0x3), /* L1D_ALL_REF */
219 EVENT_CONSTRAINT(0x4e, 0x3), /* L1D_PREFETCH */
220 EVENT_CONSTRAINT(0x4c, 0x3), /* LOAD_HIT_PRE */
221 EVENT_CONSTRAINT(0x51, 0x3), /* L1D */
222 EVENT_CONSTRAINT(0x52, 0x3), /* L1D_CACHE_PREFETCH_LOCK_FB_HIT */
223 EVENT_CONSTRAINT(0x53, 0x3), /* L1D_CACHE_LOCK_FB_HIT */
224 EVENT_CONSTRAINT(0xc5, 0x3), /* CACHE_LOCK_CYCLES */
225 EVENT_CONSTRAINT_END
226};
227
dfc65094 228static u64 intel_pmu_event_map(int hw_event)
b56a3802 229{
dfc65094 230 return intel_perfmon_event_map[hw_event];
b56a3802 231}
241771ef 232
8326f44d 233/*
dfc65094 234 * Generalized hw caching related hw_event table, filled
8326f44d 235 * in on a per model basis. A value of 0 means
dfc65094
IM		 236 * 'not supported', -1 means 'hw_event makes no sense on
237 * this CPU', any other value means the raw hw_event
8326f44d
IM		 238 * ID.
239 */
240
241#define C(x) PERF_COUNT_HW_CACHE_##x
242
243static u64 __read_mostly hw_cache_event_ids
244 [PERF_COUNT_HW_CACHE_MAX]
245 [PERF_COUNT_HW_CACHE_OP_MAX]
246 [PERF_COUNT_HW_CACHE_RESULT_MAX];
247
db48cccc 248static __initconst u64 nehalem_hw_cache_event_ids
8326f44d
IM		 249 [PERF_COUNT_HW_CACHE_MAX]
250 [PERF_COUNT_HW_CACHE_OP_MAX]
251 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
252{
253 [ C(L1D) ] = {
254 [ C(OP_READ) ] = {
255 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI */
256 [ C(RESULT_MISS) ] = 0x0140, /* L1D_CACHE_LD.I_STATE */
257 },
258 [ C(OP_WRITE) ] = {
259 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI */
260 [ C(RESULT_MISS) ] = 0x0141, /* L1D_CACHE_ST.I_STATE */
261 },
262 [ C(OP_PREFETCH) ] = {
263 [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS */
264 [ C(RESULT_MISS) ] = 0x024e, /* L1D_PREFETCH.MISS */
265 },
266 },
267 [ C(L1I ) ] = {
268 [ C(OP_READ) ] = {
fecc8ac8 269 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS */
8326f44d
IM		 270 [ C(RESULT_MISS) ] = 0x0280, /* L1I.MISSES */
271 },
272 [ C(OP_WRITE) ] = {
273 [ C(RESULT_ACCESS) ] = -1,
274 [ C(RESULT_MISS) ] = -1,
275 },
276 [ C(OP_PREFETCH) ] = {
277 [ C(RESULT_ACCESS) ] = 0x0,
278 [ C(RESULT_MISS) ] = 0x0,
279 },
280 },
8be6e8f3 281 [ C(LL ) ] = {
8326f44d
IM		 282 [ C(OP_READ) ] = {
283 [ C(RESULT_ACCESS) ] = 0x0324, /* L2_RQSTS.LOADS */
284 [ C(RESULT_MISS) ] = 0x0224, /* L2_RQSTS.LD_MISS */
285 },
286 [ C(OP_WRITE) ] = {
287 [ C(RESULT_ACCESS) ] = 0x0c24, /* L2_RQSTS.RFOS */
288 [ C(RESULT_MISS) ] = 0x0824, /* L2_RQSTS.RFO_MISS */
289 },
290 [ C(OP_PREFETCH) ] = {
8be6e8f3
PZ		 291 [ C(RESULT_ACCESS) ] = 0x4f2e, /* LLC Reference */
292 [ C(RESULT_MISS) ] = 0x412e, /* LLC Misses */
8326f44d
IM		 293 },
294 },
295 [ C(DTLB) ] = {
296 [ C(OP_READ) ] = {
297 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI (alias) */
298 [ C(RESULT_MISS) ] = 0x0108, /* DTLB_LOAD_MISSES.ANY */
299 },
300 [ C(OP_WRITE) ] = {
301 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI (alias) */
302 [ C(RESULT_MISS) ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS */
303 },
304 [ C(OP_PREFETCH) ] = {
305 [ C(RESULT_ACCESS) ] = 0x0,
306 [ C(RESULT_MISS) ] = 0x0,
307 },
308 },
309 [ C(ITLB) ] = {
310 [ C(OP_READ) ] = {
311 [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P */
fecc8ac8 312 [ C(RESULT_MISS) ] = 0x20c8, /* ITLB_MISS_RETIRED */
8326f44d
IM		 313 },
314 [ C(OP_WRITE) ] = {
315 [ C(RESULT_ACCESS) ] = -1,
316 [ C(RESULT_MISS) ] = -1,
317 },
318 [ C(OP_PREFETCH) ] = {
319 [ C(RESULT_ACCESS) ] = -1,
320 [ C(RESULT_MISS) ] = -1,
321 },
322 },
323 [ C(BPU ) ] = {
324 [ C(OP_READ) ] = {
325 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
326 [ C(RESULT_MISS) ] = 0x03e8, /* BPU_CLEARS.ANY */
327 },
328 [ C(OP_WRITE) ] = {
329 [ C(RESULT_ACCESS) ] = -1,
330 [ C(RESULT_MISS) ] = -1,
331 },
332 [ C(OP_PREFETCH) ] = {
333 [ C(RESULT_ACCESS) ] = -1,
334 [ C(RESULT_MISS) ] = -1,
335 },
336 },
337};
338
db48cccc 339static __initconst u64 core2_hw_cache_event_ids
8326f44d
IM		 340 [PERF_COUNT_HW_CACHE_MAX]
341 [PERF_COUNT_HW_CACHE_OP_MAX]
342 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
343{
0312af84
TG		 344 [ C(L1D) ] = {
345 [ C(OP_READ) ] = {
346 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI */
347 [ C(RESULT_MISS) ] = 0x0140, /* L1D_CACHE_LD.I_STATE */
348 },
349 [ C(OP_WRITE) ] = {
350 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI */
351 [ C(RESULT_MISS) ] = 0x0141, /* L1D_CACHE_ST.I_STATE */
352 },
353 [ C(OP_PREFETCH) ] = {
354 [ C(RESULT_ACCESS) ] = 0x104e, /* L1D_PREFETCH.REQUESTS */
355 [ C(RESULT_MISS) ] = 0,
356 },
357 },
358 [ C(L1I ) ] = {
359 [ C(OP_READ) ] = {
360 [ C(RESULT_ACCESS) ] = 0x0080, /* L1I.READS */
361 [ C(RESULT_MISS) ] = 0x0081, /* L1I.MISSES */
362 },
363 [ C(OP_WRITE) ] = {
364 [ C(RESULT_ACCESS) ] = -1,
365 [ C(RESULT_MISS) ] = -1,
366 },
367 [ C(OP_PREFETCH) ] = {
368 [ C(RESULT_ACCESS) ] = 0,
369 [ C(RESULT_MISS) ] = 0,
370 },
371 },
8be6e8f3 372 [ C(LL ) ] = {
0312af84
TG		 373 [ C(OP_READ) ] = {
374 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI */
375 [ C(RESULT_MISS) ] = 0x4129, /* L2_LD.ISTATE */
376 },
377 [ C(OP_WRITE) ] = {
378 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI */
379 [ C(RESULT_MISS) ] = 0x412A, /* L2_ST.ISTATE */
380 },
381 [ C(OP_PREFETCH) ] = {
382 [ C(RESULT_ACCESS) ] = 0,
383 [ C(RESULT_MISS) ] = 0,
384 },
385 },
386 [ C(DTLB) ] = {
387 [ C(OP_READ) ] = {
388 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI (alias) */
389 [ C(RESULT_MISS) ] = 0x0208, /* DTLB_MISSES.MISS_LD */
390 },
391 [ C(OP_WRITE) ] = {
392 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI (alias) */
393 [ C(RESULT_MISS) ] = 0x0808, /* DTLB_MISSES.MISS_ST */
394 },
395 [ C(OP_PREFETCH) ] = {
396 [ C(RESULT_ACCESS) ] = 0,
397 [ C(RESULT_MISS) ] = 0,
398 },
399 },
400 [ C(ITLB) ] = {
401 [ C(OP_READ) ] = {
402 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P */
403 [ C(RESULT_MISS) ] = 0x1282, /* ITLBMISSES */
404 },
405 [ C(OP_WRITE) ] = {
406 [ C(RESULT_ACCESS) ] = -1,
407 [ C(RESULT_MISS) ] = -1,
408 },
409 [ C(OP_PREFETCH) ] = {
410 [ C(RESULT_ACCESS) ] = -1,
411 [ C(RESULT_MISS) ] = -1,
412 },
413 },
414 [ C(BPU ) ] = {
415 [ C(OP_READ) ] = {
416 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY */
417 [ C(RESULT_MISS) ] = 0x00c5, /* BP_INST_RETIRED.MISPRED */
418 },
419 [ C(OP_WRITE) ] = {
420 [ C(RESULT_ACCESS) ] = -1,
421 [ C(RESULT_MISS) ] = -1,
422 },
423 [ C(OP_PREFETCH) ] = {
424 [ C(RESULT_ACCESS) ] = -1,
425 [ C(RESULT_MISS) ] = -1,
426 },
427 },
8326f44d
IM		 428};
429
db48cccc 430static __initconst u64 atom_hw_cache_event_ids
8326f44d
IM		 431 [PERF_COUNT_HW_CACHE_MAX]
432 [PERF_COUNT_HW_CACHE_OP_MAX]
433 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
434{
ad689220
TG		 435 [ C(L1D) ] = {
436 [ C(OP_READ) ] = {
437 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE.LD */
438 [ C(RESULT_MISS) ] = 0,
439 },
440 [ C(OP_WRITE) ] = {
fecc8ac8 441 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE.ST */
ad689220
TG		 442 [ C(RESULT_MISS) ] = 0,
443 },
444 [ C(OP_PREFETCH) ] = {
445 [ C(RESULT_ACCESS) ] = 0x0,
446 [ C(RESULT_MISS) ] = 0,
447 },
448 },
449 [ C(L1I ) ] = {
450 [ C(OP_READ) ] = {
fecc8ac8
YW		 451 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS */
452 [ C(RESULT_MISS) ] = 0x0280, /* L1I.MISSES */
ad689220
TG		 453 },
454 [ C(OP_WRITE) ] = {
455 [ C(RESULT_ACCESS) ] = -1,
456 [ C(RESULT_MISS) ] = -1,
457 },
458 [ C(OP_PREFETCH) ] = {
459 [ C(RESULT_ACCESS) ] = 0,
460 [ C(RESULT_MISS) ] = 0,
461 },
462 },
8be6e8f3 463 [ C(LL ) ] = {
ad689220
TG		 464 [ C(OP_READ) ] = {
465 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI */
466 [ C(RESULT_MISS) ] = 0x4129, /* L2_LD.ISTATE */
467 },
468 [ C(OP_WRITE) ] = {
469 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI */
470 [ C(RESULT_MISS) ] = 0x412A, /* L2_ST.ISTATE */
471 },
472 [ C(OP_PREFETCH) ] = {
473 [ C(RESULT_ACCESS) ] = 0,
474 [ C(RESULT_MISS) ] = 0,
475 },
476 },
477 [ C(DTLB) ] = {
478 [ C(OP_READ) ] = {
fecc8ac8 479 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE_LD.MESI (alias) */
ad689220
TG		 480 [ C(RESULT_MISS) ] = 0x0508, /* DTLB_MISSES.MISS_LD */
481 },
482 [ C(OP_WRITE) ] = {
fecc8ac8 483 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE_ST.MESI (alias) */
ad689220
TG		 484 [ C(RESULT_MISS) ] = 0x0608, /* DTLB_MISSES.MISS_ST */
485 },
486 [ C(OP_PREFETCH) ] = {
487 [ C(RESULT_ACCESS) ] = 0,
488 [ C(RESULT_MISS) ] = 0,
489 },
490 },
491 [ C(ITLB) ] = {
492 [ C(OP_READ) ] = {
493 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P */
494 [ C(RESULT_MISS) ] = 0x0282, /* ITLB.MISSES */
495 },
496 [ C(OP_WRITE) ] = {
497 [ C(RESULT_ACCESS) ] = -1,
498 [ C(RESULT_MISS) ] = -1,
499 },
500 [ C(OP_PREFETCH) ] = {
501 [ C(RESULT_ACCESS) ] = -1,
502 [ C(RESULT_MISS) ] = -1,
503 },
504 },
505 [ C(BPU ) ] = {
506 [ C(OP_READ) ] = {
507 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY */
508 [ C(RESULT_MISS) ] = 0x00c5, /* BP_INST_RETIRED.MISPRED */
509 },
510 [ C(OP_WRITE) ] = {
511 [ C(RESULT_ACCESS) ] = -1,
512 [ C(RESULT_MISS) ] = -1,
513 },
514 [ C(OP_PREFETCH) ] = {
515 [ C(RESULT_ACCESS) ] = -1,
516 [ C(RESULT_MISS) ] = -1,
517 },
518 },
8326f44d
IM		 519};
520
dfc65094 521static u64 intel_pmu_raw_event(u64 hw_event)
b0f3f28e 522{
82bae4f8
PZ		 523#define CORE_EVNTSEL_EVENT_MASK 0x000000FFULL
524#define CORE_EVNTSEL_UNIT_MASK 0x0000FF00ULL
ff99be57
PZ		 525#define CORE_EVNTSEL_EDGE_MASK 0x00040000ULL
526#define CORE_EVNTSEL_INV_MASK 0x00800000ULL
fe9081cc 527#define CORE_EVNTSEL_REG_MASK 0xFF000000ULL
b0f3f28e 528
128f048f 529#define CORE_EVNTSEL_MASK \
b0f3f28e
PZ		 530 (CORE_EVNTSEL_EVENT_MASK | \
531 CORE_EVNTSEL_UNIT_MASK | \
ff99be57
PZ		 532 CORE_EVNTSEL_EDGE_MASK | \
533 CORE_EVNTSEL_INV_MASK | \
cdd6c482 534 CORE_EVNTSEL_REG_MASK)
b0f3f28e 535
dfc65094 536 return hw_event & CORE_EVNTSEL_MASK;
b0f3f28e
PZ		 537}
538
db48cccc 539static __initconst u64 amd_hw_cache_event_ids
f86748e9
TG		 540 [PERF_COUNT_HW_CACHE_MAX]
541 [PERF_COUNT_HW_CACHE_OP_MAX]
542 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
543{
544 [ C(L1D) ] = {
545 [ C(OP_READ) ] = {
f4db43a3
JSR		 546 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses */
547 [ C(RESULT_MISS) ] = 0x0041, /* Data Cache Misses */
f86748e9
TG		 548 },
549 [ C(OP_WRITE) ] = {
d9f2a5ec 550 [ C(RESULT_ACCESS) ] = 0x0142, /* Data Cache Refills :system */
f86748e9
TG		 551 [ C(RESULT_MISS) ] = 0,
552 },
553 [ C(OP_PREFETCH) ] = {
f4db43a3
JSR		 554 [ C(RESULT_ACCESS) ] = 0x0267, /* Data Prefetcher :attempts */
555 [ C(RESULT_MISS) ] = 0x0167, /* Data Prefetcher :cancelled */
f86748e9
TG		 556 },
557 },
558 [ C(L1I ) ] = {
559 [ C(OP_READ) ] = {
560 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction cache fetches */
561 [ C(RESULT_MISS) ] = 0x0081, /* Instruction cache misses */
562 },
563 [ C(OP_WRITE) ] = {
564 [ C(RESULT_ACCESS) ] = -1,
565 [ C(RESULT_MISS) ] = -1,
566 },
567 [ C(OP_PREFETCH) ] = {
f4db43a3 568 [ C(RESULT_ACCESS) ] = 0x014B, /* Prefetch Instructions :Load */
f86748e9
TG		 569 [ C(RESULT_MISS) ] = 0,
570 },
571 },
8be6e8f3 572 [ C(LL ) ] = {
f86748e9 573 [ C(OP_READ) ] = {
f4db43a3
JSR		 574 [ C(RESULT_ACCESS) ] = 0x037D, /* Requests to L2 Cache :IC+DC */
575 [ C(RESULT_MISS) ] = 0x037E, /* L2 Cache Misses : IC+DC */
f86748e9
TG		 576 },
577 [ C(OP_WRITE) ] = {
f4db43a3 578 [ C(RESULT_ACCESS) ] = 0x017F, /* L2 Fill/Writeback */
f86748e9
TG		 579 [ C(RESULT_MISS) ] = 0,
580 },
581 [ C(OP_PREFETCH) ] = {
582 [ C(RESULT_ACCESS) ] = 0,
583 [ C(RESULT_MISS) ] = 0,
584 },
585 },
586 [ C(DTLB) ] = {
587 [ C(OP_READ) ] = {
f4db43a3
JSR		 588 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses */
589 [ C(RESULT_MISS) ] = 0x0046, /* L1 DTLB and L2 DLTB Miss */
f86748e9
TG		 590 },
591 [ C(OP_WRITE) ] = {
592 [ C(RESULT_ACCESS) ] = 0,
593 [ C(RESULT_MISS) ] = 0,
594 },
595 [ C(OP_PREFETCH) ] = {
596 [ C(RESULT_ACCESS) ] = 0,
597 [ C(RESULT_MISS) ] = 0,
598 },
599 },
600 [ C(ITLB) ] = {
601 [ C(OP_READ) ] = {
602 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes */
603 [ C(RESULT_MISS) ] = 0x0085, /* Instr. fetch ITLB misses */
604 },
605 [ C(OP_WRITE) ] = {
606 [ C(RESULT_ACCESS) ] = -1,
607 [ C(RESULT_MISS) ] = -1,
608 },
609 [ C(OP_PREFETCH) ] = {
610 [ C(RESULT_ACCESS) ] = -1,
611 [ C(RESULT_MISS) ] = -1,
612 },
613 },
614 [ C(BPU ) ] = {
615 [ C(OP_READ) ] = {
616 [ C(RESULT_ACCESS) ] = 0x00c2, /* Retired Branch Instr. */
617 [ C(RESULT_MISS) ] = 0x00c3, /* Retired Mispredicted BI */
618 },
619 [ C(OP_WRITE) ] = {
620 [ C(RESULT_ACCESS) ] = -1,
621 [ C(RESULT_MISS) ] = -1,
622 },
623 [ C(OP_PREFETCH) ] = {
624 [ C(RESULT_ACCESS) ] = -1,
625 [ C(RESULT_MISS) ] = -1,
626 },
627 },
628};
629
f87ad35d
JSR		 630/*
631 * AMD Performance Monitor K7 and later.
632 */
b0f3f28e 633static const u64 amd_perfmon_event_map[] =
f87ad35d 634{
f4dbfa8f
PZ		 635 [PERF_COUNT_HW_CPU_CYCLES] = 0x0076,
636 [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0,
637 [PERF_COUNT_HW_CACHE_REFERENCES] = 0x0080,
638 [PERF_COUNT_HW_CACHE_MISSES] = 0x0081,
639 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c4,
640 [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c5,
f87ad35d
JSR		 641};
642
dfc65094 643static u64 amd_pmu_event_map(int hw_event)
f87ad35d 644{
dfc65094 645 return amd_perfmon_event_map[hw_event];
f87ad35d
JSR		 646}
647
dfc65094 648static u64 amd_pmu_raw_event(u64 hw_event)
b0f3f28e 649{
82bae4f8
PZ		 650#define K7_EVNTSEL_EVENT_MASK 0x7000000FFULL
651#define K7_EVNTSEL_UNIT_MASK 0x00000FF00ULL
ff99be57
PZ		 652#define K7_EVNTSEL_EDGE_MASK 0x000040000ULL
653#define K7_EVNTSEL_INV_MASK 0x000800000ULL
cdd6c482 654#define K7_EVNTSEL_REG_MASK 0x0FF000000ULL
b0f3f28e
PZ		 655
656#define K7_EVNTSEL_MASK \
657 (K7_EVNTSEL_EVENT_MASK | \
658 K7_EVNTSEL_UNIT_MASK | \
ff99be57
PZ		 659 K7_EVNTSEL_EDGE_MASK | \
660 K7_EVNTSEL_INV_MASK | \
cdd6c482 661 K7_EVNTSEL_REG_MASK)
b0f3f28e 662
dfc65094 663 return hw_event & K7_EVNTSEL_MASK;
b0f3f28e
PZ		 664}
665
ee06094f 666/*
cdd6c482
IM		 667 * Propagate event elapsed time into the generic event.
668 * Can only be executed on the CPU where the event is active.
ee06094f
IM		 669 * Returns the delta events processed.
670 */
4b7bfd0d 671static u64
cdd6c482
IM		 672x86_perf_event_update(struct perf_event *event,
673 struct hw_perf_event *hwc, int idx)
ee06094f 674{
cdd6c482 675 int shift = 64 - x86_pmu.event_bits;
ec3232bd
PZ		 676 u64 prev_raw_count, new_raw_count;
677 s64 delta;
ee06094f 678
30dd568c
MM		 679 if (idx == X86_PMC_IDX_FIXED_BTS)
680 return 0;
681
ee06094f 682 /*
cdd6c482 683 * Careful: an NMI might modify the previous event value.
ee06094f
IM		 684 *
685 * Our tactic to handle this is to first atomically read and
686 * exchange a new raw count - then add that new-prev delta
cdd6c482 687 * count to the generic event atomically:
ee06094f
IM		 688 */
689again:
690 prev_raw_count = atomic64_read(&hwc->prev_count);
cdd6c482 691 rdmsrl(hwc->event_base + idx, new_raw_count);
ee06094f
IM		 692
693 if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
694 new_raw_count) != prev_raw_count)
695 goto again;
696
697 /*
698 * Now we have the new raw value and have updated the prev
699 * timestamp already. We can now calculate the elapsed delta
cdd6c482 700 * (event-)time and add that to the generic event.
ee06094f
IM		 701 *
702 * Careful, not all hw sign-extends above the physical width
ec3232bd 703 * of the count.
ee06094f 704 */
ec3232bd
PZ		 705 delta = (new_raw_count << shift) - (prev_raw_count << shift);
706 delta >>= shift;
ee06094f 707
cdd6c482 708 atomic64_add(delta, &event->count);
ee06094f 709 atomic64_sub(delta, &hwc->period_left);
4b7bfd0d
RR		 710
711 return new_raw_count;
ee06094f
IM		 712}
713
cdd6c482 714static atomic_t active_events;
4e935e47
PZ		 715static DEFINE_MUTEX(pmc_reserve_mutex);
716
717static bool reserve_pmc_hardware(void)
718{
04da8a43 719#ifdef CONFIG_X86_LOCAL_APIC
4e935e47
PZ		 720 int i;
721
722 if (nmi_watchdog == NMI_LOCAL_APIC)
723 disable_lapic_nmi_watchdog();
724
cdd6c482 725 for (i = 0; i < x86_pmu.num_events; i++) {
4a06bd85 726 if (!reserve_perfctr_nmi(x86_pmu.perfctr + i))
4e935e47
PZ		 727 goto perfctr_fail;
728 }
729
cdd6c482 730 for (i = 0; i < x86_pmu.num_events; i++) {
4a06bd85 731 if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
4e935e47
PZ		 732 goto eventsel_fail;
733 }
04da8a43 734#endif
4e935e47
PZ		 735
736 return true;
737
04da8a43 738#ifdef CONFIG_X86_LOCAL_APIC
4e935e47
PZ		 739eventsel_fail:
740 for (i--; i >= 0; i--)
4a06bd85 741 release_evntsel_nmi(x86_pmu.eventsel + i);
4e935e47 742
cdd6c482 743 i = x86_pmu.num_events;
4e935e47
PZ		 744
745perfctr_fail:
746 for (i--; i >= 0; i--)
4a06bd85 747 release_perfctr_nmi(x86_pmu.perfctr + i);
4e935e47
PZ		 748
749 if (nmi_watchdog == NMI_LOCAL_APIC)
750 enable_lapic_nmi_watchdog();
751
752 return false;
04da8a43 753#endif
4e935e47
PZ		 754}
755
756static void release_pmc_hardware(void)
757{
04da8a43 758#ifdef CONFIG_X86_LOCAL_APIC
4e935e47
PZ		 759 int i;
760
cdd6c482 761 for (i = 0; i < x86_pmu.num_events; i++) {
4a06bd85
RR		 762 release_perfctr_nmi(x86_pmu.perfctr + i);
763 release_evntsel_nmi(x86_pmu.eventsel + i);
4e935e47
PZ		 764 }
765
766 if (nmi_watchdog == NMI_LOCAL_APIC)
767 enable_lapic_nmi_watchdog();
04da8a43 768#endif
4e935e47
PZ		 769}
770
30dd568c
MM		 771static inline bool bts_available(void)
772{
773 return x86_pmu.enable_bts != NULL;
774}
775
776static inline void init_debug_store_on_cpu(int cpu)
777{
cdd6c482 778 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
30dd568c
MM		 779
780 if (!ds)
781 return;
782
783 wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA,
596da17f 784 (u32)((u64)(unsigned long)ds),
785 (u32)((u64)(unsigned long)ds >> 32));
30dd568c
MM		 786}
787
788static inline void fini_debug_store_on_cpu(int cpu)
789{
cdd6c482 790 if (!per_cpu(cpu_hw_events, cpu).ds)
30dd568c
MM		 791 return;
792
793 wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA, 0, 0);
794}
795
796static void release_bts_hardware(void)
797{
798 int cpu;
799
800 if (!bts_available())
801 return;
802
803 get_online_cpus();
804
805 for_each_online_cpu(cpu)
806 fini_debug_store_on_cpu(cpu);
807
808 for_each_possible_cpu(cpu) {
cdd6c482 809 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
30dd568c
MM		 810
811 if (!ds)
812 continue;
813
cdd6c482 814 per_cpu(cpu_hw_events, cpu).ds = NULL;
30dd568c 815
596da17f 816 kfree((void *)(unsigned long)ds->bts_buffer_base);
30dd568c
MM		 817 kfree(ds);
818 }
819
820 put_online_cpus();
821}
822
823static int reserve_bts_hardware(void)
824{
825 int cpu, err = 0;
826
827 if (!bts_available())
747b50aa 828 return 0;
30dd568c
MM		 829
830 get_online_cpus();
831
832 for_each_possible_cpu(cpu) {
833 struct debug_store *ds;
834 void *buffer;
835
836 err = -ENOMEM;
837 buffer = kzalloc(BTS_BUFFER_SIZE, GFP_KERNEL);
838 if (unlikely(!buffer))
839 break;
840
841 ds = kzalloc(sizeof(*ds), GFP_KERNEL);
842 if (unlikely(!ds)) {
843 kfree(buffer);
844 break;
845 }
846
596da17f 847 ds->bts_buffer_base = (u64)(unsigned long)buffer;
30dd568c
MM		 848 ds->bts_index = ds->bts_buffer_base;
849 ds->bts_absolute_maximum =
850 ds->bts_buffer_base + BTS_BUFFER_SIZE;
851 ds->bts_interrupt_threshold =
852 ds->bts_absolute_maximum - BTS_OVFL_TH;
853
cdd6c482 854 per_cpu(cpu_hw_events, cpu).ds = ds;
30dd568c
MM		 855 err = 0;
856 }
857
858 if (err)
859 release_bts_hardware();
860 else {
861 for_each_online_cpu(cpu)
862 init_debug_store_on_cpu(cpu);
863 }
864
865 put_online_cpus();
866
867 return err;
868}
869
cdd6c482 870static void hw_perf_event_destroy(struct perf_event *event)
4e935e47 871{
cdd6c482 872 if (atomic_dec_and_mutex_lock(&active_events, &pmc_reserve_mutex)) {
4e935e47 873 release_pmc_hardware();
30dd568c 874 release_bts_hardware();
4e935e47
PZ		 875 mutex_unlock(&pmc_reserve_mutex);
876 }
877}
878
85cf9dba
RR		 879static inline int x86_pmu_initialized(void)
880{
881 return x86_pmu.handle_irq != NULL;
882}
883
8326f44d 884static inline int
cdd6c482 885set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event_attr *attr)
8326f44d
IM		 886{
887 unsigned int cache_type, cache_op, cache_result;
888 u64 config, val;
889
890 config = attr->config;
891
892 cache_type = (config >> 0) & 0xff;
893 if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
894 return -EINVAL;
895
896 cache_op = (config >> 8) & 0xff;
897 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
898 return -EINVAL;
899
900 cache_result = (config >> 16) & 0xff;
901 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
902 return -EINVAL;
903
904 val = hw_cache_event_ids[cache_type][cache_op][cache_result];
905
906 if (val == 0)
907 return -ENOENT;
908
909 if (val == -1)
910 return -EINVAL;
911
912 hwc->config |= val;
913
914 return 0;
915}
916
30dd568c
MM		 917static void intel_pmu_enable_bts(u64 config)
918{
919 unsigned long debugctlmsr;
920
921 debugctlmsr = get_debugctlmsr();
922
923 debugctlmsr |= X86_DEBUGCTL_TR;
924 debugctlmsr |= X86_DEBUGCTL_BTS;
925 debugctlmsr |= X86_DEBUGCTL_BTINT;
926
927 if (!(config & ARCH_PERFMON_EVENTSEL_OS))
928 debugctlmsr |= X86_DEBUGCTL_BTS_OFF_OS;
929
930 if (!(config & ARCH_PERFMON_EVENTSEL_USR))
931 debugctlmsr |= X86_DEBUGCTL_BTS_OFF_USR;
932
933 update_debugctlmsr(debugctlmsr);
934}
935
936static void intel_pmu_disable_bts(void)
937{
cdd6c482 938 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
30dd568c
MM		 939 unsigned long debugctlmsr;
940
941 if (!cpuc->ds)
942 return;
943
944 debugctlmsr = get_debugctlmsr();
945
946 debugctlmsr &=
947 ~(X86_DEBUGCTL_TR | X86_DEBUGCTL_BTS | X86_DEBUGCTL_BTINT |
948 X86_DEBUGCTL_BTS_OFF_OS | X86_DEBUGCTL_BTS_OFF_USR);
949
950 update_debugctlmsr(debugctlmsr);
951}
952
241771ef 953/*
0d48696f 954 * Setup the hardware configuration for a given attr_type
241771ef 955 */
cdd6c482 956static int __hw_perf_event_init(struct perf_event *event)
241771ef 957{
cdd6c482
IM		 958 struct perf_event_attr *attr = &event->attr;
959 struct hw_perf_event *hwc = &event->hw;
9c74fb50 960 u64 config;
4e935e47 961 int err;
241771ef 962
85cf9dba
RR		 963 if (!x86_pmu_initialized())
964 return -ENODEV;
241771ef 965
4e935e47 966 err = 0;
cdd6c482 967 if (!atomic_inc_not_zero(&active_events)) {
4e935e47 968 mutex_lock(&pmc_reserve_mutex);
cdd6c482 969 if (atomic_read(&active_events) == 0) {
30dd568c
MM		 970 if (!reserve_pmc_hardware())
971 err = -EBUSY;
972 else
747b50aa 973 err = reserve_bts_hardware();
30dd568c
MM		 974 }
975 if (!err)
cdd6c482 976 atomic_inc(&active_events);
4e935e47
PZ		 977 mutex_unlock(&pmc_reserve_mutex);
978 }
979 if (err)
980 return err;
981
cdd6c482 982 event->destroy = hw_perf_event_destroy;
a1792cda 983
241771ef 984 /*
0475f9ea 985 * Generate PMC IRQs:
241771ef
IM		 986 * (keep 'enabled' bit clear for now)
987 */
0475f9ea 988 hwc->config = ARCH_PERFMON_EVENTSEL_INT;
241771ef 989
b690081d
SE		 990 hwc->idx = -1;
991
241771ef 992 /*
0475f9ea 993 * Count user and OS events unless requested not to.
241771ef 994 */
0d48696f 995 if (!attr->exclude_user)
0475f9ea 996 hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
0d48696f 997 if (!attr->exclude_kernel)
241771ef 998 hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
0475f9ea 999
bd2b5b12 1000 if (!hwc->sample_period) {
b23f3325 1001 hwc->sample_period = x86_pmu.max_period;
9e350de3 1002 hwc->last_period = hwc->sample_period;
bd2b5b12 1003 atomic64_set(&hwc->period_left, hwc->sample_period);
04da8a43
IM		 1004 } else {
1005 /*
1006 * If we have a PMU initialized but no APIC
1007 * interrupts, we cannot sample hardware
cdd6c482
IM		 1008 * events (user-space has to fall back and
1009 * sample via a hrtimer based software event):
04da8a43
IM		 1010 */
1011 if (!x86_pmu.apic)
1012 return -EOPNOTSUPP;
bd2b5b12 1013 }
d2517a49 1014
241771ef 1015 /*
dfc65094 1016 * Raw hw_event type provide the config in the hw_event structure
241771ef 1017 */
a21ca2ca
IM		 1018 if (attr->type == PERF_TYPE_RAW) {
1019 hwc->config |= x86_pmu.raw_event(attr->config);
8326f44d 1020 return 0;
241771ef 1021 }
241771ef 1022
8326f44d
IM		 1023 if (attr->type == PERF_TYPE_HW_CACHE)
1024 return set_ext_hw_attr(hwc, attr);
1025
1026 if (attr->config >= x86_pmu.max_events)
1027 return -EINVAL;
9c74fb50 1028
8326f44d
IM		 1029 /*
1030 * The generic map:
1031 */
9c74fb50
PZ		 1032 config = x86_pmu.event_map(attr->config);
1033
1034 if (config == 0)
1035 return -ENOENT;
1036
1037 if (config == -1LL)
1038 return -EINVAL;
1039
747b50aa 1040 /*
1041 * Branch tracing:
1042 */
1043 if ((attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS) &&
1653192f 1044 (hwc->sample_period == 1)) {
1045 /* BTS is not supported by this architecture. */
1046 if (!bts_available())
1047 return -EOPNOTSUPP;
1048
1049 /* BTS is currently only allowed for user-mode. */
1050 if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
1051 return -EOPNOTSUPP;
1052 }
747b50aa 1053
9c74fb50 1054 hwc->config |= config;
4e935e47 1055
241771ef
IM		 1056 return 0;
1057}
1058
11d1578f
VW		 1059static void p6_pmu_disable_all(void)
1060{
cdd6c482 1061 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
9c74fb50 1062 u64 val;
11d1578f
VW		 1063
1064 if (!cpuc->enabled)
1065 return;
1066
1067 cpuc->enabled = 0;
1068 barrier();
1069
1070 /* p6 only has one enable register */
1071 rdmsrl(MSR_P6_EVNTSEL0, val);
1072 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
1073 wrmsrl(MSR_P6_EVNTSEL0, val);
1074}
1075
9e35ad38 1076static void intel_pmu_disable_all(void)
4ac13294 1077{
cdd6c482 1078 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
30dd568c
MM		 1079
1080 if (!cpuc->enabled)
1081 return;
1082
1083 cpuc->enabled = 0;
1084 barrier();
1085
862a1a5f 1086 wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
30dd568c
MM		 1087
1088 if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask))
1089 intel_pmu_disable_bts();
241771ef 1090}
b56a3802 1091
9e35ad38 1092static void amd_pmu_disable_all(void)
f87ad35d 1093{
cdd6c482 1094 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
9e35ad38
PZ		 1095 int idx;
1096
1097 if (!cpuc->enabled)
1098 return;
b0f3f28e 1099
b0f3f28e 1100 cpuc->enabled = 0;
60b3df9c
PZ		 1101 /*
1102 * ensure we write the disable before we start disabling the
cdd6c482 1103 * events proper, so that amd_pmu_enable_event() does the
5f4ec28f 1104 * right thing.
60b3df9c 1105 */
b0f3f28e 1106 barrier();
f87ad35d 1107
cdd6c482 1108 for (idx = 0; idx < x86_pmu.num_events; idx++) {
b0f3f28e
PZ		 1109 u64 val;
1110
43f6201a 1111 if (!test_bit(idx, cpuc->active_mask))
4295ee62 1112 continue;
f87ad35d 1113 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
4295ee62
RR		 1114 if (!(val & ARCH_PERFMON_EVENTSEL0_ENABLE))
1115 continue;
1116 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
1117 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
f87ad35d 1118 }
f87ad35d
JSR		 1119}
1120
9e35ad38 1121void hw_perf_disable(void)
b56a3802 1122{
85cf9dba 1123 if (!x86_pmu_initialized())
9e35ad38
PZ		 1124 return;
1125 return x86_pmu.disable_all();
b56a3802 1126}
241771ef 1127
11d1578f
VW		 1128static void p6_pmu_enable_all(void)
1129{
cdd6c482 1130 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
11d1578f
VW		 1131 unsigned long val;
1132
1133 if (cpuc->enabled)
1134 return;
1135
1136 cpuc->enabled = 1;
1137 barrier();
1138
1139 /* p6 only has one enable register */
1140 rdmsrl(MSR_P6_EVNTSEL0, val);
1141 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1142 wrmsrl(MSR_P6_EVNTSEL0, val);
1143}
1144
9e35ad38 1145static void intel_pmu_enable_all(void)
b56a3802 1146{
cdd6c482 1147 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
30dd568c
MM		 1148
1149 if (cpuc->enabled)
1150 return;
1151
1152 cpuc->enabled = 1;
1153 barrier();
1154
9e35ad38 1155 wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
30dd568c
MM		 1156
1157 if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {
cdd6c482
IM		 1158 struct perf_event *event =
1159 cpuc->events[X86_PMC_IDX_FIXED_BTS];
30dd568c 1160
cdd6c482 1161 if (WARN_ON_ONCE(!event))
30dd568c
MM		 1162 return;
1163
cdd6c482 1164 intel_pmu_enable_bts(event->hw.config);
30dd568c 1165 }
b56a3802
JSR		 1166}
1167
9e35ad38 1168static void amd_pmu_enable_all(void)
f87ad35d 1169{
cdd6c482 1170 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
f87ad35d
JSR		 1171 int idx;
1172
9e35ad38 1173 if (cpuc->enabled)
b0f3f28e
PZ		 1174 return;
1175
9e35ad38
PZ		 1176 cpuc->enabled = 1;
1177 barrier();
1178
cdd6c482
IM		 1179 for (idx = 0; idx < x86_pmu.num_events; idx++) {
1180 struct perf_event *event = cpuc->events[idx];
4295ee62 1181 u64 val;
b0f3f28e 1182
43f6201a 1183 if (!test_bit(idx, cpuc->active_mask))
4295ee62 1184 continue;
984b838c 1185
cdd6c482 1186 val = event->hw.config;
4295ee62
RR		 1187 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1188 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
f87ad35d
JSR		 1189 }
1190}
1191
9e35ad38 1192void hw_perf_enable(void)
ee06094f 1193{
85cf9dba 1194 if (!x86_pmu_initialized())
2b9ff0db 1195 return;
9e35ad38 1196 x86_pmu.enable_all();
ee06094f 1197}
ee06094f 1198
19d84dab 1199static inline u64 intel_pmu_get_status(void)
b0f3f28e
PZ		 1200{
1201 u64 status;
1202
b7f8859a 1203 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
b0f3f28e 1204
b7f8859a 1205 return status;
b0f3f28e
PZ		 1206}
1207
dee5d906 1208static inline void intel_pmu_ack_status(u64 ack)
b0f3f28e
PZ		 1209{
1210 wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
1211}
1212
cdd6c482 1213static inline void x86_pmu_enable_event(struct hw_perf_event *hwc, int idx)
b0f3f28e 1214{
11d1578f 1215 (void)checking_wrmsrl(hwc->config_base + idx,
7c90cc45 1216 hwc->config | ARCH_PERFMON_EVENTSEL0_ENABLE);
b0f3f28e
PZ		 1217}
1218
cdd6c482 1219static inline void x86_pmu_disable_event(struct hw_perf_event *hwc, int idx)
b0f3f28e 1220{
11d1578f 1221 (void)checking_wrmsrl(hwc->config_base + idx, hwc->config);
b0f3f28e
PZ		 1222}
1223
2f18d1e8 1224static inline void
cdd6c482 1225intel_pmu_disable_fixed(struct hw_perf_event *hwc, int __idx)
2f18d1e8
IM		 1226{
1227 int idx = __idx - X86_PMC_IDX_FIXED;
1228 u64 ctrl_val, mask;
2f18d1e8
IM		 1229
1230 mask = 0xfULL << (idx * 4);
1231
1232 rdmsrl(hwc->config_base, ctrl_val);
1233 ctrl_val &= ~mask;
11d1578f
VW		 1234 (void)checking_wrmsrl(hwc->config_base, ctrl_val);
1235}
1236
1237static inline void
cdd6c482 1238p6_pmu_disable_event(struct hw_perf_event *hwc, int idx)
11d1578f 1239{
cdd6c482
IM		 1240 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1241 u64 val = P6_NOP_EVENT;
11d1578f 1242
9c74fb50
PZ		 1243 if (cpuc->enabled)
1244 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
11d1578f
VW		 1245
1246 (void)checking_wrmsrl(hwc->config_base + idx, val);
2f18d1e8
IM		 1247}
1248
7e2ae347 1249static inline void
cdd6c482 1250intel_pmu_disable_event(struct hw_perf_event *hwc, int idx)
7e2ae347 1251{
30dd568c
MM		 1252 if (unlikely(idx == X86_PMC_IDX_FIXED_BTS)) {
1253 intel_pmu_disable_bts();
1254 return;
1255 }
1256
d4369891
RR		 1257 if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
1258 intel_pmu_disable_fixed(hwc, idx);
1259 return;
1260 }
1261
cdd6c482 1262 x86_pmu_disable_event(hwc, idx);
d4369891
RR		 1263}
1264
1265static inline void
cdd6c482 1266amd_pmu_disable_event(struct hw_perf_event *hwc, int idx)
d4369891 1267{
cdd6c482 1268 x86_pmu_disable_event(hwc, idx);
7e2ae347
IM		 1269}
1270
245b2e70 1271static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
241771ef 1272
ee06094f
IM		 1273/*
1274 * Set the next IRQ period, based on the hwc->period_left value.
cdd6c482 1275 * To be called with the event disabled in hw:
ee06094f 1276 */
e4abb5d4 1277static int
cdd6c482
IM		 1278x86_perf_event_set_period(struct perf_event *event,
1279 struct hw_perf_event *hwc, int idx)
241771ef 1280{
2f18d1e8 1281 s64 left = atomic64_read(&hwc->period_left);
e4abb5d4
PZ		 1282 s64 period = hwc->sample_period;
1283 int err, ret = 0;
ee06094f 1284
30dd568c
MM		 1285 if (idx == X86_PMC_IDX_FIXED_BTS)
1286 return 0;
1287
ee06094f 1288 /*
af901ca1 1289 * If we are way outside a reasonable range then just skip forward:
ee06094f
IM		 1290 */
1291 if (unlikely(left <= -period)) {
1292 left = period;
1293 atomic64_set(&hwc->period_left, left);
9e350de3 1294 hwc->last_period = period;
e4abb5d4 1295 ret = 1;
ee06094f
IM		 1296 }
1297
1298 if (unlikely(left <= 0)) {
1299 left += period;
1300 atomic64_set(&hwc->period_left, left);
9e350de3 1301 hwc->last_period = period;
e4abb5d4 1302 ret = 1;
ee06094f 1303 }
1c80f4b5 1304 /*
dfc65094 1305 * Quirk: certain CPUs dont like it if just 1 hw_event is left:
1c80f4b5
IM		 1306 */
1307 if (unlikely(left < 2))
1308 left = 2;
241771ef 1309
e4abb5d4
PZ		 1310 if (left > x86_pmu.max_period)
1311 left = x86_pmu.max_period;
1312
245b2e70 1313 per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
ee06094f
IM		 1314
1315 /*
cdd6c482 1316 * The hw event starts counting from this event offset,
ee06094f
IM		 1317 * mark it to be able to extra future deltas:
1318 */
2f18d1e8 1319 atomic64_set(&hwc->prev_count, (u64)-left);
ee06094f 1320
cdd6c482
IM		 1321 err = checking_wrmsrl(hwc->event_base + idx,
1322 (u64)(-left) & x86_pmu.event_mask);
e4abb5d4 1323
cdd6c482 1324 perf_event_update_userpage(event);
194002b2 1325
e4abb5d4 1326 return ret;
2f18d1e8
IM		 1327}
1328
1329static inline void
cdd6c482 1330intel_pmu_enable_fixed(struct hw_perf_event *hwc, int __idx)
2f18d1e8
IM		 1331{
1332 int idx = __idx - X86_PMC_IDX_FIXED;
1333 u64 ctrl_val, bits, mask;
1334 int err;
1335
1336 /*
0475f9ea
PM		 1337 * Enable IRQ generation (0x8),
1338 * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
1339 * if requested:
2f18d1e8 1340 */
0475f9ea
PM		 1341 bits = 0x8ULL;
1342 if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
1343 bits |= 0x2;
2f18d1e8
IM		 1344 if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
1345 bits |= 0x1;
1346 bits <<= (idx * 4);
1347 mask = 0xfULL << (idx * 4);
1348
1349 rdmsrl(hwc->config_base, ctrl_val);
1350 ctrl_val &= ~mask;
1351 ctrl_val |= bits;
1352 err = checking_wrmsrl(hwc->config_base, ctrl_val);
7e2ae347
IM		 1353}
1354
cdd6c482 1355static void p6_pmu_enable_event(struct hw_perf_event *hwc, int idx)
11d1578f 1356{
cdd6c482 1357 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
984b838c 1358 u64 val;
11d1578f 1359
984b838c 1360 val = hwc->config;
11d1578f 1361 if (cpuc->enabled)
984b838c
PZ		 1362 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1363
1364 (void)checking_wrmsrl(hwc->config_base + idx, val);
11d1578f
VW		 1365}
1366
1367
cdd6c482 1368static void intel_pmu_enable_event(struct hw_perf_event *hwc, int idx)
7e2ae347 1369{
30dd568c 1370 if (unlikely(idx == X86_PMC_IDX_FIXED_BTS)) {
cdd6c482 1371 if (!__get_cpu_var(cpu_hw_events).enabled)
30dd568c
MM		 1372 return;
1373
1374 intel_pmu_enable_bts(hwc->config);
1375 return;
1376 }
1377
7c90cc45
RR		 1378 if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
1379 intel_pmu_enable_fixed(hwc, idx);
1380 return;
1381 }
1382
cdd6c482 1383 x86_pmu_enable_event(hwc, idx);
7c90cc45
RR		 1384}
1385
cdd6c482 1386static void amd_pmu_enable_event(struct hw_perf_event *hwc, int idx)
7c90cc45 1387{
cdd6c482 1388 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
7c90cc45
RR		 1389
1390 if (cpuc->enabled)
cdd6c482 1391 x86_pmu_enable_event(hwc, idx);
241771ef
IM		 1392}
1393
fe9081cc 1394static int fixed_mode_idx(struct hw_perf_event *hwc)
862a1a5f 1395{
dfc65094 1396 unsigned int hw_event;
2f18d1e8 1397
dfc65094 1398 hw_event = hwc->config & ARCH_PERFMON_EVENT_MASK;
30dd568c 1399
dfc65094 1400 if (unlikely((hw_event ==
30dd568c
MM		 1401 x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS)) &&
1402 (hwc->sample_period == 1)))
1403 return X86_PMC_IDX_FIXED_BTS;
1404
cdd6c482 1405 if (!x86_pmu.num_events_fixed)
f87ad35d
JSR		 1406 return -1;
1407
04a705df
SE		 1408 /*
1409 * fixed counters do not take all possible filters
1410 */
1411 if (hwc->config & ARCH_PERFMON_EVENT_FILTER_MASK)
1412 return -1;
1413
dfc65094 1414 if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_INSTRUCTIONS)))
2f18d1e8 1415 return X86_PMC_IDX_FIXED_INSTRUCTIONS;
dfc65094 1416 if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_CPU_CYCLES)))
2f18d1e8 1417 return X86_PMC_IDX_FIXED_CPU_CYCLES;
dfc65094 1418 if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_BUS_CYCLES)))
2f18d1e8
IM		 1419 return X86_PMC_IDX_FIXED_BUS_CYCLES;
1420
862a1a5f
IM		 1421 return -1;
1422}
1423
b690081d
SE		 1424/*
1425 * generic counter allocator: get next free counter
1426 */
fe9081cc
PZ		 1427static int
1428gen_get_event_idx(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
b690081d 1429{
b690081d
SE		 1430 int idx;
1431
1432 idx = find_first_zero_bit(cpuc->used_mask, x86_pmu.num_events);
1433 return idx == x86_pmu.num_events ? -1 : idx;
1434}
1435
1436/*
1437 * intel-specific counter allocator: check event constraints
1438 */
fe9081cc
PZ		 1439static int
1440intel_get_event_idx(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
b690081d 1441{
b690081d
SE		 1442 const struct event_constraint *event_constraint;
1443 int i, code;
1444
7a693d3f 1445 if (!event_constraints)
b690081d
SE		 1446 goto skip;
1447
fe9081cc 1448 code = hwc->config & CORE_EVNTSEL_EVENT_MASK;
b690081d 1449
7a693d3f 1450 for_each_event_constraint(event_constraint, event_constraints) {
b690081d
SE		 1451 if (code == event_constraint->code) {
1452 for_each_bit(i, event_constraint->idxmsk, X86_PMC_IDX_MAX) {
1453 if (!test_and_set_bit(i, cpuc->used_mask))
1454 return i;
1455 }
1456 return -1;
1457 }
1458 }
1459skip:
fe9081cc 1460 return gen_get_event_idx(cpuc, hwc);
b690081d
SE		 1461}
1462
fe9081cc
PZ		 1463static int
1464x86_schedule_event(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
241771ef 1465{
2f18d1e8 1466 int idx;
241771ef 1467
fe9081cc 1468 idx = fixed_mode_idx(hwc);
30dd568c 1469 if (idx == X86_PMC_IDX_FIXED_BTS) {
747b50aa 1470 /* BTS is already occupied. */
30dd568c 1471 if (test_and_set_bit(idx, cpuc->used_mask))
747b50aa 1472 return -EAGAIN;
30dd568c
MM		 1473
1474 hwc->config_base = 0;
fe9081cc 1475 hwc->event_base = 0;
30dd568c
MM		 1476 hwc->idx = idx;
1477 } else if (idx >= 0) {
2f18d1e8 1478 /*
cdd6c482
IM		 1479 * Try to get the fixed event, if that is already taken
1480 * then try to get a generic event:
2f18d1e8 1481 */
43f6201a 1482 if (test_and_set_bit(idx, cpuc->used_mask))
2f18d1e8 1483 goto try_generic;
0dff86aa 1484
2f18d1e8
IM		 1485 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
1486 /*
cdd6c482 1487 * We set it so that event_base + idx in wrmsr/rdmsr maps to
2f18d1e8
IM		 1488 * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
1489 */
cdd6c482 1490 hwc->event_base =
2f18d1e8 1491 MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
241771ef 1492 hwc->idx = idx;
2f18d1e8
IM		 1493 } else {
1494 idx = hwc->idx;
cdd6c482 1495 /* Try to get the previous generic event again */
b690081d 1496 if (idx == -1 || test_and_set_bit(idx, cpuc->used_mask)) {
2f18d1e8 1497try_generic:
fe9081cc 1498 idx = x86_pmu.get_event_idx(cpuc, hwc);
b690081d 1499 if (idx == -1)
2f18d1e8
IM		 1500 return -EAGAIN;
1501
43f6201a 1502 set_bit(idx, cpuc->used_mask);
2f18d1e8
IM		 1503 hwc->idx = idx;
1504 }
fe9081cc
PZ		 1505 hwc->config_base = x86_pmu.eventsel;
1506 hwc->event_base = x86_pmu.perfctr;
241771ef
IM		 1507 }
1508
fe9081cc
PZ		 1509 return idx;
1510}
1511
1512/*
1513 * Find a PMC slot for the freshly enabled / scheduled in event:
1514 */
1515static int x86_pmu_enable(struct perf_event *event)
1516{
1517 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1518 struct hw_perf_event *hwc = &event->hw;
1519 int idx;
1520
1521 idx = x86_schedule_event(cpuc, hwc);
1522 if (idx < 0)
1523 return idx;
1524
cdd6c482 1525 perf_events_lapic_init();
53b441a5 1526
d4369891 1527 x86_pmu.disable(hwc, idx);
241771ef 1528
cdd6c482 1529 cpuc->events[idx] = event;
43f6201a 1530 set_bit(idx, cpuc->active_mask);
7e2ae347 1531
cdd6c482 1532 x86_perf_event_set_period(event, hwc, idx);
7c90cc45 1533 x86_pmu.enable(hwc, idx);
95cdd2e7 1534
cdd6c482 1535 perf_event_update_userpage(event);
194002b2 1536
95cdd2e7 1537 return 0;
241771ef
IM		 1538}
1539
cdd6c482 1540static void x86_pmu_unthrottle(struct perf_event *event)
a78ac325 1541{
cdd6c482
IM		 1542 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1543 struct hw_perf_event *hwc = &event->hw;
a78ac325
PZ		 1544
1545 if (WARN_ON_ONCE(hwc->idx >= X86_PMC_IDX_MAX ||
cdd6c482 1546 cpuc->events[hwc->idx] != event))
a78ac325
PZ		 1547 return;
1548
1549 x86_pmu.enable(hwc, hwc->idx);
1550}
1551
cdd6c482 1552void perf_event_print_debug(void)
241771ef 1553{
2f18d1e8 1554 u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
cdd6c482 1555 struct cpu_hw_events *cpuc;
5bb9efe3 1556 unsigned long flags;
1e125676
IM		 1557 int cpu, idx;
1558
cdd6c482 1559 if (!x86_pmu.num_events)
1e125676 1560 return;
241771ef 1561
5bb9efe3 1562 local_irq_save(flags);
241771ef
IM		 1563
1564 cpu = smp_processor_id();
cdd6c482 1565 cpuc = &per_cpu(cpu_hw_events, cpu);
241771ef 1566
faa28ae0 1567 if (x86_pmu.version >= 2) {
a1ef58f4
JSR		 1568 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
1569 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1570 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
1571 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
1572
1573 pr_info("\n");
1574 pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl);
1575 pr_info("CPU#%d: status: %016llx\n", cpu, status);
1576 pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow);
1577 pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed);
f87ad35d 1578 }
43f6201a 1579 pr_info("CPU#%d: used: %016llx\n", cpu, *(u64 *)cpuc->used_mask);
241771ef 1580
cdd6c482 1581 for (idx = 0; idx < x86_pmu.num_events; idx++) {
4a06bd85
RR		 1582 rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
1583 rdmsrl(x86_pmu.perfctr + idx, pmc_count);
241771ef 1584
245b2e70 1585 prev_left = per_cpu(pmc_prev_left[idx], cpu);
241771ef 1586
a1ef58f4 1587 pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n",
241771ef 1588 cpu, idx, pmc_ctrl);
a1ef58f4 1589 pr_info("CPU#%d: gen-PMC%d count: %016llx\n",
241771ef 1590 cpu, idx, pmc_count);
a1ef58f4 1591 pr_info("CPU#%d: gen-PMC%d left: %016llx\n",
ee06094f 1592 cpu, idx, prev_left);
241771ef 1593 }
cdd6c482 1594 for (idx = 0; idx < x86_pmu.num_events_fixed; idx++) {
2f18d1e8
IM		 1595 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
1596
a1ef58f4 1597 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
2f18d1e8
IM		 1598 cpu, idx, pmc_count);
1599 }
5bb9efe3 1600 local_irq_restore(flags);
241771ef
IM		 1601}
1602
cdd6c482 1603static void intel_pmu_drain_bts_buffer(struct cpu_hw_events *cpuc)
30dd568c
MM		 1604{
1605 struct debug_store *ds = cpuc->ds;
1606 struct bts_record {
1607 u64 from;
1608 u64 to;
1609 u64 flags;
1610 };
cdd6c482 1611 struct perf_event *event = cpuc->events[X86_PMC_IDX_FIXED_BTS];
596da17f 1612 struct bts_record *at, *top;
5622f295
MM		 1613 struct perf_output_handle handle;
1614 struct perf_event_header header;
1615 struct perf_sample_data data;
1616 struct pt_regs regs;
30dd568c 1617
cdd6c482 1618 if (!event)
30dd568c
MM		 1619 return;
1620
1621 if (!ds)
1622 return;
1623
596da17f 1624 at = (struct bts_record *)(unsigned long)ds->bts_buffer_base;
1625 top = (struct bts_record *)(unsigned long)ds->bts_index;
30dd568c 1626
5622f295
MM		 1627 if (top <= at)
1628 return;
1629
596da17f 1630 ds->bts_index = ds->bts_buffer_base;
1631
5622f295 1632
cdd6c482 1633 data.period = event->hw.last_period;
5622f295 1634 data.addr = 0;
5e855db5 1635 data.raw = NULL;
5622f295
MM		 1636 regs.ip = 0;
1637
1638 /*
1639 * Prepare a generic sample, i.e. fill in the invariant fields.
1640 * We will overwrite the from and to address before we output
1641 * the sample.
1642 */
cdd6c482 1643 perf_prepare_sample(&header, &data, event, &regs);
5622f295 1644
cdd6c482 1645 if (perf_output_begin(&handle, event,
5622f295
MM		 1646 header.size * (top - at), 1, 1))
1647 return;
1648
596da17f 1649 for (; at < top; at++) {
5622f295
MM		 1650 data.ip = at->from;
1651 data.addr = at->to;
30dd568c 1652
cdd6c482 1653 perf_output_sample(&handle, &header, &data, event);
30dd568c
MM		 1654 }
1655
5622f295 1656 perf_output_end(&handle);
30dd568c
MM		 1657
1658 /* There's new data available. */
cdd6c482
IM		 1659 event->hw.interrupts++;
1660 event->pending_kill = POLL_IN;
30dd568c
MM		 1661}
1662
cdd6c482 1663static void x86_pmu_disable(struct perf_event *event)
241771ef 1664{
cdd6c482
IM		 1665 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1666 struct hw_perf_event *hwc = &event->hw;
6f00cada 1667 int idx = hwc->idx;
241771ef 1668
09534238
RR		 1669 /*
1670 * Must be done before we disable, otherwise the nmi handler
1671 * could reenable again:
1672 */
43f6201a 1673 clear_bit(idx, cpuc->active_mask);
d4369891 1674 x86_pmu.disable(hwc, idx);
241771ef 1675
2f18d1e8
IM		 1676 /*
1677 * Make sure the cleared pointer becomes visible before we
cdd6c482 1678 * (potentially) free the event:
2f18d1e8 1679 */
527e26af 1680 barrier();
241771ef 1681
ee06094f 1682 /*
cdd6c482 1683 * Drain the remaining delta count out of a event
ee06094f
IM		 1684 * that we are disabling:
1685 */
cdd6c482 1686 x86_perf_event_update(event, hwc, idx);
30dd568c
MM		 1687
1688 /* Drain the remaining BTS records. */
5622f295
MM		 1689 if (unlikely(idx == X86_PMC_IDX_FIXED_BTS))
1690 intel_pmu_drain_bts_buffer(cpuc);
30dd568c 1691
cdd6c482 1692 cpuc->events[idx] = NULL;
43f6201a 1693 clear_bit(idx, cpuc->used_mask);
194002b2 1694
cdd6c482 1695 perf_event_update_userpage(event);
241771ef
IM		 1696}
1697
7e2ae347 1698/*
cdd6c482
IM		 1699 * Save and restart an expired event. Called by NMI contexts,
1700 * so it has to be careful about preempting normal event ops:
7e2ae347 1701 */
cdd6c482 1702static int intel_pmu_save_and_restart(struct perf_event *event)
241771ef 1703{
cdd6c482 1704 struct hw_perf_event *hwc = &event->hw;
241771ef 1705 int idx = hwc->idx;
e4abb5d4 1706 int ret;
241771ef 1707
cdd6c482
IM		 1708 x86_perf_event_update(event, hwc, idx);
1709 ret = x86_perf_event_set_period(event, hwc, idx);
7e2ae347 1710
cdd6c482
IM		 1711 if (event->state == PERF_EVENT_STATE_ACTIVE)
1712 intel_pmu_enable_event(hwc, idx);
e4abb5d4
PZ		 1713
1714 return ret;
241771ef
IM		 1715}
1716
aaba9801
IM		 1717static void intel_pmu_reset(void)
1718{
cdd6c482 1719 struct debug_store *ds = __get_cpu_var(cpu_hw_events).ds;
aaba9801
IM		 1720 unsigned long flags;
1721 int idx;
1722
cdd6c482 1723 if (!x86_pmu.num_events)
aaba9801
IM		 1724 return;
1725
1726 local_irq_save(flags);
1727
1728 printk("clearing PMU state on CPU#%d\n", smp_processor_id());
1729
cdd6c482 1730 for (idx = 0; idx < x86_pmu.num_events; idx++) {
aaba9801
IM		 1731 checking_wrmsrl(x86_pmu.eventsel + idx, 0ull);
1732 checking_wrmsrl(x86_pmu.perfctr + idx, 0ull);
1733 }
cdd6c482 1734 for (idx = 0; idx < x86_pmu.num_events_fixed; idx++) {
aaba9801
IM		 1735 checking_wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
1736 }
30dd568c
MM		 1737 if (ds)
1738 ds->bts_index = ds->bts_buffer_base;
aaba9801
IM		 1739
1740 local_irq_restore(flags);
1741}
1742
11d1578f
VW		 1743static int p6_pmu_handle_irq(struct pt_regs *regs)
1744{
1745 struct perf_sample_data data;
cdd6c482
IM		 1746 struct cpu_hw_events *cpuc;
1747 struct perf_event *event;
1748 struct hw_perf_event *hwc;
11d1578f
VW		 1749 int idx, handled = 0;
1750 u64 val;
1751
11d1578f 1752 data.addr = 0;
5e855db5 1753 data.raw = NULL;
11d1578f 1754
cdd6c482 1755 cpuc = &__get_cpu_var(cpu_hw_events);
11d1578f 1756
cdd6c482 1757 for (idx = 0; idx < x86_pmu.num_events; idx++) {
11d1578f
VW		 1758 if (!test_bit(idx, cpuc->active_mask))
1759 continue;
1760
cdd6c482
IM		 1761 event = cpuc->events[idx];
1762 hwc = &event->hw;
11d1578f 1763
cdd6c482
IM		 1764 val = x86_perf_event_update(event, hwc, idx);
1765 if (val & (1ULL << (x86_pmu.event_bits - 1)))
11d1578f
VW		 1766 continue;
1767
1768 /*
cdd6c482 1769 * event overflow
11d1578f
VW		 1770 */
1771 handled = 1;
cdd6c482 1772 data.period = event->hw.last_period;
11d1578f 1773
cdd6c482 1774 if (!x86_perf_event_set_period(event, hwc, idx))
11d1578f
VW		 1775 continue;
1776
cdd6c482
IM		 1777 if (perf_event_overflow(event, 1, &data, regs))
1778 p6_pmu_disable_event(hwc, idx);
11d1578f
VW		 1779 }
1780
1781 if (handled)
1782 inc_irq_stat(apic_perf_irqs);
1783
1784 return handled;
1785}
aaba9801 1786
241771ef
IM		 1787/*
1788 * This handler is triggered by the local APIC, so the APIC IRQ handling
1789 * rules apply:
1790 */
a3288106 1791static int intel_pmu_handle_irq(struct pt_regs *regs)
241771ef 1792{
df1a132b 1793 struct perf_sample_data data;
cdd6c482 1794 struct cpu_hw_events *cpuc;
11d1578f 1795 int bit, loops;
4b39fd96 1796 u64 ack, status;
9029a5e3 1797
df1a132b 1798 data.addr = 0;
5e855db5 1799 data.raw = NULL;
df1a132b 1800
cdd6c482 1801 cpuc = &__get_cpu_var(cpu_hw_events);
241771ef 1802
9e35ad38 1803 perf_disable();
5622f295 1804 intel_pmu_drain_bts_buffer(cpuc);
19d84dab 1805 status = intel_pmu_get_status();
9e35ad38
PZ		 1806 if (!status) {
1807 perf_enable();
1808 return 0;
1809 }
87b9cf46 1810
9029a5e3 1811 loops = 0;
241771ef 1812again:
9029a5e3 1813 if (++loops > 100) {
cdd6c482
IM		 1814 WARN_ONCE(1, "perfevents: irq loop stuck!\n");
1815 perf_event_print_debug();
aaba9801
IM		 1816 intel_pmu_reset();
1817 perf_enable();
9029a5e3
IM		 1818 return 1;
1819 }
1820
d278c484 1821 inc_irq_stat(apic_perf_irqs);
241771ef 1822 ack = status;
2f18d1e8 1823 for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
cdd6c482 1824 struct perf_event *event = cpuc->events[bit];
241771ef
IM		 1825
1826 clear_bit(bit, (unsigned long *) &status);
43f6201a 1827 if (!test_bit(bit, cpuc->active_mask))
241771ef
IM		 1828 continue;
1829
cdd6c482 1830 if (!intel_pmu_save_and_restart(event))
e4abb5d4
PZ		 1831 continue;
1832
cdd6c482 1833 data.period = event->hw.last_period;
60f916de 1834
cdd6c482
IM		 1835 if (perf_event_overflow(event, 1, &data, regs))
1836 intel_pmu_disable_event(&event->hw, bit);
241771ef
IM		 1837 }
1838
dee5d906 1839 intel_pmu_ack_status(ack);
241771ef
IM		 1840
1841 /*
1842 * Repeat if there is more work to be done:
1843 */
19d84dab 1844 status = intel_pmu_get_status();
241771ef
IM		 1845 if (status)
1846 goto again;
b0f3f28e 1847
48e22d56 1848 perf_enable();
9e35ad38
PZ		 1849
1850 return 1;
1b023a96
MG		 1851}
1852
a3288106 1853static int amd_pmu_handle_irq(struct pt_regs *regs)
a29aa8a7 1854{
df1a132b 1855 struct perf_sample_data data;
cdd6c482
IM		 1856 struct cpu_hw_events *cpuc;
1857 struct perf_event *event;
1858 struct hw_perf_event *hwc;
11d1578f 1859 int idx, handled = 0;
9029a5e3
IM		 1860 u64 val;
1861
df1a132b 1862 data.addr = 0;
5e855db5 1863 data.raw = NULL;
df1a132b 1864
cdd6c482 1865 cpuc = &__get_cpu_var(cpu_hw_events);
962bf7a6 1866
cdd6c482 1867 for (idx = 0; idx < x86_pmu.num_events; idx++) {
43f6201a 1868 if (!test_bit(idx, cpuc->active_mask))
a29aa8a7 1869 continue;
962bf7a6 1870
cdd6c482
IM		 1871 event = cpuc->events[idx];
1872 hwc = &event->hw;
a4016a79 1873
cdd6c482
IM		 1874 val = x86_perf_event_update(event, hwc, idx);
1875 if (val & (1ULL << (x86_pmu.event_bits - 1)))
48e22d56 1876 continue;
962bf7a6 1877
9e350de3 1878 /*
cdd6c482 1879 * event overflow
9e350de3
PZ		 1880 */
1881 handled = 1;
cdd6c482 1882 data.period = event->hw.last_period;
9e350de3 1883
cdd6c482 1884 if (!x86_perf_event_set_period(event, hwc, idx))
e4abb5d4
PZ		 1885 continue;
1886
cdd6c482
IM		 1887 if (perf_event_overflow(event, 1, &data, regs))
1888 amd_pmu_disable_event(hwc, idx);
a29aa8a7 1889 }
962bf7a6 1890
9e350de3
PZ		 1891 if (handled)
1892 inc_irq_stat(apic_perf_irqs);
1893
a29aa8a7
RR		 1894 return handled;
1895}
39d81eab 1896
b6276f35
PZ		 1897void smp_perf_pending_interrupt(struct pt_regs *regs)
1898{
1899 irq_enter();
1900 ack_APIC_irq();
1901 inc_irq_stat(apic_pending_irqs);
cdd6c482 1902 perf_event_do_pending();
b6276f35
PZ		 1903 irq_exit();
1904}
1905
cdd6c482 1906void set_perf_event_pending(void)
b6276f35 1907{
04da8a43 1908#ifdef CONFIG_X86_LOCAL_APIC
7d428966
PZ		 1909 if (!x86_pmu.apic || !x86_pmu_initialized())
1910 return;
1911
b6276f35 1912 apic->send_IPI_self(LOCAL_PENDING_VECTOR);
04da8a43 1913#endif
b6276f35
PZ		 1914}
1915
cdd6c482 1916void perf_events_lapic_init(void)
241771ef 1917{
04da8a43
IM		 1918#ifdef CONFIG_X86_LOCAL_APIC
1919 if (!x86_pmu.apic || !x86_pmu_initialized())
241771ef 1920 return;
85cf9dba 1921
241771ef 1922 /*
c323d95f 1923 * Always use NMI for PMU
241771ef 1924 */
c323d95f 1925 apic_write(APIC_LVTPC, APIC_DM_NMI);
04da8a43 1926#endif
241771ef
IM		 1927}
1928
1929static int __kprobes
cdd6c482 1930perf_event_nmi_handler(struct notifier_block *self,
241771ef
IM		 1931 unsigned long cmd, void *__args)
1932{
1933 struct die_args *args = __args;
1934 struct pt_regs *regs;
b0f3f28e 1935
cdd6c482 1936 if (!atomic_read(&active_events))
63a809a2
PZ		 1937 return NOTIFY_DONE;
1938
b0f3f28e
PZ		 1939 switch (cmd) {
1940 case DIE_NMI:
1941 case DIE_NMI_IPI:
1942 break;
241771ef 1943
b0f3f28e 1944 default:
241771ef 1945 return NOTIFY_DONE;
b0f3f28e 1946 }
241771ef
IM		 1947
1948 regs = args->regs;
1949
04da8a43 1950#ifdef CONFIG_X86_LOCAL_APIC
241771ef 1951 apic_write(APIC_LVTPC, APIC_DM_NMI);
04da8a43 1952#endif
a4016a79
PZ		 1953 /*
1954 * Can't rely on the handled return value to say it was our NMI, two
cdd6c482 1955 * events could trigger 'simultaneously' raising two back-to-back NMIs.
a4016a79
PZ		 1956 *
1957 * If the first NMI handles both, the latter will be empty and daze
1958 * the CPU.
1959 */
a3288106 1960 x86_pmu.handle_irq(regs);
241771ef 1961
a4016a79 1962 return NOTIFY_STOP;
241771ef
IM		 1963}
1964
cdd6c482
IM		 1965static __read_mostly struct notifier_block perf_event_nmi_notifier = {
1966 .notifier_call = perf_event_nmi_handler,
5b75af0a
MG		 1967 .next = NULL,
1968 .priority = 1
241771ef
IM		 1969};
1970
db48cccc 1971static __initconst struct x86_pmu p6_pmu = {
11d1578f
VW		 1972 .name = "p6",
1973 .handle_irq = p6_pmu_handle_irq,
1974 .disable_all = p6_pmu_disable_all,
1975 .enable_all = p6_pmu_enable_all,
cdd6c482
IM		 1976 .enable = p6_pmu_enable_event,
1977 .disable = p6_pmu_disable_event,
11d1578f
VW		 1978 .eventsel = MSR_P6_EVNTSEL0,
1979 .perfctr = MSR_P6_PERFCTR0,
1980 .event_map = p6_pmu_event_map,
1981 .raw_event = p6_pmu_raw_event,
1982 .max_events = ARRAY_SIZE(p6_perfmon_event_map),
04da8a43 1983 .apic = 1,
11d1578f
VW		 1984 .max_period = (1ULL << 31) - 1,
1985 .version = 0,
cdd6c482 1986 .num_events = 2,
11d1578f 1987 /*
cdd6c482 1988 * Events have 40 bits implemented. However they are designed such
11d1578f 1989 * that bits [32-39] are sign extensions of bit 31. As such the
cdd6c482 1990 * effective width of a event for P6-like PMU is 32 bits only.
11d1578f
VW		 1991 *
1992 * See IA-32 Intel Architecture Software developer manual Vol 3B
1993 */
cdd6c482
IM		 1994 .event_bits = 32,
1995 .event_mask = (1ULL << 32) - 1,
b690081d 1996 .get_event_idx = intel_get_event_idx,
11d1578f
VW		 1997};
1998
db48cccc 1999static __initconst struct x86_pmu intel_pmu = {
faa28ae0 2000 .name = "Intel",
39d81eab 2001 .handle_irq = intel_pmu_handle_irq,
9e35ad38
PZ		 2002 .disable_all = intel_pmu_disable_all,
2003 .enable_all = intel_pmu_enable_all,
cdd6c482
IM		 2004 .enable = intel_pmu_enable_event,
2005 .disable = intel_pmu_disable_event,
b56a3802
JSR		 2006 .eventsel = MSR_ARCH_PERFMON_EVENTSEL0,
2007 .perfctr = MSR_ARCH_PERFMON_PERFCTR0,
5f4ec28f
RR		 2008 .event_map = intel_pmu_event_map,
2009 .raw_event = intel_pmu_raw_event,
b56a3802 2010 .max_events = ARRAY_SIZE(intel_perfmon_event_map),
04da8a43 2011 .apic = 1,
c619b8ff
RR		 2012 /*
2013 * Intel PMCs cannot be accessed sanely above 32 bit width,
2014 * so we install an artificial 1<<31 period regardless of
cdd6c482 2015 * the generic event period:
c619b8ff
RR		 2016 */
2017 .max_period = (1ULL << 31) - 1,
30dd568c
MM		 2018 .enable_bts = intel_pmu_enable_bts,
2019 .disable_bts = intel_pmu_disable_bts,
b690081d 2020 .get_event_idx = intel_get_event_idx,
b56a3802
JSR		 2021};
2022
db48cccc 2023static __initconst struct x86_pmu amd_pmu = {
faa28ae0 2024 .name = "AMD",
39d81eab 2025 .handle_irq = amd_pmu_handle_irq,
9e35ad38
PZ		 2026 .disable_all = amd_pmu_disable_all,
2027 .enable_all = amd_pmu_enable_all,
cdd6c482
IM		 2028 .enable = amd_pmu_enable_event,
2029 .disable = amd_pmu_disable_event,
f87ad35d
JSR		 2030 .eventsel = MSR_K7_EVNTSEL0,
2031 .perfctr = MSR_K7_PERFCTR0,
5f4ec28f
RR		 2032 .event_map = amd_pmu_event_map,
2033 .raw_event = amd_pmu_raw_event,
f87ad35d 2034 .max_events = ARRAY_SIZE(amd_perfmon_event_map),
cdd6c482
IM		 2035 .num_events = 4,
2036 .event_bits = 48,
2037 .event_mask = (1ULL << 48) - 1,
04da8a43 2038 .apic = 1,
c619b8ff
RR		 2039 /* use highest bit to detect overflow */
2040 .max_period = (1ULL << 47) - 1,
b690081d 2041 .get_event_idx = gen_get_event_idx,
f87ad35d
JSR		 2042};
2043
db48cccc 2044static __init int p6_pmu_init(void)
11d1578f 2045{
11d1578f
VW		 2046 switch (boot_cpu_data.x86_model) {
2047 case 1:
2048 case 3: /* Pentium Pro */
2049 case 5:
2050 case 6: /* Pentium II */
2051 case 7:
2052 case 8:
2053 case 11: /* Pentium III */
7a693d3f 2054 event_constraints = intel_p6_event_constraints;
11d1578f
VW		 2055 break;
2056 case 9:
2057 case 13:
f1c6a581 2058 /* Pentium M */
7a693d3f 2059 event_constraints = intel_p6_event_constraints;
f1c6a581 2060 break;
11d1578f
VW		 2061 default:
2062 pr_cont("unsupported p6 CPU model %d ",
2063 boot_cpu_data.x86_model);
2064 return -ENODEV;
2065 }
2066
04da8a43
IM		 2067 x86_pmu = p6_pmu;
2068
11d1578f
VW		 2069 return 0;
2070}
2071
db48cccc 2072static __init int intel_pmu_init(void)
241771ef 2073{
7bb497bd 2074 union cpuid10_edx edx;
241771ef 2075 union cpuid10_eax eax;
703e937c 2076 unsigned int unused;
7bb497bd 2077 unsigned int ebx;
faa28ae0 2078 int version;
241771ef 2079
11d1578f
VW		 2080 if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
2081 /* check for P6 processor family */
2082 if (boot_cpu_data.x86 == 6) {
2083 return p6_pmu_init();
2084 } else {
72eae04d 2085 return -ENODEV;
11d1578f
VW		 2086 }
2087 }
da1a776b 2088
241771ef
IM		 2089 /*
2090 * Check whether the Architectural PerfMon supports
dfc65094 2091 * Branch Misses Retired hw_event or not.
241771ef 2092 */
703e937c 2093 cpuid(10, &eax.full, &ebx, &unused, &edx.full);
241771ef 2094 if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
72eae04d 2095 return -ENODEV;
241771ef 2096
faa28ae0
RR		 2097 version = eax.split.version_id;
2098 if (version < 2)
72eae04d 2099 return -ENODEV;
7bb497bd 2100
1123e3ad
IM		 2101 x86_pmu = intel_pmu;
2102 x86_pmu.version = version;
cdd6c482
IM		 2103 x86_pmu.num_events = eax.split.num_events;
2104 x86_pmu.event_bits = eax.split.bit_width;
2105 x86_pmu.event_mask = (1ULL << eax.split.bit_width) - 1;
066d7dea
IM		 2106
2107 /*
cdd6c482
IM		 2108 * Quirk: v2 perfmon does not report fixed-purpose events, so
2109 * assume at least 3 events:
066d7dea 2110 */
cdd6c482 2111 x86_pmu.num_events_fixed = max((int)edx.split.num_events_fixed, 3);
b56a3802 2112
8326f44d 2113 /*
1123e3ad 2114 * Install the hw-cache-events table:
8326f44d
IM		 2115 */
2116 switch (boot_cpu_data.x86_model) {
dc81081b
YW		 2117 case 15: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
2118 case 22: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
2119 case 23: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
2120 case 29: /* six-core 45 nm xeon "Dunnington" */
8326f44d 2121 memcpy(hw_cache_event_ids, core2_hw_cache_event_ids,
820a6442 2122 sizeof(hw_cache_event_ids));
8326f44d 2123
1123e3ad 2124 pr_cont("Core2 events, ");
7a693d3f 2125 event_constraints = intel_core_event_constraints;
8326f44d
IM		 2126 break;
2127 default:
2128 case 26:
2129 memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids,
820a6442 2130 sizeof(hw_cache_event_ids));
8326f44d 2131
7a693d3f 2132 event_constraints = intel_nehalem_event_constraints;
1123e3ad 2133 pr_cont("Nehalem/Corei7 events, ");
8326f44d
IM		 2134 break;
2135 case 28:
2136 memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
820a6442 2137 sizeof(hw_cache_event_ids));
8326f44d 2138
1123e3ad 2139 pr_cont("Atom events, ");
8326f44d
IM		 2140 break;
2141 }
72eae04d 2142 return 0;
b56a3802
JSR		 2143}
2144
db48cccc 2145static __init int amd_pmu_init(void)
f87ad35d 2146{
4d2be126
JSR		 2147 /* Performance-monitoring supported from K7 and later: */
2148 if (boot_cpu_data.x86 < 6)
2149 return -ENODEV;
2150
4a06bd85 2151 x86_pmu = amd_pmu;
f86748e9 2152
f4db43a3
JSR		 2153 /* Events are common for all AMDs */
2154 memcpy(hw_cache_event_ids, amd_hw_cache_event_ids,
2155 sizeof(hw_cache_event_ids));
f86748e9 2156
72eae04d 2157 return 0;
f87ad35d
JSR		 2158}
2159
12558038
CG		 2160static void __init pmu_check_apic(void)
2161{
2162 if (cpu_has_apic)
2163 return;
2164
2165 x86_pmu.apic = 0;
2166 pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
2167 pr_info("no hardware sampling interrupt available.\n");
2168}
2169
cdd6c482 2170void __init init_hw_perf_events(void)
b56a3802 2171{
72eae04d
RR		 2172 int err;
2173
cdd6c482 2174 pr_info("Performance Events: ");
1123e3ad 2175
b56a3802
JSR		 2176 switch (boot_cpu_data.x86_vendor) {
2177 case X86_VENDOR_INTEL:
72eae04d 2178 err = intel_pmu_init();
b56a3802 2179 break;
f87ad35d 2180 case X86_VENDOR_AMD:
72eae04d 2181 err = amd_pmu_init();
f87ad35d 2182 break;
4138960a
RR		 2183 default:
2184 return;
b56a3802 2185 }
1123e3ad 2186 if (err != 0) {
cdd6c482 2187 pr_cont("no PMU driver, software events only.\n");
b56a3802 2188 return;
1123e3ad 2189 }
b56a3802 2190
12558038
CG		 2191 pmu_check_apic();
2192
1123e3ad 2193 pr_cont("%s PMU driver.\n", x86_pmu.name);
faa28ae0 2194
cdd6c482
IM		 2195 if (x86_pmu.num_events > X86_PMC_MAX_GENERIC) {
2196 WARN(1, KERN_ERR "hw perf events %d > max(%d), clipping!",
2197 x86_pmu.num_events, X86_PMC_MAX_GENERIC);
2198 x86_pmu.num_events = X86_PMC_MAX_GENERIC;
241771ef 2199 }
cdd6c482
IM		 2200 perf_event_mask = (1 << x86_pmu.num_events) - 1;
2201 perf_max_events = x86_pmu.num_events;
241771ef 2202
cdd6c482
IM		 2203 if (x86_pmu.num_events_fixed > X86_PMC_MAX_FIXED) {
2204 WARN(1, KERN_ERR "hw perf events fixed %d > max(%d), clipping!",
2205 x86_pmu.num_events_fixed, X86_PMC_MAX_FIXED);
2206 x86_pmu.num_events_fixed = X86_PMC_MAX_FIXED;
703e937c 2207 }
862a1a5f 2208
cdd6c482
IM		 2209 perf_event_mask |=
2210 ((1LL << x86_pmu.num_events_fixed)-1) << X86_PMC_IDX_FIXED;
2211 x86_pmu.intel_ctrl = perf_event_mask;
241771ef 2212
cdd6c482
IM		 2213 perf_events_lapic_init();
2214 register_die_notifier(&perf_event_nmi_notifier);
1123e3ad 2215
57c0c15b
IM		 2216 pr_info("... version: %d\n", x86_pmu.version);
2217 pr_info("... bit width: %d\n", x86_pmu.event_bits);
2218 pr_info("... generic registers: %d\n", x86_pmu.num_events);
2219 pr_info("... value mask: %016Lx\n", x86_pmu.event_mask);
2220 pr_info("... max period: %016Lx\n", x86_pmu.max_period);
2221 pr_info("... fixed-purpose events: %d\n", x86_pmu.num_events_fixed);
2222 pr_info("... event mask: %016Lx\n", perf_event_mask);
241771ef 2223}
621a01ea 2224
cdd6c482 2225static inline void x86_pmu_read(struct perf_event *event)
ee06094f 2226{
cdd6c482 2227 x86_perf_event_update(event, &event->hw, event->hw.idx);
ee06094f
IM		 2228}
2229
4aeb0b42
RR		 2230static const struct pmu pmu = {
2231 .enable = x86_pmu_enable,
2232 .disable = x86_pmu_disable,
2233 .read = x86_pmu_read,
a78ac325 2234 .unthrottle = x86_pmu_unthrottle,
621a01ea
IM		 2235};
2236
fe9081cc
PZ		 2237static int
2238validate_event(struct cpu_hw_events *cpuc, struct perf_event *event)
2239{
2240 struct hw_perf_event fake_event = event->hw;
2241
1261a02a 2242 if (event->pmu && event->pmu != &pmu)
fe9081cc
PZ		 2243 return 0;
2244
1261a02a 2245 return x86_schedule_event(cpuc, &fake_event) >= 0;
fe9081cc
PZ		 2246}
2247
2248static int validate_group(struct perf_event *event)
2249{
2250 struct perf_event *sibling, *leader = event->group_leader;
2251 struct cpu_hw_events fake_pmu;
2252
2253 memset(&fake_pmu, 0, sizeof(fake_pmu));
2254
2255 if (!validate_event(&fake_pmu, leader))
2256 return -ENOSPC;
2257
2258 list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
2259 if (!validate_event(&fake_pmu, sibling))
2260 return -ENOSPC;
2261 }
2262
2263 if (!validate_event(&fake_pmu, event))
2264 return -ENOSPC;
2265
2266 return 0;
2267}
2268
cdd6c482 2269const struct pmu *hw_perf_event_init(struct perf_event *event)
621a01ea
IM		 2270{
2271 int err;
2272
cdd6c482 2273 err = __hw_perf_event_init(event);
fe9081cc
PZ		 2274 if (!err) {
2275 if (event->group_leader != event)
2276 err = validate_group(event);
2277 }
a1792cda 2278 if (err) {
cdd6c482
IM		 2279 if (event->destroy)
2280 event->destroy(event);
9ea98e19 2281 return ERR_PTR(err);
a1792cda 2282 }
621a01ea 2283
4aeb0b42 2284 return &pmu;
621a01ea 2285}
d7d59fb3
PZ		 2286
2287/*
2288 * callchain support
2289 */
2290
2291static inline
f9188e02 2292void callchain_store(struct perf_callchain_entry *entry, u64 ip)
d7d59fb3 2293{
f9188e02 2294 if (entry->nr < PERF_MAX_STACK_DEPTH)
d7d59fb3
PZ		 2295 entry->ip[entry->nr++] = ip;
2296}
2297
245b2e70
TH		 2298static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_irq_entry);
2299static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_nmi_entry);
d7d59fb3
PZ		 2300
2301
2302static void
2303backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
2304{
2305 /* Ignore warnings */
2306}
2307
2308static void backtrace_warning(void *data, char *msg)
2309{
2310 /* Ignore warnings */
2311}
2312
2313static int backtrace_stack(void *data, char *name)
2314{
038e836e 2315 return 0;
d7d59fb3
PZ		 2316}
2317
2318static void backtrace_address(void *data, unsigned long addr, int reliable)
2319{
2320 struct perf_callchain_entry *entry = data;
2321
2322 if (reliable)
2323 callchain_store(entry, addr);
2324}
2325
2326static const struct stacktrace_ops backtrace_ops = {
2327 .warning = backtrace_warning,
2328 .warning_symbol = backtrace_warning_symbol,
2329 .stack = backtrace_stack,
2330 .address = backtrace_address,
06d65bda 2331 .walk_stack = print_context_stack_bp,
d7d59fb3
PZ		 2332};
2333
038e836e
IM		 2334#include "../dumpstack.h"
2335
d7d59fb3
PZ		 2336static void
2337perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
2338{
f9188e02 2339 callchain_store(entry, PERF_CONTEXT_KERNEL);
038e836e 2340 callchain_store(entry, regs->ip);
d7d59fb3 2341
48b5ba9c 2342 dump_trace(NULL, regs, NULL, regs->bp, &backtrace_ops, entry);
d7d59fb3
PZ		 2343}
2344
74193ef0
PZ		 2345/*
2346 * best effort, GUP based copy_from_user() that assumes IRQ or NMI context
2347 */
2348static unsigned long
2349copy_from_user_nmi(void *to, const void __user *from, unsigned long n)
d7d59fb3 2350{
74193ef0
PZ		 2351 unsigned long offset, addr = (unsigned long)from;
2352 int type = in_nmi() ? KM_NMI : KM_IRQ0;
2353 unsigned long size, len = 0;
2354 struct page *page;
2355 void *map;
d7d59fb3
PZ		 2356 int ret;
2357
74193ef0
PZ		 2358 do {
2359 ret = __get_user_pages_fast(addr, 1, 0, &page);
2360 if (!ret)
2361 break;
d7d59fb3 2362
74193ef0
PZ		 2363 offset = addr & (PAGE_SIZE - 1);
2364 size = min(PAGE_SIZE - offset, n - len);
d7d59fb3 2365
74193ef0
PZ		 2366 map = kmap_atomic(page, type);
2367 memcpy(to, map+offset, size);
2368 kunmap_atomic(map, type);
2369 put_page(page);
2370
2371 len += size;
2372 to += size;
2373 addr += size;
2374
2375 } while (len < n);
2376
2377 return len;
2378}
2379
2380static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
2381{
2382 unsigned long bytes;
2383
2384 bytes = copy_from_user_nmi(frame, fp, sizeof(*frame));
2385
2386 return bytes == sizeof(*frame);
d7d59fb3
PZ		 2387}
2388
2389static void
2390perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
2391{
2392 struct stack_frame frame;
2393 const void __user *fp;
2394
5a6cec3a
IM		 2395 if (!user_mode(regs))
2396 regs = task_pt_regs(current);
2397
74193ef0 2398 fp = (void __user *)regs->bp;
d7d59fb3 2399
f9188e02 2400 callchain_store(entry, PERF_CONTEXT_USER);
d7d59fb3
PZ		 2401 callchain_store(entry, regs->ip);
2402
f9188e02 2403 while (entry->nr < PERF_MAX_STACK_DEPTH) {
038e836e 2404 frame.next_frame = NULL;
d7d59fb3
PZ		 2405 frame.return_address = 0;
2406
2407 if (!copy_stack_frame(fp, &frame))
2408 break;
2409
5a6cec3a 2410 if ((unsigned long)fp < regs->sp)
d7d59fb3
PZ		 2411 break;
2412
2413 callchain_store(entry, frame.return_address);
038e836e 2414 fp = frame.next_frame;
d7d59fb3
PZ		 2415 }
2416}
2417
2418static void
2419perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
2420{
2421 int is_user;
2422
2423 if (!regs)
2424 return;
2425
2426 is_user = user_mode(regs);
2427
d7d59fb3
PZ		 2428 if (is_user && current->state != TASK_RUNNING)
2429 return;
2430
2431 if (!is_user)
2432 perf_callchain_kernel(regs, entry);
2433
2434 if (current->mm)
2435 perf_callchain_user(regs, entry);
2436}
2437
2438struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
2439{
2440 struct perf_callchain_entry *entry;
2441
2442 if (in_nmi())
245b2e70 2443 entry = &__get_cpu_var(pmc_nmi_entry);
d7d59fb3 2444 else
245b2e70 2445 entry = &__get_cpu_var(pmc_irq_entry);
d7d59fb3
PZ		 2446
2447 entry->nr = 0;
2448
2449 perf_do_callchain(regs, entry);
2450
2451 return entry;
2452}
30dd568c 2453
cdd6c482 2454void hw_perf_event_setup_online(int cpu)
30dd568c
MM		 2455{
2456 init_debug_store_on_cpu(cpu);
2457}
Clone of davem's net-next-2.6 tree