1 #ifdef CONFIG_CPU_SUP_INTEL
7 LBR_FORMAT_EIP_FLAGS = 0x03,
11 * We only support LBR implementations that have FREEZE_LBRS_ON_PMI
12 * otherwise it becomes near impossible to get a reliable stack.
15 #define X86_DEBUGCTL_LBR (1 << 0)
16 #define X86_DEBUGCTL_FREEZE_LBRS_ON_PMI (1 << 11)
18 static void __intel_pmu_lbr_enable(void)
22 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
23 debugctl |= (X86_DEBUGCTL_LBR | X86_DEBUGCTL_FREEZE_LBRS_ON_PMI);
24 wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
27 static void __intel_pmu_lbr_disable(void)
31 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
32 debugctl &= ~(X86_DEBUGCTL_LBR | X86_DEBUGCTL_FREEZE_LBRS_ON_PMI);
33 wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
36 static void intel_pmu_lbr_reset_32(void)
40 for (i = 0; i < x86_pmu.lbr_nr; i++)
41 wrmsrl(x86_pmu.lbr_from + i, 0);
44 static void intel_pmu_lbr_reset_64(void)
48 for (i = 0; i < x86_pmu.lbr_nr; i++) {
49 wrmsrl(x86_pmu.lbr_from + i, 0);
50 wrmsrl(x86_pmu.lbr_to + i, 0);
54 static void intel_pmu_lbr_reset(void)
59 if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
60 intel_pmu_lbr_reset_32();
62 intel_pmu_lbr_reset_64();
65 static void intel_pmu_lbr_enable(struct perf_event *event)
67 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
72 WARN_ON_ONCE(cpuc->enabled);
75 * Reset the LBR stack if this is the first LBR user or
76 * we changed task context so as to avoid data leaks.
79 if (!cpuc->lbr_users ||
80 (event->ctx->task && cpuc->lbr_context != event->ctx)) {
81 intel_pmu_lbr_reset();
82 cpuc->lbr_context = event->ctx;
88 static void intel_pmu_lbr_disable(struct perf_event *event)
90 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
96 BUG_ON(cpuc->lbr_users < 0);
98 if (cpuc->enabled && !cpuc->lbr_users)
99 __intel_pmu_lbr_disable();
102 static void intel_pmu_lbr_enable_all(void)
104 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
107 __intel_pmu_lbr_enable();
110 static void intel_pmu_lbr_disable_all(void)
112 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
115 __intel_pmu_lbr_disable();
118 static inline u64 intel_pmu_lbr_tos(void)
122 rdmsrl(x86_pmu.lbr_tos, tos);
127 static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
129 unsigned long mask = x86_pmu.lbr_nr - 1;
130 u64 tos = intel_pmu_lbr_tos();
133 for (i = 0; i < x86_pmu.lbr_nr; i++, tos--) {
134 unsigned long lbr_idx = (tos - i) & mask;
143 rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr);
145 cpuc->lbr_entries[i].from = msr_lastbranch.from;
146 cpuc->lbr_entries[i].to = msr_lastbranch.to;
147 cpuc->lbr_entries[i].flags = 0;
149 cpuc->lbr_stack.nr = i;
152 #define LBR_FROM_FLAG_MISPRED (1ULL << 63)
155 * Due to lack of segmentation in Linux the effective address (offset)
156 * is the same as the linear address, allowing us to merge the LIP and EIP
159 static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
161 unsigned long mask = x86_pmu.lbr_nr - 1;
162 int lbr_format = x86_pmu.intel_cap.lbr_format;
163 u64 tos = intel_pmu_lbr_tos();
166 for (i = 0; i < x86_pmu.lbr_nr; i++, tos--) {
167 unsigned long lbr_idx = (tos - i) & mask;
168 u64 from, to, flags = 0;
170 rdmsrl(x86_pmu.lbr_from + lbr_idx, from);
171 rdmsrl(x86_pmu.lbr_to + lbr_idx, to);
173 if (lbr_format == LBR_FORMAT_EIP_FLAGS) {
174 flags = !!(from & LBR_FROM_FLAG_MISPRED);
175 from = (u64)((((s64)from) << 1) >> 1);
178 cpuc->lbr_entries[i].from = from;
179 cpuc->lbr_entries[i].to = to;
180 cpuc->lbr_entries[i].flags = flags;
182 cpuc->lbr_stack.nr = i;
185 static void intel_pmu_lbr_read(void)
187 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
189 if (!cpuc->lbr_users)
192 if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
193 intel_pmu_lbr_read_32(cpuc);
195 intel_pmu_lbr_read_64(cpuc);
198 static void intel_pmu_lbr_init_core(void)
201 x86_pmu.lbr_tos = 0x01c9;
202 x86_pmu.lbr_from = 0x40;
203 x86_pmu.lbr_to = 0x60;
206 static void intel_pmu_lbr_init_nhm(void)
209 x86_pmu.lbr_tos = 0x01c9;
210 x86_pmu.lbr_from = 0x680;
211 x86_pmu.lbr_to = 0x6c0;
214 static void intel_pmu_lbr_init_atom(void)
217 x86_pmu.lbr_tos = 0x01c9;
218 x86_pmu.lbr_from = 0x40;
219 x86_pmu.lbr_to = 0x60;
222 #endif /* CONFIG_CPU_SUP_INTEL */