]>
Commit | Line | Data |
---|---|---|
fd7b231f AK |
1 | /* |
2 | * Kernel Probes (KProbes) | |
3 | * arch/ia64/kernel/kprobes.c | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify | |
6 | * it under the terms of the GNU General Public License as published by | |
7 | * the Free Software Foundation; either version 2 of the License, or | |
8 | * (at your option) any later version. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, | |
11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
13 | * GNU General Public License for more details. | |
14 | * | |
15 | * You should have received a copy of the GNU General Public License | |
16 | * along with this program; if not, write to the Free Software | |
17 | * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. | |
18 | * | |
19 | * Copyright (C) IBM Corporation, 2002, 2004 | |
20 | * Copyright (C) Intel Corporation, 2005 | |
21 | * | |
22 | * 2005-Apr Rusty Lynch <rusty.lynch@intel.com> and Anil S Keshavamurthy | |
23 | * <anil.s.keshavamurthy@intel.com> adapted from i386 | |
24 | */ | |
25 | ||
fd7b231f AK |
26 | #include <linux/kprobes.h> |
27 | #include <linux/ptrace.h> | |
fd7b231f AK |
28 | #include <linux/string.h> |
29 | #include <linux/slab.h> | |
30 | #include <linux/preempt.h> | |
31 | #include <linux/moduleloader.h> | |
32 | ||
33 | #include <asm/pgtable.h> | |
34 | #include <asm/kdebug.h> | |
c7b645f9 | 35 | #include <asm/sections.h> |
c04c1c81 | 36 | #include <asm/uaccess.h> |
fd7b231f | 37 | |
b2761dc2 AK |
38 | extern void jprobe_inst_return(void); |
39 | ||
8a5c4dc5 AM |
40 | DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; |
41 | DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); | |
fd7b231f AK |
42 | |
43 | enum instruction_type {A, I, M, F, B, L, X, u}; | |
44 | static enum instruction_type bundle_encoding[32][3] = { | |
45 | { M, I, I }, /* 00 */ | |
46 | { M, I, I }, /* 01 */ | |
47 | { M, I, I }, /* 02 */ | |
48 | { M, I, I }, /* 03 */ | |
49 | { M, L, X }, /* 04 */ | |
50 | { M, L, X }, /* 05 */ | |
51 | { u, u, u }, /* 06 */ | |
52 | { u, u, u }, /* 07 */ | |
53 | { M, M, I }, /* 08 */ | |
54 | { M, M, I }, /* 09 */ | |
55 | { M, M, I }, /* 0A */ | |
56 | { M, M, I }, /* 0B */ | |
57 | { M, F, I }, /* 0C */ | |
58 | { M, F, I }, /* 0D */ | |
59 | { M, M, F }, /* 0E */ | |
60 | { M, M, F }, /* 0F */ | |
61 | { M, I, B }, /* 10 */ | |
62 | { M, I, B }, /* 11 */ | |
63 | { M, B, B }, /* 12 */ | |
64 | { M, B, B }, /* 13 */ | |
65 | { u, u, u }, /* 14 */ | |
66 | { u, u, u }, /* 15 */ | |
67 | { B, B, B }, /* 16 */ | |
68 | { B, B, B }, /* 17 */ | |
69 | { M, M, B }, /* 18 */ | |
70 | { M, M, B }, /* 19 */ | |
71 | { u, u, u }, /* 1A */ | |
72 | { u, u, u }, /* 1B */ | |
73 | { M, F, B }, /* 1C */ | |
74 | { M, F, B }, /* 1D */ | |
75 | { u, u, u }, /* 1E */ | |
76 | { u, u, u }, /* 1F */ | |
77 | }; | |
78 | ||
a5403183 AK |
79 | /* |
80 | * In this function we check to see if the instruction | |
81 | * is IP relative instruction and update the kprobe | |
82 | * inst flag accordingly | |
83 | */ | |
1f7ad57b PP |
84 | static void __kprobes update_kprobe_inst_flag(uint template, uint slot, |
85 | uint major_opcode, | |
86 | unsigned long kprobe_inst, | |
87 | struct kprobe *p) | |
fd7b231f | 88 | { |
8bc76772 RL |
89 | p->ainsn.inst_flag = 0; |
90 | p->ainsn.target_br_reg = 0; | |
fd7b231f | 91 | |
deac66ae | 92 | /* Check for Break instruction |
62c27be0 | 93 | * Bits 37:40 Major opcode to be zero |
deac66ae KA |
94 | * Bits 27:32 X6 to be zero |
95 | * Bits 32:35 X3 to be zero | |
96 | */ | |
97 | if ((!major_opcode) && (!((kprobe_inst >> 27) & 0x1FF)) ) { | |
98 | /* is a break instruction */ | |
99 | p->ainsn.inst_flag |= INST_FLAG_BREAK_INST; | |
100 | return; | |
101 | } | |
102 | ||
a5403183 AK |
103 | if (bundle_encoding[template][slot] == B) { |
104 | switch (major_opcode) { | |
105 | case INDIRECT_CALL_OPCODE: | |
106 | p->ainsn.inst_flag |= INST_FLAG_FIX_BRANCH_REG; | |
62c27be0 | 107 | p->ainsn.target_br_reg = ((kprobe_inst >> 6) & 0x7); |
108 | break; | |
a5403183 AK |
109 | case IP_RELATIVE_PREDICT_OPCODE: |
110 | case IP_RELATIVE_BRANCH_OPCODE: | |
111 | p->ainsn.inst_flag |= INST_FLAG_FIX_RELATIVE_IP_ADDR; | |
62c27be0 | 112 | break; |
a5403183 | 113 | case IP_RELATIVE_CALL_OPCODE: |
62c27be0 | 114 | p->ainsn.inst_flag |= INST_FLAG_FIX_RELATIVE_IP_ADDR; |
115 | p->ainsn.inst_flag |= INST_FLAG_FIX_BRANCH_REG; | |
116 | p->ainsn.target_br_reg = ((kprobe_inst >> 6) & 0x7); | |
117 | break; | |
a5403183 | 118 | } |
62c27be0 | 119 | } else if (bundle_encoding[template][slot] == X) { |
a5403183 AK |
120 | switch (major_opcode) { |
121 | case LONG_CALL_OPCODE: | |
122 | p->ainsn.inst_flag |= INST_FLAG_FIX_BRANCH_REG; | |
123 | p->ainsn.target_br_reg = ((kprobe_inst >> 6) & 0x7); | |
124 | break; | |
125 | } | |
126 | } | |
127 | return; | |
128 | } | |
fd7b231f | 129 | |
708de8f1 AK |
130 | /* |
131 | * In this function we check to see if the instruction | |
132 | * on which we are inserting kprobe is supported. | |
133 | * Returns 0 if supported | |
134 | * Returns -EINVAL if unsupported | |
135 | */ | |
1f7ad57b PP |
136 | static int __kprobes unsupported_inst(uint template, uint slot, |
137 | uint major_opcode, | |
138 | unsigned long kprobe_inst, | |
214ddde2 | 139 | unsigned long addr) |
708de8f1 | 140 | { |
708de8f1 AK |
141 | if (bundle_encoding[template][slot] == I) { |
142 | switch (major_opcode) { | |
143 | case 0x0: //I_UNIT_MISC_OPCODE: | |
144 | /* | |
145 | * Check for Integer speculation instruction | |
146 | * - Bit 33-35 to be equal to 0x1 | |
147 | */ | |
148 | if (((kprobe_inst >> 33) & 0x7) == 1) { | |
149 | printk(KERN_WARNING | |
150 | "Kprobes on speculation inst at <0x%lx> not supported\n", | |
151 | addr); | |
152 | return -EINVAL; | |
153 | } | |
154 | ||
155 | /* | |
156 | * IP relative mov instruction | |
157 | * - Bit 27-35 to be equal to 0x30 | |
158 | */ | |
159 | if (((kprobe_inst >> 27) & 0x1FF) == 0x30) { | |
160 | printk(KERN_WARNING | |
161 | "Kprobes on \"mov r1=ip\" at <0x%lx> not supported\n", | |
162 | addr); | |
163 | return -EINVAL; | |
164 | ||
165 | } | |
166 | } | |
167 | } | |
168 | return 0; | |
169 | } | |
170 | ||
171 | ||
1674eafc AK |
172 | /* |
173 | * In this function we check to see if the instruction | |
174 | * (qp) cmpx.crel.ctype p1,p2=r2,r3 | |
175 | * on which we are inserting kprobe is cmp instruction | |
176 | * with ctype as unc. | |
177 | */ | |
1f7ad57b PP |
178 | static uint __kprobes is_cmp_ctype_unc_inst(uint template, uint slot, |
179 | uint major_opcode, | |
180 | unsigned long kprobe_inst) | |
1674eafc AK |
181 | { |
182 | cmp_inst_t cmp_inst; | |
183 | uint ctype_unc = 0; | |
184 | ||
185 | if (!((bundle_encoding[template][slot] == I) || | |
186 | (bundle_encoding[template][slot] == M))) | |
187 | goto out; | |
188 | ||
189 | if (!((major_opcode == 0xC) || (major_opcode == 0xD) || | |
190 | (major_opcode == 0xE))) | |
191 | goto out; | |
192 | ||
193 | cmp_inst.l = kprobe_inst; | |
194 | if ((cmp_inst.f.x2 == 0) || (cmp_inst.f.x2 == 1)) { | |
195 | /* Integere compare - Register Register (A6 type)*/ | |
196 | if ((cmp_inst.f.tb == 0) && (cmp_inst.f.ta == 0) | |
197 | &&(cmp_inst.f.c == 1)) | |
198 | ctype_unc = 1; | |
199 | } else if ((cmp_inst.f.x2 == 2)||(cmp_inst.f.x2 == 3)) { | |
200 | /* Integere compare - Immediate Register (A8 type)*/ | |
201 | if ((cmp_inst.f.ta == 0) &&(cmp_inst.f.c == 1)) | |
202 | ctype_unc = 1; | |
203 | } | |
204 | out: | |
205 | return ctype_unc; | |
206 | } | |
207 | ||
a5403183 AK |
208 | /* |
209 | * In this function we override the bundle with | |
210 | * the break instruction at the given slot. | |
211 | */ | |
1f7ad57b PP |
212 | static void __kprobes prepare_break_inst(uint template, uint slot, |
213 | uint major_opcode, | |
214 | unsigned long kprobe_inst, | |
215 | struct kprobe *p) | |
a5403183 AK |
216 | { |
217 | unsigned long break_inst = BREAK_INST; | |
214ddde2 | 218 | bundle_t *bundle = &p->opcode.bundle; |
a5403183 AK |
219 | |
220 | /* | |
221 | * Copy the original kprobe_inst qualifying predicate(qp) | |
1674eafc AK |
222 | * to the break instruction iff !is_cmp_ctype_unc_inst |
223 | * because for cmp instruction with ctype equal to unc, | |
224 | * which is a special instruction always needs to be | |
225 | * executed regradless of qp | |
a5403183 | 226 | */ |
1674eafc AK |
227 | if (!is_cmp_ctype_unc_inst(template, slot, major_opcode, kprobe_inst)) |
228 | break_inst |= (0x3f & kprobe_inst); | |
a5403183 AK |
229 | |
230 | switch (slot) { | |
231 | case 0: | |
232 | bundle->quad0.slot0 = break_inst; | |
233 | break; | |
234 | case 1: | |
235 | bundle->quad0.slot1_p0 = break_inst; | |
236 | bundle->quad1.slot1_p1 = break_inst >> (64-46); | |
237 | break; | |
238 | case 2: | |
239 | bundle->quad1.slot2 = break_inst; | |
240 | break; | |
8bc76772 | 241 | } |
cd2675bf | 242 | |
a5403183 AK |
243 | /* |
244 | * Update the instruction flag, so that we can | |
245 | * emulate the instruction properly after we | |
246 | * single step on original instruction | |
247 | */ | |
248 | update_kprobe_inst_flag(template, slot, major_opcode, kprobe_inst, p); | |
249 | } | |
250 | ||
3ca269d8 | 251 | static void __kprobes get_kprobe_inst(bundle_t *bundle, uint slot, |
a5403183 AK |
252 | unsigned long *kprobe_inst, uint *major_opcode) |
253 | { | |
254 | unsigned long kprobe_inst_p0, kprobe_inst_p1; | |
255 | unsigned int template; | |
256 | ||
257 | template = bundle->quad0.template; | |
fd7b231f | 258 | |
fd7b231f | 259 | switch (slot) { |
a5403183 | 260 | case 0: |
62c27be0 | 261 | *major_opcode = (bundle->quad0.slot0 >> SLOT0_OPCODE_SHIFT); |
262 | *kprobe_inst = bundle->quad0.slot0; | |
263 | break; | |
a5403183 | 264 | case 1: |
62c27be0 | 265 | *major_opcode = (bundle->quad1.slot1_p1 >> SLOT1_p1_OPCODE_SHIFT); |
266 | kprobe_inst_p0 = bundle->quad0.slot1_p0; | |
267 | kprobe_inst_p1 = bundle->quad1.slot1_p1; | |
268 | *kprobe_inst = kprobe_inst_p0 | (kprobe_inst_p1 << (64-46)); | |
fd7b231f | 269 | break; |
a5403183 | 270 | case 2: |
62c27be0 | 271 | *major_opcode = (bundle->quad1.slot2 >> SLOT2_OPCODE_SHIFT); |
272 | *kprobe_inst = bundle->quad1.slot2; | |
fd7b231f AK |
273 | break; |
274 | } | |
a5403183 | 275 | } |
fd7b231f | 276 | |
c7b645f9 | 277 | /* Returns non-zero if the addr is in the Interrupt Vector Table */ |
3ca269d8 | 278 | static int __kprobes in_ivt_functions(unsigned long addr) |
c7b645f9 KA |
279 | { |
280 | return (addr >= (unsigned long)__start_ivt_text | |
281 | && addr < (unsigned long)__end_ivt_text); | |
282 | } | |
283 | ||
1f7ad57b PP |
284 | static int __kprobes valid_kprobe_addr(int template, int slot, |
285 | unsigned long addr) | |
a5403183 AK |
286 | { |
287 | if ((slot > 2) || ((bundle_encoding[template][1] == L) && slot > 1)) { | |
c7b645f9 KA |
288 | printk(KERN_WARNING "Attempting to insert unaligned kprobe " |
289 | "at 0x%lx\n", addr); | |
a5403183 | 290 | return -EINVAL; |
8bc76772 | 291 | } |
a528e21c | 292 | |
62c27be0 | 293 | if (in_ivt_functions(addr)) { |
294 | printk(KERN_WARNING "Kprobes can't be inserted inside " | |
c7b645f9 | 295 | "IVT functions at 0x%lx\n", addr); |
62c27be0 | 296 | return -EINVAL; |
297 | } | |
c7b645f9 | 298 | |
a528e21c RL |
299 | if (slot == 1 && bundle_encoding[template][1] != L) { |
300 | printk(KERN_WARNING "Inserting kprobes on slot #1 " | |
301 | "is not supported\n"); | |
302 | return -EINVAL; | |
303 | } | |
304 | ||
a5403183 AK |
305 | return 0; |
306 | } | |
307 | ||
3ca269d8 | 308 | static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) |
852caccc | 309 | { |
8a5c4dc5 AM |
310 | kcb->prev_kprobe.kp = kprobe_running(); |
311 | kcb->prev_kprobe.status = kcb->kprobe_status; | |
852caccc AK |
312 | } |
313 | ||
3ca269d8 | 314 | static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) |
852caccc | 315 | { |
8a5c4dc5 AM |
316 | __get_cpu_var(current_kprobe) = kcb->prev_kprobe.kp; |
317 | kcb->kprobe_status = kcb->prev_kprobe.status; | |
852caccc AK |
318 | } |
319 | ||
3ca269d8 | 320 | static void __kprobes set_current_kprobe(struct kprobe *p, |
8a5c4dc5 | 321 | struct kprobe_ctlblk *kcb) |
852caccc | 322 | { |
8a5c4dc5 | 323 | __get_cpu_var(current_kprobe) = p; |
852caccc AK |
324 | } |
325 | ||
9508dbfe RL |
326 | static void kretprobe_trampoline(void) |
327 | { | |
328 | } | |
329 | ||
330 | /* | |
331 | * At this point the target function has been tricked into | |
332 | * returning into our trampoline. Lookup the associated instance | |
333 | * and then: | |
334 | * - call the handler function | |
335 | * - cleanup by marking the instance as unused | |
336 | * - long jump back to the original return address | |
337 | */ | |
1f7ad57b | 338 | int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) |
9508dbfe RL |
339 | { |
340 | struct kretprobe_instance *ri = NULL; | |
99219a3f | 341 | struct hlist_head *head, empty_rp; |
9508dbfe | 342 | struct hlist_node *node, *tmp; |
991a51d8 | 343 | unsigned long flags, orig_ret_address = 0; |
9508dbfe RL |
344 | unsigned long trampoline_address = |
345 | ((struct fnptr *)kretprobe_trampoline)->ip; | |
346 | ||
99219a3f | 347 | INIT_HLIST_HEAD(&empty_rp); |
991a51d8 | 348 | spin_lock_irqsave(&kretprobe_lock, flags); |
9138d581 | 349 | head = kretprobe_inst_table_head(current); |
9508dbfe RL |
350 | |
351 | /* | |
352 | * It is possible to have multiple instances associated with a given | |
353 | * task either because an multiple functions in the call path | |
354 | * have a return probe installed on them, and/or more then one return | |
355 | * return probe was registered for a target function. | |
356 | * | |
357 | * We can handle this because: | |
358 | * - instances are always inserted at the head of the list | |
359 | * - when multiple return probes are registered for the same | |
360 | * function, the first instance's ret_addr will point to the | |
361 | * real return address, and all the rest will point to | |
362 | * kretprobe_trampoline | |
363 | */ | |
364 | hlist_for_each_entry_safe(ri, node, tmp, head, hlist) { | |
9138d581 | 365 | if (ri->task != current) |
9508dbfe | 366 | /* another task is sharing our hash bucket */ |
9138d581 | 367 | continue; |
9508dbfe RL |
368 | |
369 | if (ri->rp && ri->rp->handler) | |
370 | ri->rp->handler(ri, regs); | |
371 | ||
372 | orig_ret_address = (unsigned long)ri->ret_addr; | |
99219a3f | 373 | recycle_rp_inst(ri, &empty_rp); |
9508dbfe RL |
374 | |
375 | if (orig_ret_address != trampoline_address) | |
376 | /* | |
377 | * This is the real return address. Any other | |
378 | * instances associated with this task are for | |
379 | * other calls deeper on the call stack | |
380 | */ | |
381 | break; | |
382 | } | |
383 | ||
384 | BUG_ON(!orig_ret_address || (orig_ret_address == trampoline_address)); | |
385 | regs->cr_iip = orig_ret_address; | |
386 | ||
8a5c4dc5 | 387 | reset_current_kprobe(); |
991a51d8 | 388 | spin_unlock_irqrestore(&kretprobe_lock, flags); |
9508dbfe RL |
389 | preempt_enable_no_resched(); |
390 | ||
99219a3f | 391 | hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) { |
392 | hlist_del(&ri->hlist); | |
393 | kfree(ri); | |
394 | } | |
d217d545 AM |
395 | /* |
396 | * By returning a non-zero value, we are telling | |
397 | * kprobe_handler() that we don't want the post_handler | |
398 | * to run (and have re-enabled preemption) | |
399 | */ | |
9138d581 | 400 | return 1; |
9508dbfe RL |
401 | } |
402 | ||
991a51d8 | 403 | /* Called with kretprobe_lock held */ |
1f7ad57b PP |
404 | void __kprobes arch_prepare_kretprobe(struct kretprobe *rp, |
405 | struct pt_regs *regs) | |
9508dbfe RL |
406 | { |
407 | struct kretprobe_instance *ri; | |
408 | ||
409 | if ((ri = get_free_rp_inst(rp)) != NULL) { | |
410 | ri->rp = rp; | |
411 | ri->task = current; | |
412 | ri->ret_addr = (kprobe_opcode_t *)regs->b0; | |
413 | ||
414 | /* Replace the return addr with trampoline addr */ | |
415 | regs->b0 = ((struct fnptr *)kretprobe_trampoline)->ip; | |
416 | ||
417 | add_rp_inst(ri); | |
418 | } else { | |
419 | rp->nmissed++; | |
420 | } | |
421 | } | |
422 | ||
1f7ad57b | 423 | int __kprobes arch_prepare_kprobe(struct kprobe *p) |
a5403183 AK |
424 | { |
425 | unsigned long addr = (unsigned long) p->addr; | |
426 | unsigned long *kprobe_addr = (unsigned long *)(addr & ~0xFULL); | |
427 | unsigned long kprobe_inst=0; | |
428 | unsigned int slot = addr & 0xf, template, major_opcode = 0; | |
214ddde2 | 429 | bundle_t *bundle; |
a5403183 | 430 | |
214ddde2 | 431 | bundle = &((kprobe_opcode_t *)kprobe_addr)->bundle; |
62c27be0 | 432 | template = bundle->quad0.template; |
a5403183 AK |
433 | |
434 | if(valid_kprobe_addr(template, slot, addr)) | |
435 | return -EINVAL; | |
436 | ||
437 | /* Move to slot 2, if bundle is MLX type and kprobe slot is 1 */ | |
62c27be0 | 438 | if (slot == 1 && bundle_encoding[template][1] == L) |
439 | slot++; | |
a5403183 AK |
440 | |
441 | /* Get kprobe_inst and major_opcode from the bundle */ | |
442 | get_kprobe_inst(bundle, slot, &kprobe_inst, &major_opcode); | |
443 | ||
214ddde2 | 444 | if (unsupported_inst(template, slot, major_opcode, kprobe_inst, addr)) |
708de8f1 AK |
445 | return -EINVAL; |
446 | ||
8bc76772 | 447 | |
214ddde2 | 448 | p->ainsn.insn = get_insn_slot(); |
449 | if (!p->ainsn.insn) | |
450 | return -ENOMEM; | |
451 | memcpy(&p->opcode, kprobe_addr, sizeof(kprobe_opcode_t)); | |
452 | memcpy(p->ainsn.insn, kprobe_addr, sizeof(kprobe_opcode_t)); | |
8bc76772 | 453 | |
214ddde2 | 454 | prepare_break_inst(template, slot, major_opcode, kprobe_inst, p); |
a9ad965e | 455 | |
214ddde2 | 456 | return 0; |
a9ad965e | 457 | } |
458 | ||
1f7ad57b | 459 | void __kprobes arch_arm_kprobe(struct kprobe *p) |
8bc76772 RL |
460 | { |
461 | unsigned long addr = (unsigned long)p->addr; | |
462 | unsigned long arm_addr = addr & ~0xFULL; | |
463 | ||
214ddde2 | 464 | flush_icache_range((unsigned long)p->ainsn.insn, |
465 | (unsigned long)p->ainsn.insn + sizeof(kprobe_opcode_t)); | |
466 | memcpy((char *)arm_addr, &p->opcode, sizeof(kprobe_opcode_t)); | |
467 | flush_icache_range(arm_addr, arm_addr + sizeof(kprobe_opcode_t)); | |
fd7b231f AK |
468 | } |
469 | ||
1f7ad57b | 470 | void __kprobes arch_disarm_kprobe(struct kprobe *p) |
fd7b231f AK |
471 | { |
472 | unsigned long addr = (unsigned long)p->addr; | |
473 | unsigned long arm_addr = addr & ~0xFULL; | |
474 | ||
214ddde2 | 475 | /* p->ainsn.insn contains the original unaltered kprobe_opcode_t */ |
476 | memcpy((char *) arm_addr, (char *) p->ainsn.insn, | |
477 | sizeof(kprobe_opcode_t)); | |
478 | flush_icache_range(arm_addr, arm_addr + sizeof(kprobe_opcode_t)); | |
fd7b231f AK |
479 | } |
480 | ||
214ddde2 | 481 | void __kprobes arch_remove_kprobe(struct kprobe *p) |
482 | { | |
483 | mutex_lock(&kprobe_mutex); | |
484 | free_insn_slot(p->ainsn.insn); | |
485 | mutex_unlock(&kprobe_mutex); | |
486 | } | |
fd7b231f AK |
487 | /* |
488 | * We are resuming execution after a single step fault, so the pt_regs | |
489 | * structure reflects the register state after we executed the instruction | |
490 | * located in the kprobe (p->ainsn.insn.bundle). We still need to adjust | |
cd2675bf AK |
491 | * the ip to point back to the original stack address. To set the IP address |
492 | * to original stack address, handle the case where we need to fixup the | |
493 | * relative IP address and/or fixup branch register. | |
fd7b231f | 494 | */ |
1f7ad57b | 495 | static void __kprobes resume_execution(struct kprobe *p, struct pt_regs *regs) |
fd7b231f | 496 | { |
62c27be0 | 497 | unsigned long bundle_addr = (unsigned long) (&p->ainsn.insn->bundle); |
498 | unsigned long resume_addr = (unsigned long)p->addr & ~0xFULL; | |
499 | unsigned long template; | |
500 | int slot = ((unsigned long)p->addr & 0xf); | |
fd7b231f | 501 | |
214ddde2 | 502 | template = p->ainsn.insn->bundle.quad0.template; |
cd2675bf | 503 | |
62c27be0 | 504 | if (slot == 1 && bundle_encoding[template][1] == L) |
505 | slot = 2; | |
cd2675bf AK |
506 | |
507 | if (p->ainsn.inst_flag) { | |
508 | ||
509 | if (p->ainsn.inst_flag & INST_FLAG_FIX_RELATIVE_IP_ADDR) { | |
510 | /* Fix relative IP address */ | |
62c27be0 | 511 | regs->cr_iip = (regs->cr_iip - bundle_addr) + |
512 | resume_addr; | |
cd2675bf AK |
513 | } |
514 | ||
515 | if (p->ainsn.inst_flag & INST_FLAG_FIX_BRANCH_REG) { | |
516 | /* | |
517 | * Fix target branch register, software convention is | |
518 | * to use either b0 or b6 or b7, so just checking | |
519 | * only those registers | |
520 | */ | |
521 | switch (p->ainsn.target_br_reg) { | |
522 | case 0: | |
523 | if ((regs->b0 == bundle_addr) || | |
524 | (regs->b0 == bundle_addr + 0x10)) { | |
525 | regs->b0 = (regs->b0 - bundle_addr) + | |
526 | resume_addr; | |
527 | } | |
528 | break; | |
529 | case 6: | |
530 | if ((regs->b6 == bundle_addr) || | |
531 | (regs->b6 == bundle_addr + 0x10)) { | |
532 | regs->b6 = (regs->b6 - bundle_addr) + | |
533 | resume_addr; | |
534 | } | |
535 | break; | |
536 | case 7: | |
537 | if ((regs->b7 == bundle_addr) || | |
538 | (regs->b7 == bundle_addr + 0x10)) { | |
539 | regs->b7 = (regs->b7 - bundle_addr) + | |
540 | resume_addr; | |
541 | } | |
542 | break; | |
543 | } /* end switch */ | |
544 | } | |
545 | goto turn_ss_off; | |
546 | } | |
fd7b231f | 547 | |
cd2675bf | 548 | if (slot == 2) { |
62c27be0 | 549 | if (regs->cr_iip == bundle_addr + 0x10) { |
550 | regs->cr_iip = resume_addr + 0x10; | |
551 | } | |
552 | } else { | |
553 | if (regs->cr_iip == bundle_addr) { | |
554 | regs->cr_iip = resume_addr; | |
555 | } | |
a5403183 | 556 | } |
fd7b231f | 557 | |
cd2675bf | 558 | turn_ss_off: |
62c27be0 | 559 | /* Turn off Single Step bit */ |
560 | ia64_psr(regs)->ss = 0; | |
fd7b231f AK |
561 | } |
562 | ||
1f7ad57b | 563 | static void __kprobes prepare_ss(struct kprobe *p, struct pt_regs *regs) |
fd7b231f | 564 | { |
214ddde2 | 565 | unsigned long bundle_addr = (unsigned long) &p->ainsn.insn->bundle; |
fd7b231f AK |
566 | unsigned long slot = (unsigned long)p->addr & 0xf; |
567 | ||
deac66ae KA |
568 | /* single step inline if break instruction */ |
569 | if (p->ainsn.inst_flag == INST_FLAG_BREAK_INST) | |
570 | regs->cr_iip = (unsigned long)p->addr & ~0xFULL; | |
571 | else | |
572 | regs->cr_iip = bundle_addr & ~0xFULL; | |
fd7b231f AK |
573 | |
574 | if (slot > 2) | |
575 | slot = 0; | |
576 | ||
577 | ia64_psr(regs)->ri = slot; | |
578 | ||
579 | /* turn on single stepping */ | |
580 | ia64_psr(regs)->ss = 1; | |
581 | } | |
582 | ||
661e5a3d KA |
583 | static int __kprobes is_ia64_break_inst(struct pt_regs *regs) |
584 | { | |
585 | unsigned int slot = ia64_psr(regs)->ri; | |
586 | unsigned int template, major_opcode; | |
587 | unsigned long kprobe_inst; | |
588 | unsigned long *kprobe_addr = (unsigned long *)regs->cr_iip; | |
589 | bundle_t bundle; | |
590 | ||
591 | memcpy(&bundle, kprobe_addr, sizeof(bundle_t)); | |
592 | template = bundle.quad0.template; | |
593 | ||
594 | /* Move to slot 2, if bundle is MLX type and kprobe slot is 1 */ | |
595 | if (slot == 1 && bundle_encoding[template][1] == L) | |
62c27be0 | 596 | slot++; |
661e5a3d KA |
597 | |
598 | /* Get Kprobe probe instruction at given slot*/ | |
599 | get_kprobe_inst(&bundle, slot, &kprobe_inst, &major_opcode); | |
600 | ||
601 | /* For break instruction, | |
602 | * Bits 37:40 Major opcode to be zero | |
603 | * Bits 27:32 X6 to be zero | |
604 | * Bits 32:35 X3 to be zero | |
605 | */ | |
606 | if (major_opcode || ((kprobe_inst >> 27) & 0x1FF) ) { | |
607 | /* Not a break instruction */ | |
608 | return 0; | |
609 | } | |
610 | ||
611 | /* Is a break instruction */ | |
612 | return 1; | |
613 | } | |
614 | ||
1f7ad57b | 615 | static int __kprobes pre_kprobes_handler(struct die_args *args) |
fd7b231f AK |
616 | { |
617 | struct kprobe *p; | |
618 | int ret = 0; | |
89cb14c0 | 619 | struct pt_regs *regs = args->regs; |
fd7b231f | 620 | kprobe_opcode_t *addr = (kprobe_opcode_t *)instruction_pointer(regs); |
d217d545 AM |
621 | struct kprobe_ctlblk *kcb; |
622 | ||
623 | /* | |
624 | * We don't want to be preempted for the entire | |
625 | * duration of kprobe processing | |
626 | */ | |
627 | preempt_disable(); | |
628 | kcb = get_kprobe_ctlblk(); | |
fd7b231f | 629 | |
fd7b231f AK |
630 | /* Handle recursion cases */ |
631 | if (kprobe_running()) { | |
632 | p = get_kprobe(addr); | |
633 | if (p) { | |
8a5c4dc5 | 634 | if ((kcb->kprobe_status == KPROBE_HIT_SS) && |
deac66ae | 635 | (p->ainsn.inst_flag == INST_FLAG_BREAK_INST)) { |
62c27be0 | 636 | ia64_psr(regs)->ss = 0; |
fd7b231f AK |
637 | goto no_kprobe; |
638 | } | |
852caccc AK |
639 | /* We have reentered the pre_kprobe_handler(), since |
640 | * another probe was hit while within the handler. | |
641 | * We here save the original kprobes variables and | |
642 | * just single step on the instruction of the new probe | |
643 | * without calling any user handlers. | |
644 | */ | |
8a5c4dc5 AM |
645 | save_previous_kprobe(kcb); |
646 | set_current_kprobe(p, kcb); | |
bf8d5c52 | 647 | kprobes_inc_nmissed_count(p); |
852caccc | 648 | prepare_ss(p, regs); |
8a5c4dc5 | 649 | kcb->kprobe_status = KPROBE_REENTER; |
852caccc | 650 | return 1; |
89cb14c0 | 651 | } else if (args->err == __IA64_BREAK_JPROBE) { |
fd7b231f AK |
652 | /* |
653 | * jprobe instrumented function just completed | |
654 | */ | |
8a5c4dc5 | 655 | p = __get_cpu_var(current_kprobe); |
fd7b231f AK |
656 | if (p->break_handler && p->break_handler(p, regs)) { |
657 | goto ss_probe; | |
658 | } | |
eb3a7292 KA |
659 | } else if (!is_ia64_break_inst(regs)) { |
660 | /* The breakpoint instruction was removed by | |
661 | * another cpu right after we hit, no further | |
662 | * handling of this interrupt is appropriate | |
663 | */ | |
664 | ret = 1; | |
665 | goto no_kprobe; | |
89cb14c0 KA |
666 | } else { |
667 | /* Not our break */ | |
668 | goto no_kprobe; | |
fd7b231f AK |
669 | } |
670 | } | |
671 | ||
fd7b231f AK |
672 | p = get_kprobe(addr); |
673 | if (!p) { | |
661e5a3d KA |
674 | if (!is_ia64_break_inst(regs)) { |
675 | /* | |
676 | * The breakpoint instruction was removed right | |
677 | * after we hit it. Another cpu has removed | |
678 | * either a probepoint or a debugger breakpoint | |
679 | * at this address. In either case, no further | |
680 | * handling of this interrupt is appropriate. | |
681 | */ | |
682 | ret = 1; | |
683 | ||
684 | } | |
685 | ||
686 | /* Not one of our break, let kernel handle it */ | |
fd7b231f AK |
687 | goto no_kprobe; |
688 | } | |
689 | ||
8a5c4dc5 AM |
690 | set_current_kprobe(p, kcb); |
691 | kcb->kprobe_status = KPROBE_HIT_ACTIVE; | |
fd7b231f AK |
692 | |
693 | if (p->pre_handler && p->pre_handler(p, regs)) | |
694 | /* | |
695 | * Our pre-handler is specifically requesting that we just | |
9508dbfe RL |
696 | * do a return. This is used for both the jprobe pre-handler |
697 | * and the kretprobe trampoline | |
fd7b231f AK |
698 | */ |
699 | return 1; | |
700 | ||
701 | ss_probe: | |
702 | prepare_ss(p, regs); | |
8a5c4dc5 | 703 | kcb->kprobe_status = KPROBE_HIT_SS; |
fd7b231f AK |
704 | return 1; |
705 | ||
706 | no_kprobe: | |
d217d545 | 707 | preempt_enable_no_resched(); |
fd7b231f AK |
708 | return ret; |
709 | } | |
710 | ||
1f7ad57b | 711 | static int __kprobes post_kprobes_handler(struct pt_regs *regs) |
fd7b231f | 712 | { |
8a5c4dc5 AM |
713 | struct kprobe *cur = kprobe_running(); |
714 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); | |
715 | ||
716 | if (!cur) | |
fd7b231f AK |
717 | return 0; |
718 | ||
8a5c4dc5 AM |
719 | if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) { |
720 | kcb->kprobe_status = KPROBE_HIT_SSDONE; | |
721 | cur->post_handler(cur, regs, 0); | |
852caccc | 722 | } |
fd7b231f | 723 | |
8a5c4dc5 | 724 | resume_execution(cur, regs); |
fd7b231f | 725 | |
852caccc | 726 | /*Restore back the original saved kprobes variables and continue. */ |
8a5c4dc5 AM |
727 | if (kcb->kprobe_status == KPROBE_REENTER) { |
728 | restore_previous_kprobe(kcb); | |
852caccc AK |
729 | goto out; |
730 | } | |
8a5c4dc5 | 731 | reset_current_kprobe(); |
852caccc AK |
732 | |
733 | out: | |
fd7b231f AK |
734 | preempt_enable_no_resched(); |
735 | return 1; | |
736 | } | |
737 | ||
1f7ad57b | 738 | static int __kprobes kprobes_fault_handler(struct pt_regs *regs, int trapnr) |
fd7b231f | 739 | { |
8a5c4dc5 AM |
740 | struct kprobe *cur = kprobe_running(); |
741 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); | |
742 | ||
fd7b231f | 743 | |
c04c1c81 PP |
744 | switch(kcb->kprobe_status) { |
745 | case KPROBE_HIT_SS: | |
746 | case KPROBE_REENTER: | |
747 | /* | |
748 | * We are here because the instruction being single | |
749 | * stepped caused a page fault. We reset the current | |
750 | * kprobe and the instruction pointer points back to | |
751 | * the probe address and allow the page fault handler | |
752 | * to continue as a normal page fault. | |
753 | */ | |
754 | regs->cr_iip = ((unsigned long)cur->addr) & ~0xFULL; | |
755 | ia64_psr(regs)->ri = ((unsigned long)cur->addr) & 0xf; | |
756 | if (kcb->kprobe_status == KPROBE_REENTER) | |
757 | restore_previous_kprobe(kcb); | |
758 | else | |
759 | reset_current_kprobe(); | |
fd7b231f | 760 | preempt_enable_no_resched(); |
c04c1c81 PP |
761 | break; |
762 | case KPROBE_HIT_ACTIVE: | |
763 | case KPROBE_HIT_SSDONE: | |
764 | /* | |
765 | * We increment the nmissed count for accounting, | |
766 | * we can also use npre/npostfault count for accouting | |
767 | * these specific fault cases. | |
768 | */ | |
769 | kprobes_inc_nmissed_count(cur); | |
770 | ||
771 | /* | |
772 | * We come here because instructions in the pre/post | |
773 | * handler caused the page_fault, this could happen | |
774 | * if handler tries to access user space by | |
775 | * copy_from_user(), get_user() etc. Let the | |
776 | * user-specified handler try to fix it first. | |
777 | */ | |
778 | if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr)) | |
779 | return 1; | |
fd32cb3a KA |
780 | /* |
781 | * In case the user-specified fault handler returned | |
782 | * zero, try to fix up. | |
783 | */ | |
784 | if (ia64_done_with_exception(regs)) | |
785 | return 1; | |
c04c1c81 PP |
786 | |
787 | /* | |
788 | * Let ia64_do_page_fault() fix it. | |
789 | */ | |
790 | break; | |
791 | default: | |
792 | break; | |
fd7b231f AK |
793 | } |
794 | ||
795 | return 0; | |
796 | } | |
797 | ||
1f7ad57b PP |
798 | int __kprobes kprobe_exceptions_notify(struct notifier_block *self, |
799 | unsigned long val, void *data) | |
fd7b231f AK |
800 | { |
801 | struct die_args *args = (struct die_args *)data; | |
66ff2d06 AM |
802 | int ret = NOTIFY_DONE; |
803 | ||
2326c770 | 804 | if (args->regs && user_mode(args->regs)) |
805 | return ret; | |
806 | ||
fd7b231f AK |
807 | switch(val) { |
808 | case DIE_BREAK: | |
9138d581 | 809 | /* err is break number from ia64_bad_break() */ |
5a94bcfd | 810 | if (args->err == 0x80200 || args->err == 0x80300 || args->err == 0) |
9138d581 KO |
811 | if (pre_kprobes_handler(args)) |
812 | ret = NOTIFY_STOP; | |
fd7b231f | 813 | break; |
9138d581 KO |
814 | case DIE_FAULT: |
815 | /* err is vector number from ia64_fault() */ | |
816 | if (args->err == 36) | |
817 | if (post_kprobes_handler(args->regs)) | |
818 | ret = NOTIFY_STOP; | |
fd7b231f AK |
819 | break; |
820 | case DIE_PAGE_FAULT: | |
d217d545 AM |
821 | /* kprobe_running() needs smp_processor_id() */ |
822 | preempt_disable(); | |
823 | if (kprobe_running() && | |
824 | kprobes_fault_handler(args->regs, args->trapnr)) | |
66ff2d06 | 825 | ret = NOTIFY_STOP; |
d217d545 | 826 | preempt_enable(); |
fd7b231f AK |
827 | default: |
828 | break; | |
829 | } | |
66ff2d06 | 830 | return ret; |
fd7b231f AK |
831 | } |
832 | ||
d3ef1f5a ZY |
833 | struct param_bsp_cfm { |
834 | unsigned long ip; | |
835 | unsigned long *bsp; | |
836 | unsigned long cfm; | |
837 | }; | |
838 | ||
839 | static void ia64_get_bsp_cfm(struct unw_frame_info *info, void *arg) | |
840 | { | |
841 | unsigned long ip; | |
842 | struct param_bsp_cfm *lp = arg; | |
843 | ||
844 | do { | |
845 | unw_get_ip(info, &ip); | |
846 | if (ip == 0) | |
847 | break; | |
848 | if (ip == lp->ip) { | |
849 | unw_get_bsp(info, (unsigned long*)&lp->bsp); | |
850 | unw_get_cfm(info, (unsigned long*)&lp->cfm); | |
851 | return; | |
852 | } | |
853 | } while (unw_unwind(info) >= 0); | |
854 | lp->bsp = 0; | |
855 | lp->cfm = 0; | |
856 | return; | |
857 | } | |
858 | ||
1f7ad57b | 859 | int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) |
fd7b231f | 860 | { |
b2761dc2 AK |
861 | struct jprobe *jp = container_of(p, struct jprobe, kp); |
862 | unsigned long addr = ((struct fnptr *)(jp->entry))->ip; | |
8a5c4dc5 | 863 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
d3ef1f5a ZY |
864 | struct param_bsp_cfm pa; |
865 | int bytes; | |
866 | ||
867 | /* | |
868 | * Callee owns the argument space and could overwrite it, eg | |
869 | * tail call optimization. So to be absolutely safe | |
870 | * we save the argument space before transfering the control | |
871 | * to instrumented jprobe function which runs in | |
872 | * the process context | |
873 | */ | |
874 | pa.ip = regs->cr_iip; | |
875 | unw_init_running(ia64_get_bsp_cfm, &pa); | |
876 | bytes = (char *)ia64_rse_skip_regs(pa.bsp, pa.cfm & 0x3f) | |
877 | - (char *)pa.bsp; | |
878 | memcpy( kcb->jprobes_saved_stacked_regs, | |
879 | pa.bsp, | |
880 | bytes ); | |
881 | kcb->bsp = pa.bsp; | |
882 | kcb->cfm = pa.cfm; | |
fd7b231f | 883 | |
b2761dc2 | 884 | /* save architectural state */ |
8a5c4dc5 | 885 | kcb->jprobe_saved_regs = *regs; |
b2761dc2 AK |
886 | |
887 | /* after rfi, execute the jprobe instrumented function */ | |
888 | regs->cr_iip = addr & ~0xFULL; | |
889 | ia64_psr(regs)->ri = addr & 0xf; | |
890 | regs->r1 = ((struct fnptr *)(jp->entry))->gp; | |
891 | ||
892 | /* | |
893 | * fix the return address to our jprobe_inst_return() function | |
894 | * in the jprobes.S file | |
895 | */ | |
62c27be0 | 896 | regs->b0 = ((struct fnptr *)(jprobe_inst_return))->ip; |
b2761dc2 AK |
897 | |
898 | return 1; | |
fd7b231f AK |
899 | } |
900 | ||
1f7ad57b | 901 | int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) |
fd7b231f | 902 | { |
8a5c4dc5 | 903 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
d3ef1f5a | 904 | int bytes; |
8a5c4dc5 | 905 | |
d3ef1f5a | 906 | /* restoring architectural state */ |
8a5c4dc5 | 907 | *regs = kcb->jprobe_saved_regs; |
d3ef1f5a ZY |
908 | |
909 | /* restoring the original argument space */ | |
910 | flush_register_stack(); | |
911 | bytes = (char *)ia64_rse_skip_regs(kcb->bsp, kcb->cfm & 0x3f) | |
912 | - (char *)kcb->bsp; | |
913 | memcpy( kcb->bsp, | |
914 | kcb->jprobes_saved_stacked_regs, | |
915 | bytes ); | |
916 | invalidate_stacked_regs(); | |
917 | ||
d217d545 | 918 | preempt_enable_no_resched(); |
b2761dc2 | 919 | return 1; |
fd7b231f | 920 | } |
9508dbfe RL |
921 | |
922 | static struct kprobe trampoline_p = { | |
923 | .pre_handler = trampoline_probe_handler | |
924 | }; | |
925 | ||
6772926b | 926 | int __init arch_init_kprobes(void) |
9508dbfe RL |
927 | { |
928 | trampoline_p.addr = | |
929 | (kprobe_opcode_t *)((struct fnptr *)kretprobe_trampoline)->ip; | |
930 | return register_kprobe(&trampoline_p); | |
931 | } |