]>
Commit | Line | Data |
---|---|---|
1474855d BN |
1 | /* |
2 | * Cell Broadband Engine OProfile Support | |
3 | * | |
4 | * (C) Copyright IBM Corporation 2006 | |
5 | * | |
6 | * Author: Maynard Johnson <maynardj@us.ibm.com> | |
7 | * | |
8 | * This program is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU General Public License | |
10 | * as published by the Free Software Foundation; either version | |
11 | * 2 of the License, or (at your option) any later version. | |
12 | */ | |
13 | ||
14 | /* The purpose of this file is to handle SPU event task switching | |
15 | * and to record SPU context information into the OProfile | |
16 | * event buffer. | |
17 | * | |
18 | * Additionally, the spu_sync_buffer function is provided as a helper | |
19 | * for recoding actual SPU program counter samples to the event buffer. | |
20 | */ | |
21 | #include <linux/dcookies.h> | |
22 | #include <linux/kref.h> | |
23 | #include <linux/mm.h> | |
24 | #include <linux/module.h> | |
25 | #include <linux/notifier.h> | |
26 | #include <linux/numa.h> | |
27 | #include <linux/oprofile.h> | |
28 | #include <linux/spinlock.h> | |
29 | #include "pr_util.h" | |
30 | ||
31 | #define RELEASE_ALL 9999 | |
32 | ||
33 | static DEFINE_SPINLOCK(buffer_lock); | |
34 | static DEFINE_SPINLOCK(cache_lock); | |
35 | static int num_spu_nodes; | |
36 | int spu_prof_num_nodes; | |
37 | int last_guard_val[MAX_NUMNODES * 8]; | |
38 | ||
39 | /* Container for caching information about an active SPU task. */ | |
40 | struct cached_info { | |
41 | struct vma_to_fileoffset_map *map; | |
42 | struct spu *the_spu; /* needed to access pointer to local_store */ | |
43 | struct kref cache_ref; | |
44 | }; | |
45 | ||
46 | static struct cached_info *spu_info[MAX_NUMNODES * 8]; | |
47 | ||
48 | static void destroy_cached_info(struct kref *kref) | |
49 | { | |
50 | struct cached_info *info; | |
51 | ||
52 | info = container_of(kref, struct cached_info, cache_ref); | |
53 | vma_map_free(info->map); | |
54 | kfree(info); | |
55 | module_put(THIS_MODULE); | |
56 | } | |
57 | ||
58 | /* Return the cached_info for the passed SPU number. | |
59 | * ATTENTION: Callers are responsible for obtaining the | |
60 | * cache_lock if needed prior to invoking this function. | |
61 | */ | |
62 | static struct cached_info *get_cached_info(struct spu *the_spu, int spu_num) | |
63 | { | |
64 | struct kref *ref; | |
65 | struct cached_info *ret_info; | |
66 | ||
67 | if (spu_num >= num_spu_nodes) { | |
68 | printk(KERN_ERR "SPU_PROF: " | |
69 | "%s, line %d: Invalid index %d into spu info cache\n", | |
70 | __FUNCTION__, __LINE__, spu_num); | |
71 | ret_info = NULL; | |
72 | goto out; | |
73 | } | |
74 | if (!spu_info[spu_num] && the_spu) { | |
75 | ref = spu_get_profile_private_kref(the_spu->ctx); | |
76 | if (ref) { | |
77 | spu_info[spu_num] = container_of(ref, struct cached_info, cache_ref); | |
78 | kref_get(&spu_info[spu_num]->cache_ref); | |
79 | } | |
80 | } | |
81 | ||
82 | ret_info = spu_info[spu_num]; | |
83 | out: | |
84 | return ret_info; | |
85 | } | |
86 | ||
87 | ||
88 | /* Looks for cached info for the passed spu. If not found, the | |
89 | * cached info is created for the passed spu. | |
90 | * Returns 0 for success; otherwise, -1 for error. | |
91 | */ | |
92 | static int | |
93 | prepare_cached_spu_info(struct spu *spu, unsigned long objectId) | |
94 | { | |
95 | unsigned long flags; | |
96 | struct vma_to_fileoffset_map *new_map; | |
97 | int retval = 0; | |
98 | struct cached_info *info; | |
99 | ||
100 | /* We won't bother getting cache_lock here since | |
101 | * don't do anything with the cached_info that's returned. | |
102 | */ | |
103 | info = get_cached_info(spu, spu->number); | |
104 | ||
105 | if (info) { | |
106 | pr_debug("Found cached SPU info.\n"); | |
107 | goto out; | |
108 | } | |
109 | ||
110 | /* Create cached_info and set spu_info[spu->number] to point to it. | |
111 | * spu->number is a system-wide value, not a per-node value. | |
112 | */ | |
113 | info = kzalloc(sizeof(struct cached_info), GFP_KERNEL); | |
114 | if (!info) { | |
115 | printk(KERN_ERR "SPU_PROF: " | |
116 | "%s, line %d: create vma_map failed\n", | |
117 | __FUNCTION__, __LINE__); | |
118 | retval = -ENOMEM; | |
119 | goto err_alloc; | |
120 | } | |
121 | new_map = create_vma_map(spu, objectId); | |
122 | if (!new_map) { | |
123 | printk(KERN_ERR "SPU_PROF: " | |
124 | "%s, line %d: create vma_map failed\n", | |
125 | __FUNCTION__, __LINE__); | |
126 | retval = -ENOMEM; | |
127 | goto err_alloc; | |
128 | } | |
129 | ||
130 | pr_debug("Created vma_map\n"); | |
131 | info->map = new_map; | |
132 | info->the_spu = spu; | |
133 | kref_init(&info->cache_ref); | |
134 | spin_lock_irqsave(&cache_lock, flags); | |
135 | spu_info[spu->number] = info; | |
136 | /* Increment count before passing off ref to SPUFS. */ | |
137 | kref_get(&info->cache_ref); | |
138 | ||
139 | /* We increment the module refcount here since SPUFS is | |
140 | * responsible for the final destruction of the cached_info, | |
141 | * and it must be able to access the destroy_cached_info() | |
142 | * function defined in the OProfile module. We decrement | |
143 | * the module refcount in destroy_cached_info. | |
144 | */ | |
145 | try_module_get(THIS_MODULE); | |
146 | spu_set_profile_private_kref(spu->ctx, &info->cache_ref, | |
147 | destroy_cached_info); | |
148 | spin_unlock_irqrestore(&cache_lock, flags); | |
149 | goto out; | |
150 | ||
151 | err_alloc: | |
152 | kfree(info); | |
153 | out: | |
154 | return retval; | |
155 | } | |
156 | ||
157 | /* | |
158 | * NOTE: The caller is responsible for locking the | |
159 | * cache_lock prior to calling this function. | |
160 | */ | |
161 | static int release_cached_info(int spu_index) | |
162 | { | |
163 | int index, end; | |
164 | ||
165 | if (spu_index == RELEASE_ALL) { | |
166 | end = num_spu_nodes; | |
167 | index = 0; | |
168 | } else { | |
169 | if (spu_index >= num_spu_nodes) { | |
170 | printk(KERN_ERR "SPU_PROF: " | |
171 | "%s, line %d: " | |
172 | "Invalid index %d into spu info cache\n", | |
173 | __FUNCTION__, __LINE__, spu_index); | |
174 | goto out; | |
175 | } | |
176 | end = spu_index + 1; | |
177 | index = spu_index; | |
178 | } | |
179 | for (; index < end; index++) { | |
180 | if (spu_info[index]) { | |
181 | kref_put(&spu_info[index]->cache_ref, | |
182 | destroy_cached_info); | |
183 | spu_info[index] = NULL; | |
184 | } | |
185 | } | |
186 | ||
187 | out: | |
188 | return 0; | |
189 | } | |
190 | ||
191 | /* The source code for fast_get_dcookie was "borrowed" | |
192 | * from drivers/oprofile/buffer_sync.c. | |
193 | */ | |
194 | ||
195 | /* Optimisation. We can manage without taking the dcookie sem | |
196 | * because we cannot reach this code without at least one | |
197 | * dcookie user still being registered (namely, the reader | |
198 | * of the event buffer). | |
199 | */ | |
200 | static inline unsigned long fast_get_dcookie(struct dentry *dentry, | |
201 | struct vfsmount *vfsmnt) | |
202 | { | |
203 | unsigned long cookie; | |
204 | ||
205 | if (dentry->d_cookie) | |
206 | return (unsigned long)dentry; | |
207 | get_dcookie(dentry, vfsmnt, &cookie); | |
208 | return cookie; | |
209 | } | |
210 | ||
211 | /* Look up the dcookie for the task's first VM_EXECUTABLE mapping, | |
212 | * which corresponds loosely to "application name". Also, determine | |
213 | * the offset for the SPU ELF object. If computed offset is | |
214 | * non-zero, it implies an embedded SPU object; otherwise, it's a | |
215 | * separate SPU binary, in which case we retrieve it's dcookie. | |
216 | * For the embedded case, we must determine if SPU ELF is embedded | |
217 | * in the executable application or another file (i.e., shared lib). | |
218 | * If embedded in a shared lib, we must get the dcookie and return | |
219 | * that to the caller. | |
220 | */ | |
221 | static unsigned long | |
222 | get_exec_dcookie_and_offset(struct spu *spu, unsigned int *offsetp, | |
223 | unsigned long *spu_bin_dcookie, | |
224 | unsigned long spu_ref) | |
225 | { | |
226 | unsigned long app_cookie = 0; | |
227 | unsigned int my_offset = 0; | |
228 | struct file *app = NULL; | |
229 | struct vm_area_struct *vma; | |
230 | struct mm_struct *mm = spu->mm; | |
231 | ||
232 | if (!mm) | |
233 | goto out; | |
234 | ||
235 | down_read(&mm->mmap_sem); | |
236 | ||
237 | for (vma = mm->mmap; vma; vma = vma->vm_next) { | |
238 | if (!vma->vm_file) | |
239 | continue; | |
240 | if (!(vma->vm_flags & VM_EXECUTABLE)) | |
241 | continue; | |
242 | app_cookie = fast_get_dcookie(vma->vm_file->f_dentry, | |
243 | vma->vm_file->f_vfsmnt); | |
244 | pr_debug("got dcookie for %s\n", | |
245 | vma->vm_file->f_dentry->d_name.name); | |
246 | app = vma->vm_file; | |
247 | break; | |
248 | } | |
249 | ||
250 | for (vma = mm->mmap; vma; vma = vma->vm_next) { | |
251 | if (vma->vm_start > spu_ref || vma->vm_end <= spu_ref) | |
252 | continue; | |
253 | my_offset = spu_ref - vma->vm_start; | |
254 | if (!vma->vm_file) | |
255 | goto fail_no_image_cookie; | |
256 | ||
257 | pr_debug("Found spu ELF at %X(object-id:%lx) for file %s\n", | |
258 | my_offset, spu_ref, | |
259 | vma->vm_file->f_dentry->d_name.name); | |
260 | *offsetp = my_offset; | |
261 | break; | |
262 | } | |
263 | ||
264 | *spu_bin_dcookie = fast_get_dcookie(vma->vm_file->f_dentry, | |
265 | vma->vm_file->f_vfsmnt); | |
266 | pr_debug("got dcookie for %s\n", vma->vm_file->f_dentry->d_name.name); | |
267 | ||
268 | up_read(&mm->mmap_sem); | |
269 | ||
270 | out: | |
271 | return app_cookie; | |
272 | ||
273 | fail_no_image_cookie: | |
274 | up_read(&mm->mmap_sem); | |
275 | ||
276 | printk(KERN_ERR "SPU_PROF: " | |
277 | "%s, line %d: Cannot find dcookie for SPU binary\n", | |
278 | __FUNCTION__, __LINE__); | |
279 | goto out; | |
280 | } | |
281 | ||
282 | ||
283 | ||
284 | /* This function finds or creates cached context information for the | |
285 | * passed SPU and records SPU context information into the OProfile | |
286 | * event buffer. | |
287 | */ | |
288 | static int process_context_switch(struct spu *spu, unsigned long objectId) | |
289 | { | |
290 | unsigned long flags; | |
291 | int retval; | |
292 | unsigned int offset = 0; | |
293 | unsigned long spu_cookie = 0, app_dcookie; | |
294 | ||
295 | retval = prepare_cached_spu_info(spu, objectId); | |
296 | if (retval) | |
297 | goto out; | |
298 | ||
299 | /* Get dcookie first because a mutex_lock is taken in that | |
300 | * code path, so interrupts must not be disabled. | |
301 | */ | |
302 | app_dcookie = get_exec_dcookie_and_offset(spu, &offset, &spu_cookie, objectId); | |
303 | if (!app_dcookie || !spu_cookie) { | |
304 | retval = -ENOENT; | |
305 | goto out; | |
306 | } | |
307 | ||
308 | /* Record context info in event buffer */ | |
309 | spin_lock_irqsave(&buffer_lock, flags); | |
310 | add_event_entry(ESCAPE_CODE); | |
311 | add_event_entry(SPU_CTX_SWITCH_CODE); | |
312 | add_event_entry(spu->number); | |
313 | add_event_entry(spu->pid); | |
314 | add_event_entry(spu->tgid); | |
315 | add_event_entry(app_dcookie); | |
316 | add_event_entry(spu_cookie); | |
317 | add_event_entry(offset); | |
318 | spin_unlock_irqrestore(&buffer_lock, flags); | |
319 | smp_wmb(); /* insure spu event buffer updates are written */ | |
320 | /* don't want entries intermingled... */ | |
321 | out: | |
322 | return retval; | |
323 | } | |
324 | ||
325 | /* | |
326 | * This function is invoked on either a bind_context or unbind_context. | |
327 | * If called for an unbind_context, the val arg is 0; otherwise, | |
328 | * it is the object-id value for the spu context. | |
329 | * The data arg is of type 'struct spu *'. | |
330 | */ | |
331 | static int spu_active_notify(struct notifier_block *self, unsigned long val, | |
332 | void *data) | |
333 | { | |
334 | int retval; | |
335 | unsigned long flags; | |
336 | struct spu *the_spu = data; | |
337 | ||
338 | pr_debug("SPU event notification arrived\n"); | |
339 | if (!val) { | |
340 | spin_lock_irqsave(&cache_lock, flags); | |
341 | retval = release_cached_info(the_spu->number); | |
342 | spin_unlock_irqrestore(&cache_lock, flags); | |
343 | } else { | |
344 | retval = process_context_switch(the_spu, val); | |
345 | } | |
346 | return retval; | |
347 | } | |
348 | ||
349 | static struct notifier_block spu_active = { | |
350 | .notifier_call = spu_active_notify, | |
351 | }; | |
352 | ||
353 | static int number_of_online_nodes(void) | |
354 | { | |
355 | u32 cpu; u32 tmp; | |
356 | int nodes = 0; | |
357 | for_each_online_cpu(cpu) { | |
358 | tmp = cbe_cpu_to_node(cpu) + 1; | |
359 | if (tmp > nodes) | |
360 | nodes++; | |
361 | } | |
362 | return nodes; | |
363 | } | |
364 | ||
365 | /* The main purpose of this function is to synchronize | |
366 | * OProfile with SPUFS by registering to be notified of | |
367 | * SPU task switches. | |
368 | * | |
369 | * NOTE: When profiling SPUs, we must ensure that only | |
370 | * spu_sync_start is invoked and not the generic sync_start | |
371 | * in drivers/oprofile/oprof.c. A return value of | |
372 | * SKIP_GENERIC_SYNC or SYNC_START_ERROR will | |
373 | * accomplish this. | |
374 | */ | |
375 | int spu_sync_start(void) | |
376 | { | |
377 | int k; | |
378 | int ret = SKIP_GENERIC_SYNC; | |
379 | int register_ret; | |
380 | unsigned long flags = 0; | |
381 | ||
382 | spu_prof_num_nodes = number_of_online_nodes(); | |
383 | num_spu_nodes = spu_prof_num_nodes * 8; | |
384 | ||
385 | spin_lock_irqsave(&buffer_lock, flags); | |
386 | add_event_entry(ESCAPE_CODE); | |
387 | add_event_entry(SPU_PROFILING_CODE); | |
388 | add_event_entry(num_spu_nodes); | |
389 | spin_unlock_irqrestore(&buffer_lock, flags); | |
390 | ||
391 | /* Register for SPU events */ | |
392 | register_ret = spu_switch_event_register(&spu_active); | |
393 | if (register_ret) { | |
394 | ret = SYNC_START_ERROR; | |
395 | goto out; | |
396 | } | |
397 | ||
398 | for (k = 0; k < (MAX_NUMNODES * 8); k++) | |
399 | last_guard_val[k] = 0; | |
400 | pr_debug("spu_sync_start -- running.\n"); | |
401 | out: | |
402 | return ret; | |
403 | } | |
404 | ||
405 | /* Record SPU program counter samples to the oprofile event buffer. */ | |
406 | void spu_sync_buffer(int spu_num, unsigned int *samples, | |
407 | int num_samples) | |
408 | { | |
409 | unsigned long long file_offset; | |
410 | unsigned long flags; | |
411 | int i; | |
412 | struct vma_to_fileoffset_map *map; | |
413 | struct spu *the_spu; | |
414 | unsigned long long spu_num_ll = spu_num; | |
415 | unsigned long long spu_num_shifted = spu_num_ll << 32; | |
416 | struct cached_info *c_info; | |
417 | ||
418 | /* We need to obtain the cache_lock here because it's | |
419 | * possible that after getting the cached_info, the SPU job | |
420 | * corresponding to this cached_info may end, thus resulting | |
421 | * in the destruction of the cached_info. | |
422 | */ | |
423 | spin_lock_irqsave(&cache_lock, flags); | |
424 | c_info = get_cached_info(NULL, spu_num); | |
425 | if (!c_info) { | |
426 | /* This legitimately happens when the SPU task ends before all | |
427 | * samples are recorded. | |
428 | * No big deal -- so we just drop a few samples. | |
429 | */ | |
430 | pr_debug("SPU_PROF: No cached SPU contex " | |
431 | "for SPU #%d. Dropping samples.\n", spu_num); | |
432 | goto out; | |
433 | } | |
434 | ||
435 | map = c_info->map; | |
436 | the_spu = c_info->the_spu; | |
437 | spin_lock(&buffer_lock); | |
438 | for (i = 0; i < num_samples; i++) { | |
439 | unsigned int sample = *(samples+i); | |
440 | int grd_val = 0; | |
441 | file_offset = 0; | |
442 | if (sample == 0) | |
443 | continue; | |
444 | file_offset = vma_map_lookup( map, sample, the_spu, &grd_val); | |
445 | ||
446 | /* If overlays are used by this SPU application, the guard | |
447 | * value is non-zero, indicating which overlay section is in | |
448 | * use. We need to discard samples taken during the time | |
449 | * period which an overlay occurs (i.e., guard value changes). | |
450 | */ | |
451 | if (grd_val && grd_val != last_guard_val[spu_num]) { | |
452 | last_guard_val[spu_num] = grd_val; | |
453 | /* Drop the rest of the samples. */ | |
454 | break; | |
455 | } | |
456 | ||
457 | add_event_entry(file_offset | spu_num_shifted); | |
458 | } | |
459 | spin_unlock(&buffer_lock); | |
460 | out: | |
461 | spin_unlock_irqrestore(&cache_lock, flags); | |
462 | } | |
463 | ||
464 | ||
465 | int spu_sync_stop(void) | |
466 | { | |
467 | unsigned long flags = 0; | |
468 | int ret = spu_switch_event_unregister(&spu_active); | |
469 | if (ret) { | |
470 | printk(KERN_ERR "SPU_PROF: " | |
471 | "%s, line %d: spu_switch_event_unregister returned %d\n", | |
472 | __FUNCTION__, __LINE__, ret); | |
473 | goto out; | |
474 | } | |
475 | ||
476 | spin_lock_irqsave(&cache_lock, flags); | |
477 | ret = release_cached_info(RELEASE_ALL); | |
478 | spin_unlock_irqrestore(&cache_lock, flags); | |
479 | out: | |
480 | pr_debug("spu_sync_stop -- done.\n"); | |
481 | return ret; | |
482 | } | |
483 | ||
484 |