kernel/trace/trace_event_perf.c

   1 /*
   2  * trace event based perf event profiling/tracing
   3  *
   4  * Copyright (C) 2009 Red Hat Inc, Peter Zijlstra <pzijlstr@redhat.com>
   5  * Copyright (C) 2009-2010 Frederic Weisbecker <fweisbec@gmail.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kprobes.h>
  10 #include "trace.h"
  11
  12 DEFINE_PER_CPU(struct pt_regs, perf_trace_regs);
  13 EXPORT_PER_CPU_SYMBOL_GPL(perf_trace_regs);
  14
  15 EXPORT_SYMBOL_GPL(perf_arch_fetch_caller_regs);
  16
  17 static char *perf_trace_buf;
  18 static char *perf_trace_buf_nmi;
  19
  20 /*
  21  * Force it to be aligned to unsigned long to avoid misaligned accesses
  22  * suprises
  23  */
  24 typedef typeof(unsigned long [PERF_MAX_TRACE_SIZE / sizeof(unsigned long)])
  25         perf_trace_t;
  26
  27 /* Count the events in use (per event id, not per instance) */
  28 static int      total_ref_count;
  29
  30 static int perf_trace_event_enable(struct ftrace_event_call *event, void *data)
  31 {
  32         char *buf;
  33         int ret = -ENOMEM;
  34
  35         if (event->perf_refcount++ > 0) {
  36                 event->perf_data = NULL;
  37                 return 0;
  38         }
  39
  40         if (!total_ref_count) {
  41                 buf = (char *)alloc_percpu(perf_trace_t);
  42                 if (!buf)
  43                         goto fail_buf;
  44
  45                 rcu_assign_pointer(perf_trace_buf, buf);
  46
  47                 buf = (char *)alloc_percpu(perf_trace_t);
  48                 if (!buf)
  49                         goto fail_buf_nmi;
  50
  51                 rcu_assign_pointer(perf_trace_buf_nmi, buf);
  52         }
  53
  54         ret = event->perf_event_enable(event);
  55         if (!ret) {
  56                 event->perf_data = data;
  57                 total_ref_count++;
  58                 return 0;
  59         }
  60
  61 fail_buf_nmi:
  62         if (!total_ref_count) {
  63                 free_percpu(perf_trace_buf_nmi);
  64                 free_percpu(perf_trace_buf);
  65                 perf_trace_buf_nmi = NULL;
  66                 perf_trace_buf = NULL;
  67         }
  68 fail_buf:
  69         event->perf_refcount--;
  70
  71         return ret;
  72 }
  73
  74 int perf_trace_enable(int event_id, void *data)
  75 {
  76         struct ftrace_event_call *event;
  77         int ret = -EINVAL;
  78
  79         mutex_lock(&event_mutex);
  80         list_for_each_entry(event, &ftrace_events, list) {
  81                 if (event->id == event_id && event->perf_event_enable &&
  82                     try_module_get(event->mod)) {
  83                         ret = perf_trace_event_enable(event, data);
  84                         break;
  85                 }
  86         }
  87         mutex_unlock(&event_mutex);
  88
  89         return ret;
  90 }
  91
  92 static void perf_trace_event_disable(struct ftrace_event_call *event)
  93 {
  94         char *buf, *nmi_buf;
  95
  96         if (--event->perf_refcount > 0)
  97                 return;
  98
  99         event->perf_event_disable(event);
 100
 101         if (!--total_ref_count) {
 102                 buf = perf_trace_buf;
 103                 rcu_assign_pointer(perf_trace_buf, NULL);
 104
 105                 nmi_buf = perf_trace_buf_nmi;
 106                 rcu_assign_pointer(perf_trace_buf_nmi, NULL);
 107
 108                 /*
 109                  * Ensure every events in profiling have finished before
 110                  * releasing the buffers
 111                  */
 112                 synchronize_sched();
 113
 114                 free_percpu(buf);
 115                 free_percpu(nmi_buf);
 116         }
 117 }
 118
 119 void perf_trace_disable(int event_id)
 120 {
 121         struct ftrace_event_call *event;
 122
 123         mutex_lock(&event_mutex);
 124         list_for_each_entry(event, &ftrace_events, list) {
 125                 if (event->id == event_id) {
 126                         perf_trace_event_disable(event);
 127                         module_put(event->mod);
 128                         break;
 129                 }
 130         }
 131         mutex_unlock(&event_mutex);
 132 }
 133
 134 __kprobes void *perf_trace_buf_prepare(int size, unsigned short type,
 135                                        int *rctxp, unsigned long *irq_flags)
 136 {
 137         struct trace_entry *entry;
 138         char *trace_buf, *raw_data;
 139         int pc, cpu;
 140
 141         BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(unsigned long));
 142
 143         pc = preempt_count();
 144
 145         /* Protect the per cpu buffer, begin the rcu read side */
 146         local_irq_save(*irq_flags);
 147
 148         *rctxp = perf_swevent_get_recursion_context();
 149         if (*rctxp < 0)
 150                 goto err_recursion;
 151
 152         cpu = smp_processor_id();
 153
 154         if (in_nmi())
 155                 trace_buf = rcu_dereference_sched(perf_trace_buf_nmi);
 156         else
 157                 trace_buf = rcu_dereference_sched(perf_trace_buf);
 158
 159         if (!trace_buf)
 160                 goto err;
 161
 162         raw_data = per_cpu_ptr(trace_buf, cpu);
 163
 164         /* zero the dead bytes from align to not leak stack to user */
 165         memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));
 166
 167         entry = (struct trace_entry *)raw_data;
 168         tracing_generic_entry_update(entry, *irq_flags, pc);
 169         entry->type = type;
 170
 171         return raw_data;
 172 err:
 173         perf_swevent_put_recursion_context(*rctxp);
 174 err_recursion:
 175         local_irq_restore(*irq_flags);
 176         return NULL;
 177 }
 178 EXPORT_SYMBOL_GPL(perf_trace_buf_prepare);