perf, trace: Optimize tracepoints by using per-tracepoint-per-cpu hlist to track...

[net-next-2.6.git] / include / trace / ftrace.h

diff --git a/include/trace/ftrace.h b/include/trace/ftrace.h
index 75dd7787fb373acaec6151f95c398e0ce069c6ad..4eb2148f13216b0d06c15b28891a76460812fbef 100644 (file)
--- a/include/trace/ftrace.h
+++ b/include/trace/ftrace.h
@@ -763,13 +763,12 @@ __attribute__((section("_ftrace_events"))) event_##call = {               \
 #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
 static notrace void                                                    \
 perf_trace_templ_##call(struct ftrace_event_call *event_call,          \
-                           proto)                                      \
+                       struct pt_regs *__regs, proto)                  \
 {                                                                      \
        struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\
        struct ftrace_raw_##call *entry;                                \
        u64 __addr = 0, __count = 1;                                    \
-       unsigned long irq_flags;                                        \
-       struct pt_regs *__regs;                                         \
+       struct hlist_head *head;                                        \
        int __entry_size;                                               \
        int __data_size;                                                \
        int rctx;                                                       \
@@ -782,28 +781,30 @@ perf_trace_templ_##call(struct ftrace_event_call *event_call,             \
        if (WARN_ONCE(__entry_size > PERF_MAX_TRACE_SIZE,               \
                      "profile buffer not large enough"))               \
                return;                                                 \
+                                                                       \
        entry = (struct ftrace_raw_##call *)perf_trace_buf_prepare(     \
-               __entry_size, event_call->id, &rctx, &irq_flags);       \
+               __entry_size, event_call->id, __regs, &rctx);           \
        if (!entry)                                                     \
                return;                                                 \
+                                                                       \
        tstruct                                                         \
                                                                        \
        { assign; }                                                     \
                                                                        \
-       __regs = &__get_cpu_var(perf_trace_regs);                       \
-       perf_fetch_caller_regs(__regs, 2);                              \
-                                                                       \
+       head = per_cpu_ptr(event_call->perf_events, smp_processor_id());\
        perf_trace_buf_submit(entry, __entry_size, rctx, __addr,        \
-                              __count, irq_flags, __regs);             \
+               __count, __regs, head);                                 \
 }
 
 #undef DEFINE_EVENT
-#define DEFINE_EVENT(template, call, proto, args)              \
-static notrace void perf_trace_##call(proto)                   \
-{                                                              \
-       struct ftrace_event_call *event_call = &event_##call;   \
-                                                               \
-       perf_trace_templ_##template(event_call, args);          \
+#define DEFINE_EVENT(template, call, proto, args)                      \
+static notrace void perf_trace_##call(proto)                           \
+{                                                                      \
+       struct ftrace_event_call *event_call = &event_##call;           \
+       struct pt_regs __regs;                                          \
+                                                                       \
+       perf_fetch_caller_regs(&__regs, 1);                             \
+       perf_trace_templ_##template(event_call, &__regs, args);         \
 }
 
 #undef DEFINE_EVENT_PRINT