[net-next-2.6.git] / kernel / trace / trace_syscalls.c

#include <trace/syscall.h>
#include <trace/events/syscalls.h>
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/ftrace.h>
#include <linux/perf_event.h>
#include <asm/syscall.h>

#include "trace_output.h"
#include "trace.h"

static DEFINE_MUTEX(syscall_trace_lock);
static int sys_refcount_enter;
static int sys_refcount_exit;
static DECLARE_BITMAP(enabled_enter_syscalls, NR_syscalls);
static DECLARE_BITMAP(enabled_exit_syscalls, NR_syscalls);

static int syscall_enter_register(struct ftrace_event_call *event,
				 enum trace_reg type);
static int syscall_exit_register(struct ftrace_event_call *event,
				 enum trace_reg type);

static int syscall_enter_define_fields(struct ftrace_event_call *call);
static int syscall_exit_define_fields(struct ftrace_event_call *call);

static struct list_head *
syscall_get_enter_fields(struct ftrace_event_call *call)
{
	struct syscall_metadata *entry = call->data;

	return &entry->enter_fields;
}

static struct list_head *
syscall_get_exit_fields(struct ftrace_event_call *call)
{
	struct syscall_metadata *entry = call->data;

	return &entry->exit_fields;
}

struct trace_event_functions enter_syscall_print_funcs = {
	.trace                  = print_syscall_enter,
};

struct trace_event_functions exit_syscall_print_funcs = {
	.trace                  = print_syscall_exit,
};

struct ftrace_event_class event_class_syscall_enter = {
	.system			= "syscalls",
	.reg			= syscall_enter_register,
	.define_fields		= syscall_enter_define_fields,
	.get_fields		= syscall_get_enter_fields,
	.raw_init		= init_syscall_trace,
};

struct ftrace_event_class event_class_syscall_exit = {
	.system			= "syscalls",
	.reg			= syscall_exit_register,
	.define_fields		= syscall_exit_define_fields,
	.get_fields		= syscall_get_exit_fields,
	.raw_init		= init_syscall_trace,
};

extern unsigned long __start_syscalls_metadata[];
extern unsigned long __stop_syscalls_metadata[];

static struct syscall_metadata **syscalls_metadata;

static struct syscall_metadata *find_syscall_meta(unsigned long syscall)
{
	struct syscall_metadata *start;
	struct syscall_metadata *stop;
	char str[KSYM_SYMBOL_LEN];


	start = (struct syscall_metadata *)__start_syscalls_metadata;
	stop = (struct syscall_metadata *)__stop_syscalls_metadata;
	kallsyms_lookup(syscall, NULL, NULL, NULL, str);

	for ( ; start < stop; start++) {
		/*
		 * Only compare after the "sys" prefix. Archs that use
		 * syscall wrappers may have syscalls symbols aliases prefixed
		 * with "SyS" instead of "sys", leading to an unwanted
		 * mismatch.
		 */
		if (start->name && !strcmp(start->name + 3, str + 3))
			return start;
	}
	return NULL;
}

static struct syscall_metadata *syscall_nr_to_meta(int nr)
{
	if (!syscalls_metadata || nr >= NR_syscalls || nr < 0)
		return NULL;

	return syscalls_metadata[nr];
}

enum print_line_t
print_syscall_enter(struct trace_iterator *iter, int flags,
		    struct trace_event *event)
{
	struct trace_seq *s = &iter->seq;
	struct trace_entry *ent = iter->ent;
	struct syscall_trace_enter *trace;
	struct syscall_metadata *entry;
	int i, ret, syscall;

	trace = (typeof(trace))ent;
	syscall = trace->nr;
	entry = syscall_nr_to_meta(syscall);

	if (!entry)
		goto end;

	if (entry->enter_event->id != ent->type) {
		WARN_ON_ONCE(1);
		goto end;
	}

	ret = trace_seq_printf(s, "%s(", entry->name);
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	for (i = 0; i < entry->nb_args; i++) {
		/* parameter types */
		if (trace_flags & TRACE_ITER_VERBOSE) {
			ret = trace_seq_printf(s, "%s ", entry->types[i]);
			if (!ret)
				return TRACE_TYPE_PARTIAL_LINE;
		}
		/* parameter values */
		ret = trace_seq_printf(s, "%s: %lx%s", entry->args[i],
				       trace->args[i],
				       i == entry->nb_args - 1 ? "" : ", ");
		if (!ret)
			return TRACE_TYPE_PARTIAL_LINE;
	}

	ret = trace_seq_putc(s, ')');
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

end:
	ret =  trace_seq_putc(s, '\n');
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	return TRACE_TYPE_HANDLED;
}

enum print_line_t
print_syscall_exit(struct trace_iterator *iter, int flags,
		   struct trace_event *event)
{
	struct trace_seq *s = &iter->seq;
	struct trace_entry *ent = iter->ent;
	struct syscall_trace_exit *trace;
	int syscall;
	struct syscall_metadata *entry;
	int ret;

	trace = (typeof(trace))ent;
	syscall = trace->nr;
	entry = syscall_nr_to_meta(syscall);

	if (!entry) {
		trace_seq_printf(s, "\n");
		return TRACE_TYPE_HANDLED;
	}

	if (entry->exit_event->id != ent->type) {
		WARN_ON_ONCE(1);
		return TRACE_TYPE_UNHANDLED;
	}

	ret = trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,
				trace->ret);
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	return TRACE_TYPE_HANDLED;
}

extern char *__bad_type_size(void);

#define SYSCALL_FIELD(type, name)					\
	sizeof(type) != sizeof(trace.name) ?				\
		__bad_type_size() :					\
		#type, #name, offsetof(typeof(trace), name),		\
		sizeof(trace.name), is_signed_type(type)

static
int  __set_enter_print_fmt(struct syscall_metadata *entry, char *buf, int len)
{
	int i;
	int pos = 0;

	/* When len=0, we just calculate the needed length */
#define LEN_OR_ZERO (len ? len - pos : 0)

	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
	for (i = 0; i < entry->nb_args; i++) {
		pos += snprintf(buf + pos, LEN_OR_ZERO, "%s: 0x%%0%zulx%s",
				entry->args[i], sizeof(unsigned long),
				i == entry->nb_args - 1 ? "" : ", ");
	}
	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");

	for (i = 0; i < entry->nb_args; i++) {
		pos += snprintf(buf + pos, LEN_OR_ZERO,
				", ((unsigned long)(REC->%s))", entry->args[i]);
	}

#undef LEN_OR_ZERO

	/* return the length of print_fmt */
	return pos;
}

static int set_syscall_print_fmt(struct ftrace_event_call *call)
{
	char *print_fmt;
	int len;
	struct syscall_metadata *entry = call->data;

	if (entry->enter_event != call) {
		call->print_fmt = "\"0x%lx\", REC->ret";
		return 0;
	}

	/* First: called with 0 length to calculate the needed length */
	len = __set_enter_print_fmt(entry, NULL, 0);

	print_fmt = kmalloc(len + 1, GFP_KERNEL);
	if (!print_fmt)
		return -ENOMEM;

	/* Second: actually write the @print_fmt */
	__set_enter_print_fmt(entry, print_fmt, len + 1);
	call->print_fmt = print_fmt;

	return 0;
}

static void free_syscall_print_fmt(struct ftrace_event_call *call)
{
	struct syscall_metadata *entry = call->data;

	if (entry->enter_event == call)
		kfree(call->print_fmt);
}

static int syscall_enter_define_fields(struct ftrace_event_call *call)
{
	struct syscall_trace_enter trace;
	struct syscall_metadata *meta = call->data;
	int ret;
	int i;
	int offset = offsetof(typeof(trace), args);

	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	if (ret)
		return ret;

	for (i = 0; i < meta->nb_args; i++) {
		ret = trace_define_field(call, meta->types[i],
					 meta->args[i], offset,
					 sizeof(unsigned long), 0,
					 FILTER_OTHER);
		offset += sizeof(unsigned long);
	}

	return ret;
}

static int syscall_exit_define_fields(struct ftrace_event_call *call)
{
	struct syscall_trace_exit trace;
	int ret;

	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	if (ret)
		return ret;

	ret = trace_define_field(call, SYSCALL_FIELD(long, ret),
				 FILTER_OTHER);

	return ret;
}

void ftrace_syscall_enter(void *ignore, struct pt_regs *regs, long id)
{
	struct syscall_trace_enter *entry;
	struct syscall_metadata *sys_data;
	struct ring_buffer_event *event;
	struct ring_buffer *buffer;
	int size;
	int syscall_nr;

	syscall_nr = syscall_get_nr(current, regs);
	if (syscall_nr < 0)
		return;
	if (!test_bit(syscall_nr, enabled_enter_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	size = sizeof(*entry) + sizeof(unsigned long) * sys_data->nb_args;

	event = trace_current_buffer_lock_reserve(&buffer,
			sys_data->enter_event->id, size, 0, 0);
	if (!event)
		return;

	entry = ring_buffer_event_data(event);
	entry->nr = syscall_nr;
	syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args);

	if (!filter_current_check_discard(buffer, sys_data->enter_event,
					  entry, event))
		trace_current_buffer_unlock_commit(buffer, event, 0, 0);
}

void ftrace_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
{
	struct syscall_trace_exit *entry;
	struct syscall_metadata *sys_data;
	struct ring_buffer_event *event;
	struct ring_buffer *buffer;
	int syscall_nr;

	syscall_nr = syscall_get_nr(current, regs);
	if (syscall_nr < 0)
		return;
	if (!test_bit(syscall_nr, enabled_exit_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	event = trace_current_buffer_lock_reserve(&buffer,
			sys_data->exit_event->id, sizeof(*entry), 0, 0);
	if (!event)
		return;

	entry = ring_buffer_event_data(event);
	entry->nr = syscall_nr;
	entry->ret = syscall_get_return_value(current, regs);

	if (!filter_current_check_discard(buffer, sys_data->exit_event,
					  entry, event))
		trace_current_buffer_unlock_commit(buffer, event, 0, 0);
}

int reg_event_syscall_enter(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return -ENOSYS;
	mutex_lock(&syscall_trace_lock);
	if (!sys_refcount_enter)
		ret = register_trace_sys_enter(ftrace_syscall_enter, NULL);
	if (!ret) {
		set_bit(num, enabled_enter_syscalls);
		sys_refcount_enter++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void unreg_event_syscall_enter(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return;
	mutex_lock(&syscall_trace_lock);
	sys_refcount_enter--;
	clear_bit(num, enabled_enter_syscalls);
	if (!sys_refcount_enter)
		unregister_trace_sys_enter(ftrace_syscall_enter, NULL);
	mutex_unlock(&syscall_trace_lock);
}

int reg_event_syscall_exit(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return -ENOSYS;
	mutex_lock(&syscall_trace_lock);
	if (!sys_refcount_exit)
		ret = register_trace_sys_exit(ftrace_syscall_exit, NULL);
	if (!ret) {
		set_bit(num, enabled_exit_syscalls);
		sys_refcount_exit++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void unreg_event_syscall_exit(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return;
	mutex_lock(&syscall_trace_lock);
	sys_refcount_exit--;
	clear_bit(num, enabled_exit_syscalls);
	if (!sys_refcount_exit)
		unregister_trace_sys_exit(ftrace_syscall_exit, NULL);
	mutex_unlock(&syscall_trace_lock);
}

int init_syscall_trace(struct ftrace_event_call *call)
{
	int id;

	if (set_syscall_print_fmt(call) < 0)
		return -ENOMEM;

	id = trace_event_raw_init(call);

	if (id < 0) {
		free_syscall_print_fmt(call);
		return id;
	}

	return id;
}

unsigned long __init arch_syscall_addr(int nr)
{
	return (unsigned long)sys_call_table[nr];
}

int __init init_ftrace_syscalls(void)
{
	struct syscall_metadata *meta;
	unsigned long addr;
	int i;

	syscalls_metadata = kzalloc(sizeof(*syscalls_metadata) *
					NR_syscalls, GFP_KERNEL);
	if (!syscalls_metadata) {
		WARN_ON(1);
		return -ENOMEM;
	}

	for (i = 0; i < NR_syscalls; i++) {
		addr = arch_syscall_addr(i);
		meta = find_syscall_meta(addr);
		if (!meta)
			continue;

		meta->syscall_nr = i;
		syscalls_metadata[i] = meta;
	}

	return 0;
}
core_initcall(init_ftrace_syscalls);

#ifdef CONFIG_PERF_EVENTS

static DECLARE_BITMAP(enabled_perf_enter_syscalls, NR_syscalls);
static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls);
static int sys_perf_refcount_enter;
static int sys_perf_refcount_exit;

static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
{
	struct syscall_metadata *sys_data;
	struct syscall_trace_enter *rec;
	unsigned long flags;
	int syscall_nr;
	int rctx;
	int size;

	syscall_nr = syscall_get_nr(current, regs);
	if (!test_bit(syscall_nr, enabled_perf_enter_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	/* get the size after alignment with the u32 buffer size field */
	size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
	size = ALIGN(size + sizeof(u32), sizeof(u64));
	size -= sizeof(u32);

	if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
		      "perf buffer not large enough"))
		return;

	rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size,
				sys_data->enter_event->id, &rctx, &flags);
	if (!rec)
		return;

	rec->nr = syscall_nr;
	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
			       (unsigned long *)&rec->args);
	perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs);
}

int perf_sysenter_enable(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;

	mutex_lock(&syscall_trace_lock);
	if (!sys_perf_refcount_enter)
		ret = register_trace_sys_enter(perf_syscall_enter, NULL);
	if (ret) {
		pr_info("event trace: Could not activate"
				"syscall entry trace point");
	} else {
		set_bit(num, enabled_perf_enter_syscalls);
		sys_perf_refcount_enter++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void perf_sysenter_disable(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;

	mutex_lock(&syscall_trace_lock);
	sys_perf_refcount_enter--;
	clear_bit(num, enabled_perf_enter_syscalls);
	if (!sys_perf_refcount_enter)
		unregister_trace_sys_enter(perf_syscall_enter, NULL);
	mutex_unlock(&syscall_trace_lock);
}

static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
{
	struct syscall_metadata *sys_data;
	struct syscall_trace_exit *rec;
	unsigned long flags;
	int syscall_nr;
	int rctx;
	int size;

	syscall_nr = syscall_get_nr(current, regs);
	if (!test_bit(syscall_nr, enabled_perf_exit_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	/* We can probably do that at build time */
	size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
	size -= sizeof(u32);

	/*
	 * Impossible, but be paranoid with the future
	 * How to put this check outside runtime?
	 */
	if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
		"exit event has grown above perf buffer size"))
		return;

	rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size,
				sys_data->exit_event->id, &rctx, &flags);
	if (!rec)
		return;

	rec->nr = syscall_nr;
	rec->ret = syscall_get_return_value(current, regs);

	perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs);
}

int perf_sysexit_enable(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;

	mutex_lock(&syscall_trace_lock);
	if (!sys_perf_refcount_exit)
		ret = register_trace_sys_exit(perf_syscall_exit, NULL);
	if (ret) {
		pr_info("event trace: Could not activate"
				"syscall exit trace point");
	} else {
		set_bit(num, enabled_perf_exit_syscalls);
		sys_perf_refcount_exit++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void perf_sysexit_disable(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;

	mutex_lock(&syscall_trace_lock);
	sys_perf_refcount_exit--;
	clear_bit(num, enabled_perf_exit_syscalls);
	if (!sys_perf_refcount_exit)
		unregister_trace_sys_exit(perf_syscall_exit, NULL);
	mutex_unlock(&syscall_trace_lock);
}

#endif /* CONFIG_PERF_EVENTS */

static int syscall_enter_register(struct ftrace_event_call *event,
				 enum trace_reg type)
{
	switch (type) {
	case TRACE_REG_REGISTER:
		return reg_event_syscall_enter(event);
	case TRACE_REG_UNREGISTER:
		unreg_event_syscall_enter(event);
		return 0;

#ifdef CONFIG_PERF_EVENTS
	case TRACE_REG_PERF_REGISTER:
		return perf_sysenter_enable(event);
	case TRACE_REG_PERF_UNREGISTER:
		perf_sysenter_disable(event);
		return 0;
#endif
	}
	return 0;
}

static int syscall_exit_register(struct ftrace_event_call *event,
				 enum trace_reg type)
{
	switch (type) {
	case TRACE_REG_REGISTER:
		return reg_event_syscall_exit(event);
	case TRACE_REG_UNREGISTER:
		unreg_event_syscall_exit(event);
		return 0;

#ifdef CONFIG_PERF_EVENTS
	case TRACE_REG_PERF_REGISTER:
		return perf_sysexit_enable(event);
	case TRACE_REG_PERF_UNREGISTER:
		perf_sysexit_disable(event);
		return 0;
#endif
	}
	return 0;
}
Commit	Line	Data
47788c58	1	#include <trace/syscall.h>
1c569f02	2	#include <trace/events/syscalls.h>
5a0e3ad6	3	#include <linux/slab.h>
ee08c6ec	4	#include <linux/kernel.h>
fb34a08c	5	#include <linux/ftrace.h>
cdd6c482	6	#include <linux/perf_event.h>
ee08c6ec FW	7	#include <asm/syscall.h>
	8
	9	#include "trace_output.h"
	10	#include "trace.h"
	11
5be71b61	12	static DEFINE_MUTEX(syscall_trace_lock);
fb34a08c JB	13	static int sys_refcount_enter;
fb34a08c JB	14	static int sys_refcount_exit;
57421dbb JB	15	static DECLARE_BITMAP(enabled_enter_syscalls, NR_syscalls);
57421dbb JB	16	static DECLARE_BITMAP(enabled_exit_syscalls, NR_syscalls);
ee08c6ec	17
2239291a SR	18	static int syscall_enter_register(struct ftrace_event_call *event,
	19	enum trace_reg type);
	20	static int syscall_exit_register(struct ftrace_event_call *event,
	21	enum trace_reg type);
	22
2e33af02 SR	23	static int syscall_enter_define_fields(struct ftrace_event_call *call);
	24	static int syscall_exit_define_fields(struct ftrace_event_call *call);
	25
	26	static struct list_head *
	27	syscall_get_enter_fields(struct ftrace_event_call *call)
	28	{
	29	struct syscall_metadata *entry = call->data;
	30
	31	return &entry->enter_fields;
	32	}
	33
	34	static struct list_head *
	35	syscall_get_exit_fields(struct ftrace_event_call *call)
	36	{
	37	struct syscall_metadata *entry = call->data;
	38
	39	return &entry->exit_fields;
	40	}
	41
80decc70 SR	42	struct trace_event_functions enter_syscall_print_funcs = {
	43	.trace = print_syscall_enter,
	44	};
	45
	46	struct trace_event_functions exit_syscall_print_funcs = {
	47	.trace = print_syscall_exit,
	48	};
	49
2239291a SR	50	struct ftrace_event_class event_class_syscall_enter = {
2239291a SR	51	.system = "syscalls",
2e33af02 SR	52	.reg = syscall_enter_register,
	53	.define_fields = syscall_enter_define_fields,
	54	.get_fields = syscall_get_enter_fields,
0405ab80	55	.raw_init = init_syscall_trace,
2239291a SR	56	};
	57
	58	struct ftrace_event_class event_class_syscall_exit = {
	59	.system = "syscalls",
2e33af02 SR	60	.reg = syscall_exit_register,
	61	.define_fields = syscall_exit_define_fields,
	62	.get_fields = syscall_get_exit_fields,
0405ab80	63	.raw_init = init_syscall_trace,
8f082018 SR	64	};
8f082018 SR	65
c44fc770 FW	66	extern unsigned long __start_syscalls_metadata[];
	67	extern unsigned long __stop_syscalls_metadata[];
	68
	69	static struct syscall_metadata **syscalls_metadata;
	70
	71	static struct syscall_metadata *find_syscall_meta(unsigned long syscall)
	72	{
	73	struct syscall_metadata *start;
	74	struct syscall_metadata *stop;
	75	char str[KSYM_SYMBOL_LEN];
	76
	77
	78	start = (struct syscall_metadata *)__start_syscalls_metadata;
	79	stop = (struct syscall_metadata *)__stop_syscalls_metadata;
	80	kallsyms_lookup(syscall, NULL, NULL, NULL, str);
	81
	82	for ( ; start < stop; start++) {
	83	/*
	84	* Only compare after the "sys" prefix. Archs that use
	85	* syscall wrappers may have syscalls symbols aliases prefixed
	86	* with "SyS" instead of "sys", leading to an unwanted
	87	* mismatch.
	88	*/
	89	if (start->name && !strcmp(start->name + 3, str + 3))
	90	return start;
	91	}
	92	return NULL;
	93	}
	94
	95	static struct syscall_metadata *syscall_nr_to_meta(int nr)
	96	{
	97	if (!syscalls_metadata \|\| nr >= NR_syscalls \|\| nr < 0)
	98	return NULL;
	99
	100	return syscalls_metadata[nr];
	101	}
	102
bed1ffca	103	enum print_line_t
a9a57763 SR	104	print_syscall_enter(struct trace_iterator *iter, int flags,
a9a57763 SR	105	struct trace_event *event)
bed1ffca FW	106	{
	107	struct trace_seq *s = &iter->seq;
	108	struct trace_entry *ent = iter->ent;
	109	struct syscall_trace_enter *trace;
	110	struct syscall_metadata *entry;
	111	int i, ret, syscall;
	112
64c12e04	113	trace = (typeof(trace))ent;
bed1ffca	114	syscall = trace->nr;
bed1ffca	115	entry = syscall_nr_to_meta(syscall);
64c12e04	116
bed1ffca FW	117	if (!entry)
	118	goto end;
	119
fcc19438	120	if (entry->enter_event->id != ent->type) {
64c12e04 JB	121	WARN_ON_ONCE(1);
	122	goto end;
	123	}
	124
bed1ffca FW	125	ret = trace_seq_printf(s, "%s(", entry->name);
	126	if (!ret)
	127	return TRACE_TYPE_PARTIAL_LINE;
	128
	129	for (i = 0; i < entry->nb_args; i++) {
	130	/* parameter types */
ba8b3a40	131	if (trace_flags & TRACE_ITER_VERBOSE) {
bed1ffca FW	132	ret = trace_seq_printf(s, "%s ", entry->types[i]);
	133	if (!ret)
	134	return TRACE_TYPE_PARTIAL_LINE;
	135	}
	136	/* parameter values */
4539f077	137	ret = trace_seq_printf(s, "%s: %lx%s", entry->args[i],
bed1ffca	138	trace->args[i],
4539f077	139	i == entry->nb_args - 1 ? "" : ", ");
bed1ffca FW	140	if (!ret)
	141	return TRACE_TYPE_PARTIAL_LINE;
	142	}
	143
4539f077 LZ	144	ret = trace_seq_putc(s, ')');
	145	if (!ret)
	146	return TRACE_TYPE_PARTIAL_LINE;
	147
bed1ffca	148	end:
4539f077 LZ	149	ret = trace_seq_putc(s, '\n');
	150	if (!ret)
	151	return TRACE_TYPE_PARTIAL_LINE;
	152
bed1ffca FW	153	return TRACE_TYPE_HANDLED;
	154	}
	155
	156	enum print_line_t
a9a57763 SR	157	print_syscall_exit(struct trace_iterator *iter, int flags,
a9a57763 SR	158	struct trace_event *event)
bed1ffca FW	159	{
	160	struct trace_seq *s = &iter->seq;
	161	struct trace_entry *ent = iter->ent;
	162	struct syscall_trace_exit *trace;
	163	int syscall;
	164	struct syscall_metadata *entry;
	165	int ret;
	166
64c12e04	167	trace = (typeof(trace))ent;
bed1ffca	168	syscall = trace->nr;
bed1ffca	169	entry = syscall_nr_to_meta(syscall);
64c12e04	170
bed1ffca FW	171	if (!entry) {
	172	trace_seq_printf(s, "\n");
	173	return TRACE_TYPE_HANDLED;
	174	}
	175
fcc19438	176	if (entry->exit_event->id != ent->type) {
64c12e04 JB	177	WARN_ON_ONCE(1);
	178	return TRACE_TYPE_UNHANDLED;
	179	}
	180
bed1ffca FW	181	ret = trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,
	182	trace->ret);
	183	if (!ret)
	184	return TRACE_TYPE_PARTIAL_LINE;
	185
	186	return TRACE_TYPE_HANDLED;
	187	}
	188
e6971969 LZ	189	extern char *__bad_type_size(void);
	190
	191	#define SYSCALL_FIELD(type, name) \
	192	sizeof(type) != sizeof(trace.name) ? \
	193	__bad_type_size() : \
26a50744 TZ	194	#type, #name, offsetof(typeof(trace), name), \
26a50744 TZ	195	sizeof(trace.name), is_signed_type(type)
e6971969	196
50307a45 LJ	197	static
	198	int __set_enter_print_fmt(struct syscall_metadata entry, char buf, int len)
	199	{
	200	int i;
	201	int pos = 0;
	202
	203	/* When len=0, we just calculate the needed length */
	204	#define LEN_OR_ZERO (len ? len - pos : 0)
	205
	206	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
	207	for (i = 0; i < entry->nb_args; i++) {
	208	pos += snprintf(buf + pos, LEN_OR_ZERO, "%s: 0x%%0%zulx%s",
	209	entry->args[i], sizeof(unsigned long),
	210	i == entry->nb_args - 1 ? "" : ", ");
	211	}
	212	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
	213
	214	for (i = 0; i < entry->nb_args; i++) {
	215	pos += snprintf(buf + pos, LEN_OR_ZERO,
	216	", ((unsigned long)(REC->%s))", entry->args[i]);
	217	}
	218
	219	#undef LEN_OR_ZERO
	220
	221	/* return the length of print_fmt */
	222	return pos;
	223	}
	224
	225	static int set_syscall_print_fmt(struct ftrace_event_call *call)
	226	{
	227	char *print_fmt;
	228	int len;
	229	struct syscall_metadata *entry = call->data;
	230
	231	if (entry->enter_event != call) {
	232	call->print_fmt = "\"0x%lx\", REC->ret";
	233	return 0;
	234	}
	235
	236	/* First: called with 0 length to calculate the needed length */
	237	len = __set_enter_print_fmt(entry, NULL, 0);
	238
	239	print_fmt = kmalloc(len + 1, GFP_KERNEL);
	240	if (!print_fmt)
	241	return -ENOMEM;
	242
	243	/* Second: actually write the @print_fmt */
	244	__set_enter_print_fmt(entry, print_fmt, len + 1);
	245	call->print_fmt = print_fmt;
	246
	247	return 0;
	248	}
	249
	250	static void free_syscall_print_fmt(struct ftrace_event_call *call)
	251	{
	252	struct syscall_metadata *entry = call->data;
	253
	254	if (entry->enter_event == call)
	255	kfree(call->print_fmt);
	256	}
	257
2e33af02	258	static int syscall_enter_define_fields(struct ftrace_event_call *call)
540b7b8d LZ	259	{
540b7b8d LZ	260	struct syscall_trace_enter trace;
31c16b13	261	struct syscall_metadata *meta = call->data;
540b7b8d	262	int ret;
540b7b8d LZ	263	int i;
	264	int offset = offsetof(typeof(trace), args);
	265
0f1ef51d LJ	266	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	267	if (ret)
	268	return ret;
	269
540b7b8d	270	for (i = 0; i < meta->nb_args; i++) {
aeaeae11 FW	271	ret = trace_define_field(call, meta->types[i],
aeaeae11 FW	272	meta->args[i], offset,
43b51ead LZ	273	sizeof(unsigned long), 0,
43b51ead LZ	274	FILTER_OTHER);
540b7b8d LZ	275	offset += sizeof(unsigned long);
	276	}
	277
	278	return ret;
	279	}
	280
2e33af02	281	static int syscall_exit_define_fields(struct ftrace_event_call *call)
540b7b8d LZ	282	{
	283	struct syscall_trace_exit trace;
	284	int ret;
	285
0f1ef51d LJ	286	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	287	if (ret)
	288	return ret;
	289
26a50744	290	ret = trace_define_field(call, SYSCALL_FIELD(long, ret),
43b51ead	291	FILTER_OTHER);
540b7b8d LZ	292
	293	return ret;
	294	}
	295
38516ab5	296	void ftrace_syscall_enter(void ignore, struct pt_regs regs, long id)
ee08c6ec	297	{
bed1ffca FW	298	struct syscall_trace_enter *entry;
	299	struct syscall_metadata *sys_data;
	300	struct ring_buffer_event *event;
e77405ad	301	struct ring_buffer *buffer;
bed1ffca	302	int size;
ee08c6ec FW	303	int syscall_nr;
	304
	305	syscall_nr = syscall_get_nr(current, regs);
cd0980fc HB	306	if (syscall_nr < 0)
cd0980fc HB	307	return;
fb34a08c JB	308	if (!test_bit(syscall_nr, enabled_enter_syscalls))
fb34a08c JB	309	return;
ee08c6ec	310
bed1ffca FW	311	sys_data = syscall_nr_to_meta(syscall_nr);
	312	if (!sys_data)
	313	return;
	314
	315	size = sizeof(entry) + sizeof(unsigned long) sys_data->nb_args;
	316
fcc19438 LJ	317	event = trace_current_buffer_lock_reserve(&buffer,
fcc19438 LJ	318	sys_data->enter_event->id, size, 0, 0);
bed1ffca FW	319	if (!event)
	320	return;
	321
	322	entry = ring_buffer_event_data(event);
	323	entry->nr = syscall_nr;
	324	syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args);
	325
e77405ad SR	326	if (!filter_current_check_discard(buffer, sys_data->enter_event,
	327	entry, event))
	328	trace_current_buffer_unlock_commit(buffer, event, 0, 0);
ee08c6ec FW	329	}
ee08c6ec FW	330
38516ab5	331	void ftrace_syscall_exit(void ignore, struct pt_regs regs, long ret)
ee08c6ec	332	{
bed1ffca FW	333	struct syscall_trace_exit *entry;
	334	struct syscall_metadata *sys_data;
	335	struct ring_buffer_event *event;
e77405ad	336	struct ring_buffer *buffer;
ee08c6ec FW	337	int syscall_nr;
	338
	339	syscall_nr = syscall_get_nr(current, regs);
cd0980fc HB	340	if (syscall_nr < 0)
cd0980fc HB	341	return;
fb34a08c JB	342	if (!test_bit(syscall_nr, enabled_exit_syscalls))
fb34a08c JB	343	return;
ee08c6ec	344
bed1ffca FW	345	sys_data = syscall_nr_to_meta(syscall_nr);
	346	if (!sys_data)
	347	return;
	348
fcc19438 LJ	349	event = trace_current_buffer_lock_reserve(&buffer,
fcc19438 LJ	350	sys_data->exit_event->id, sizeof(*entry), 0, 0);
bed1ffca FW	351	if (!event)
	352	return;
	353
	354	entry = ring_buffer_event_data(event);
	355	entry->nr = syscall_nr;
	356	entry->ret = syscall_get_return_value(current, regs);
	357
e77405ad SR	358	if (!filter_current_check_discard(buffer, sys_data->exit_event,
	359	entry, event))
	360	trace_current_buffer_unlock_commit(buffer, event, 0, 0);
ee08c6ec FW	361	}
ee08c6ec FW	362
bd1a5c84	363	int reg_event_syscall_enter(struct ftrace_event_call *call)
ee08c6ec	364	{
fb34a08c JB	365	int ret = 0;
fb34a08c JB	366	int num;
fb34a08c	367
c252f657	368	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	369	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	370	return -ENOSYS;
	371	mutex_lock(&syscall_trace_lock);
	372	if (!sys_refcount_enter)
38516ab5	373	ret = register_trace_sys_enter(ftrace_syscall_enter, NULL);
3b8e4273	374	if (!ret) {
fb34a08c JB	375	set_bit(num, enabled_enter_syscalls);
	376	sys_refcount_enter++;
	377	}
	378	mutex_unlock(&syscall_trace_lock);
	379	return ret;
ee08c6ec FW	380	}
ee08c6ec FW	381
bd1a5c84	382	void unreg_event_syscall_enter(struct ftrace_event_call *call)
ee08c6ec	383	{
fb34a08c	384	int num;
ee08c6ec	385
c252f657	386	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	387	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	388	return;
	389	mutex_lock(&syscall_trace_lock);
	390	sys_refcount_enter--;
	391	clear_bit(num, enabled_enter_syscalls);
	392	if (!sys_refcount_enter)
38516ab5	393	unregister_trace_sys_enter(ftrace_syscall_enter, NULL);
fb34a08c JB	394	mutex_unlock(&syscall_trace_lock);
fb34a08c JB	395	}
ee08c6ec	396
bd1a5c84	397	int reg_event_syscall_exit(struct ftrace_event_call *call)
ee08c6ec	398	{
fb34a08c JB	399	int ret = 0;
fb34a08c JB	400	int num;
fb34a08c	401
c252f657	402	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	403	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	404	return -ENOSYS;
	405	mutex_lock(&syscall_trace_lock);
	406	if (!sys_refcount_exit)
38516ab5	407	ret = register_trace_sys_exit(ftrace_syscall_exit, NULL);
3b8e4273	408	if (!ret) {
fb34a08c JB	409	set_bit(num, enabled_exit_syscalls);
fb34a08c JB	410	sys_refcount_exit++;
ee08c6ec	411	}
fb34a08c JB	412	mutex_unlock(&syscall_trace_lock);
	413	return ret;
	414	}
ee08c6ec	415
bd1a5c84	416	void unreg_event_syscall_exit(struct ftrace_event_call *call)
fb34a08c JB	417	{
fb34a08c JB	418	int num;
ee08c6ec	419
c252f657	420	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	421	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	422	return;
	423	mutex_lock(&syscall_trace_lock);
	424	sys_refcount_exit--;
	425	clear_bit(num, enabled_exit_syscalls);
	426	if (!sys_refcount_exit)
38516ab5	427	unregister_trace_sys_exit(ftrace_syscall_exit, NULL);
fb34a08c	428	mutex_unlock(&syscall_trace_lock);
ee08c6ec	429	}
fb34a08c	430
a1301da0 LJ	431	int init_syscall_trace(struct ftrace_event_call *call)
	432	{
	433	int id;
	434
50307a45 LJ	435	if (set_syscall_print_fmt(call) < 0)
	436	return -ENOMEM;
	437
c7ef3a90 SR	438	id = trace_event_raw_init(call);
	439
	440	if (id < 0) {
50307a45	441	free_syscall_print_fmt(call);
c7ef3a90	442	return id;
50307a45	443	}
c7ef3a90 SR	444
c7ef3a90 SR	445	return id;
a1301da0 LJ	446	}
a1301da0 LJ	447
e7b8e675 MF	448	unsigned long __init arch_syscall_addr(int nr)
	449	{
	450	return (unsigned long)sys_call_table[nr];
	451	}
	452
c44fc770 FW	453	int __init init_ftrace_syscalls(void)
	454	{
	455	struct syscall_metadata *meta;
	456	unsigned long addr;
	457	int i;
	458
	459	syscalls_metadata = kzalloc(sizeof(syscalls_metadata)
	460	NR_syscalls, GFP_KERNEL);
	461	if (!syscalls_metadata) {
	462	WARN_ON(1);
	463	return -ENOMEM;
	464	}
	465
	466	for (i = 0; i < NR_syscalls; i++) {
	467	addr = arch_syscall_addr(i);
	468	meta = find_syscall_meta(addr);
c252f657 LJ	469	if (!meta)
	470	continue;
	471
	472	meta->syscall_nr = i;
c44fc770 FW	473	syscalls_metadata[i] = meta;
	474	}
	475
	476	return 0;
	477	}
	478	core_initcall(init_ftrace_syscalls);
	479
07b139c8	480	#ifdef CONFIG_PERF_EVENTS
19007a67	481
97d5a220 FW	482	static DECLARE_BITMAP(enabled_perf_enter_syscalls, NR_syscalls);
	483	static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls);
	484	static int sys_perf_refcount_enter;
	485	static int sys_perf_refcount_exit;
f4b5ffcc	486
38516ab5	487	static void perf_syscall_enter(void ignore, struct pt_regs regs, long id)
f4b5ffcc JB	488	{
f4b5ffcc JB	489	struct syscall_metadata *sys_data;
20ab4425 FW	490	struct syscall_trace_enter *rec;
20ab4425 FW	491	unsigned long flags;
f4b5ffcc	492	int syscall_nr;
4ed7c92d	493	int rctx;
19007a67	494	int size;
f4b5ffcc JB	495
f4b5ffcc JB	496	syscall_nr = syscall_get_nr(current, regs);
97d5a220	497	if (!test_bit(syscall_nr, enabled_perf_enter_syscalls))
f4b5ffcc JB	498	return;
	499
	500	sys_data = syscall_nr_to_meta(syscall_nr);
	501	if (!sys_data)
	502	return;
	503
19007a67 FW	504	/* get the size after alignment with the u32 buffer size field */
	505	size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
	506	size = ALIGN(size + sizeof(u32), sizeof(u64));
	507	size -= sizeof(u32);
	508
97d5a220 FW	509	if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
97d5a220 FW	510	"perf buffer not large enough"))
20ab4425 FW	511	return;
20ab4425 FW	512
97d5a220	513	rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size,
430ad5a6 XG	514	sys_data->enter_event->id, &rctx, &flags);
	515	if (!rec)
	516	return;
20ab4425	517
20ab4425 FW	518	rec->nr = syscall_nr;
	519	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
	520	(unsigned long *)&rec->args);
97d5a220	521	perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs);
f4b5ffcc JB	522	}
f4b5ffcc JB	523
97d5a220	524	int perf_sysenter_enable(struct ftrace_event_call *call)
f4b5ffcc JB	525	{
	526	int ret = 0;
	527	int num;
	528
3bbe84e9	529	num = ((struct syscall_metadata *)call->data)->syscall_nr;
f4b5ffcc JB	530
f4b5ffcc JB	531	mutex_lock(&syscall_trace_lock);
97d5a220	532	if (!sys_perf_refcount_enter)
38516ab5	533	ret = register_trace_sys_enter(perf_syscall_enter, NULL);
f4b5ffcc JB	534	if (ret) {
	535	pr_info("event trace: Could not activate"
	536	"syscall entry trace point");
	537	} else {
97d5a220 FW	538	set_bit(num, enabled_perf_enter_syscalls);
97d5a220 FW	539	sys_perf_refcount_enter++;
f4b5ffcc JB	540	}
	541	mutex_unlock(&syscall_trace_lock);
	542	return ret;
	543	}
	544
97d5a220	545	void perf_sysenter_disable(struct ftrace_event_call *call)
f4b5ffcc JB	546	{
	547	int num;
	548
3bbe84e9	549	num = ((struct syscall_metadata *)call->data)->syscall_nr;
f4b5ffcc JB	550
f4b5ffcc JB	551	mutex_lock(&syscall_trace_lock);
97d5a220 FW	552	sys_perf_refcount_enter--;
	553	clear_bit(num, enabled_perf_enter_syscalls);
	554	if (!sys_perf_refcount_enter)
38516ab5	555	unregister_trace_sys_enter(perf_syscall_enter, NULL);
f4b5ffcc JB	556	mutex_unlock(&syscall_trace_lock);
	557	}
	558
38516ab5	559	static void perf_syscall_exit(void ignore, struct pt_regs regs, long ret)
f4b5ffcc JB	560	{
f4b5ffcc JB	561	struct syscall_metadata *sys_data;
20ab4425 FW	562	struct syscall_trace_exit *rec;
20ab4425 FW	563	unsigned long flags;
f4b5ffcc	564	int syscall_nr;
4ed7c92d	565	int rctx;
20ab4425	566	int size;
f4b5ffcc JB	567
f4b5ffcc JB	568	syscall_nr = syscall_get_nr(current, regs);
97d5a220	569	if (!test_bit(syscall_nr, enabled_perf_exit_syscalls))
f4b5ffcc JB	570	return;
	571
	572	sys_data = syscall_nr_to_meta(syscall_nr);
	573	if (!sys_data)
	574	return;
	575
20ab4425 FW	576	/* We can probably do that at build time */
	577	size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
	578	size -= sizeof(u32);
19007a67	579
20ab4425 FW	580	/*
	581	* Impossible, but be paranoid with the future
	582	* How to put this check outside runtime?
	583	*/
97d5a220 FW	584	if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
97d5a220 FW	585	"exit event has grown above perf buffer size"))
20ab4425 FW	586	return;
20ab4425 FW	587
97d5a220	588	rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size,
430ad5a6 XG	589	sys_data->exit_event->id, &rctx, &flags);
	590	if (!rec)
	591	return;
20ab4425	592
20ab4425 FW	593	rec->nr = syscall_nr;
	594	rec->ret = syscall_get_return_value(current, regs);
	595
97d5a220	596	perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs);
f4b5ffcc JB	597	}
f4b5ffcc JB	598
97d5a220	599	int perf_sysexit_enable(struct ftrace_event_call *call)
f4b5ffcc JB	600	{
	601	int ret = 0;
	602	int num;
	603
3bbe84e9	604	num = ((struct syscall_metadata *)call->data)->syscall_nr;
f4b5ffcc JB	605
f4b5ffcc JB	606	mutex_lock(&syscall_trace_lock);
97d5a220	607	if (!sys_perf_refcount_exit)
38516ab5	608	ret = register_trace_sys_exit(perf_syscall_exit, NULL);
f4b5ffcc JB	609	if (ret) {
f4b5ffcc JB	610	pr_info("event trace: Could not activate"
6574658b	611	"syscall exit trace point");
f4b5ffcc	612	} else {
97d5a220 FW	613	set_bit(num, enabled_perf_exit_syscalls);
97d5a220 FW	614	sys_perf_refcount_exit++;
f4b5ffcc JB	615	}
	616	mutex_unlock(&syscall_trace_lock);
	617	return ret;
	618	}
	619
97d5a220	620	void perf_sysexit_disable(struct ftrace_event_call *call)
f4b5ffcc JB	621	{
	622	int num;
	623
3bbe84e9	624	num = ((struct syscall_metadata *)call->data)->syscall_nr;
f4b5ffcc JB	625
f4b5ffcc JB	626	mutex_lock(&syscall_trace_lock);
97d5a220 FW	627	sys_perf_refcount_exit--;
	628	clear_bit(num, enabled_perf_exit_syscalls);
	629	if (!sys_perf_refcount_exit)
38516ab5	630	unregister_trace_sys_exit(perf_syscall_exit, NULL);
f4b5ffcc JB	631	mutex_unlock(&syscall_trace_lock);
	632	}
	633
07b139c8	634	#endif /* CONFIG_PERF_EVENTS */
f4b5ffcc	635
2239291a SR	636	static int syscall_enter_register(struct ftrace_event_call *event,
	637	enum trace_reg type)
	638	{
	639	switch (type) {
	640	case TRACE_REG_REGISTER:
	641	return reg_event_syscall_enter(event);
	642	case TRACE_REG_UNREGISTER:
	643	unreg_event_syscall_enter(event);
	644	return 0;
	645
	646	#ifdef CONFIG_PERF_EVENTS
	647	case TRACE_REG_PERF_REGISTER:
	648	return perf_sysenter_enable(event);
	649	case TRACE_REG_PERF_UNREGISTER:
	650	perf_sysenter_disable(event);
	651	return 0;
	652	#endif
	653	}
	654	return 0;
	655	}
	656
	657	static int syscall_exit_register(struct ftrace_event_call *event,
	658	enum trace_reg type)
	659	{
	660	switch (type) {
	661	case TRACE_REG_REGISTER:
	662	return reg_event_syscall_exit(event);
	663	case TRACE_REG_UNREGISTER:
	664	unreg_event_syscall_exit(event);
	665	return 0;
	666
	667	#ifdef CONFIG_PERF_EVENTS
	668	case TRACE_REG_PERF_REGISTER:
	669	return perf_sysexit_enable(event);
	670	case TRACE_REG_PERF_UNREGISTER:
	671	perf_sysexit_disable(event);
	672	return 0;
	673	#endif
	674	}
	675	return 0;
	676	}