[net-next-2.6.git] / kernel / trace / trace_syscalls.c

#include <trace/syscall.h>
#include <trace/events/syscalls.h>
#include <linux/kernel.h>
#include <linux/ftrace.h>
#include <linux/perf_event.h>
#include <asm/syscall.h>

#include "trace_output.h"
#include "trace.h"

static DEFINE_MUTEX(syscall_trace_lock);
static int sys_refcount_enter;
static int sys_refcount_exit;
static DECLARE_BITMAP(enabled_enter_syscalls, NR_syscalls);
static DECLARE_BITMAP(enabled_exit_syscalls, NR_syscalls);

extern unsigned long __start_syscalls_metadata[];
extern unsigned long __stop_syscalls_metadata[];

static struct syscall_metadata **syscalls_metadata;

static struct syscall_metadata *find_syscall_meta(unsigned long syscall)
{
	struct syscall_metadata *start;
	struct syscall_metadata *stop;
	char str[KSYM_SYMBOL_LEN];


	start = (struct syscall_metadata *)__start_syscalls_metadata;
	stop = (struct syscall_metadata *)__stop_syscalls_metadata;
	kallsyms_lookup(syscall, NULL, NULL, NULL, str);

	for ( ; start < stop; start++) {
		/*
		 * Only compare after the "sys" prefix. Archs that use
		 * syscall wrappers may have syscalls symbols aliases prefixed
		 * with "SyS" instead of "sys", leading to an unwanted
		 * mismatch.
		 */
		if (start->name && !strcmp(start->name + 3, str + 3))
			return start;
	}
	return NULL;
}

static struct syscall_metadata *syscall_nr_to_meta(int nr)
{
	if (!syscalls_metadata || nr >= NR_syscalls || nr < 0)
		return NULL;

	return syscalls_metadata[nr];
}

static int syscall_name_to_nr(const char *name)
{
	int i;

	if (!syscalls_metadata)
		return -1;

	for (i = 0; i < NR_syscalls; i++) {
		if (syscalls_metadata[i]) {
			if (!strcmp(syscalls_metadata[i]->name, name))
				return i;
		}
	}
	return -1;
}

enum print_line_t
print_syscall_enter(struct trace_iterator *iter, int flags)
{
	struct trace_seq *s = &iter->seq;
	struct trace_entry *ent = iter->ent;
	struct syscall_trace_enter *trace;
	struct syscall_metadata *entry;
	int i, ret, syscall;

	trace = (typeof(trace))ent;
	syscall = trace->nr;
	entry = syscall_nr_to_meta(syscall);

	if (!entry)
		goto end;

	if (entry->enter_event->id != ent->type) {
		WARN_ON_ONCE(1);
		goto end;
	}

	ret = trace_seq_printf(s, "%s(", entry->name);
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	for (i = 0; i < entry->nb_args; i++) {
		/* parameter types */
		if (trace_flags & TRACE_ITER_VERBOSE) {
			ret = trace_seq_printf(s, "%s ", entry->types[i]);
			if (!ret)
				return TRACE_TYPE_PARTIAL_LINE;
		}
		/* parameter values */
		ret = trace_seq_printf(s, "%s: %lx%s", entry->args[i],
				       trace->args[i],
				       i == entry->nb_args - 1 ? "" : ", ");
		if (!ret)
			return TRACE_TYPE_PARTIAL_LINE;
	}

	ret = trace_seq_putc(s, ')');
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

end:
	ret =  trace_seq_putc(s, '\n');
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	return TRACE_TYPE_HANDLED;
}

enum print_line_t
print_syscall_exit(struct trace_iterator *iter, int flags)
{
	struct trace_seq *s = &iter->seq;
	struct trace_entry *ent = iter->ent;
	struct syscall_trace_exit *trace;
	int syscall;
	struct syscall_metadata *entry;
	int ret;

	trace = (typeof(trace))ent;
	syscall = trace->nr;
	entry = syscall_nr_to_meta(syscall);

	if (!entry) {
		trace_seq_printf(s, "\n");
		return TRACE_TYPE_HANDLED;
	}

	if (entry->exit_event->id != ent->type) {
		WARN_ON_ONCE(1);
		return TRACE_TYPE_UNHANDLED;
	}

	ret = trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,
				trace->ret);
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	return TRACE_TYPE_HANDLED;
}

extern char *__bad_type_size(void);

#define SYSCALL_FIELD(type, name)					\
	sizeof(type) != sizeof(trace.name) ?				\
		__bad_type_size() :					\
		#type, #name, offsetof(typeof(trace), name),		\
		sizeof(trace.name), is_signed_type(type)

int syscall_enter_format(struct ftrace_event_call *call, struct trace_seq *s)
{
	int i;
	int ret;
	struct syscall_metadata *entry = call->data;
	struct syscall_trace_enter trace;
	int offset = offsetof(struct syscall_trace_enter, args);

	ret = trace_seq_printf(s, "\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
			       "\tsigned:%u;\n",
			       SYSCALL_FIELD(int, nr));
	if (!ret)
		return 0;

	for (i = 0; i < entry->nb_args; i++) {
		ret = trace_seq_printf(s, "\tfield:%s %s;", entry->types[i],
				        entry->args[i]);
		if (!ret)
			return 0;
		ret = trace_seq_printf(s, "\toffset:%d;\tsize:%zu;"
				       "\tsigned:%u;\n", offset,
				       sizeof(unsigned long),
				       is_signed_type(unsigned long));
		if (!ret)
			return 0;
		offset += sizeof(unsigned long);
	}

	trace_seq_puts(s, "\nprint fmt: \"");
	for (i = 0; i < entry->nb_args; i++) {
		ret = trace_seq_printf(s, "%s: 0x%%0%zulx%s", entry->args[i],
				        sizeof(unsigned long),
					i == entry->nb_args - 1 ? "" : ", ");
		if (!ret)
			return 0;
	}
	trace_seq_putc(s, '"');

	for (i = 0; i < entry->nb_args; i++) {
		ret = trace_seq_printf(s, ", ((unsigned long)(REC->%s))",
				       entry->args[i]);
		if (!ret)
			return 0;
	}

	return trace_seq_putc(s, '\n');
}

int syscall_exit_format(struct ftrace_event_call *call, struct trace_seq *s)
{
	int ret;
	struct syscall_trace_exit trace;

	ret = trace_seq_printf(s,
			       "\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
			       "\tsigned:%u;\n"
			       "\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
			       "\tsigned:%u;\n",
			       SYSCALL_FIELD(int, nr),
			       SYSCALL_FIELD(long, ret));
	if (!ret)
		return 0;

	return trace_seq_printf(s, "\nprint fmt: \"0x%%lx\", REC->ret\n");
}

int syscall_enter_define_fields(struct ftrace_event_call *call)
{
	struct syscall_trace_enter trace;
	struct syscall_metadata *meta = call->data;
	int ret;
	int i;
	int offset = offsetof(typeof(trace), args);

	ret = trace_define_common_fields(call);
	if (ret)
		return ret;

	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	if (ret)
		return ret;

	for (i = 0; i < meta->nb_args; i++) {
		ret = trace_define_field(call, meta->types[i],
					 meta->args[i], offset,
					 sizeof(unsigned long), 0,
					 FILTER_OTHER);
		offset += sizeof(unsigned long);
	}

	return ret;
}

int syscall_exit_define_fields(struct ftrace_event_call *call)
{
	struct syscall_trace_exit trace;
	int ret;

	ret = trace_define_common_fields(call);
	if (ret)
		return ret;

	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	if (ret)
		return ret;

	ret = trace_define_field(call, SYSCALL_FIELD(long, ret),
				 FILTER_OTHER);

	return ret;
}

void ftrace_syscall_enter(struct pt_regs *regs, long id)
{
	struct syscall_trace_enter *entry;
	struct syscall_metadata *sys_data;
	struct ring_buffer_event *event;
	struct ring_buffer *buffer;
	int size;
	int syscall_nr;

	syscall_nr = syscall_get_nr(current, regs);
	if (syscall_nr < 0)
		return;
	if (!test_bit(syscall_nr, enabled_enter_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	size = sizeof(*entry) + sizeof(unsigned long) * sys_data->nb_args;

	event = trace_current_buffer_lock_reserve(&buffer,
			sys_data->enter_event->id, size, 0, 0);
	if (!event)
		return;

	entry = ring_buffer_event_data(event);
	entry->nr = syscall_nr;
	syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args);

	if (!filter_current_check_discard(buffer, sys_data->enter_event,
					  entry, event))
		trace_current_buffer_unlock_commit(buffer, event, 0, 0);
}

void ftrace_syscall_exit(struct pt_regs *regs, long ret)
{
	struct syscall_trace_exit *entry;
	struct syscall_metadata *sys_data;
	struct ring_buffer_event *event;
	struct ring_buffer *buffer;
	int syscall_nr;

	syscall_nr = syscall_get_nr(current, regs);
	if (syscall_nr < 0)
		return;
	if (!test_bit(syscall_nr, enabled_exit_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	event = trace_current_buffer_lock_reserve(&buffer,
			sys_data->exit_event->id, sizeof(*entry), 0, 0);
	if (!event)
		return;

	entry = ring_buffer_event_data(event);
	entry->nr = syscall_nr;
	entry->ret = syscall_get_return_value(current, regs);

	if (!filter_current_check_discard(buffer, sys_data->exit_event,
					  entry, event))
		trace_current_buffer_unlock_commit(buffer, event, 0, 0);
}

int reg_event_syscall_enter(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return -ENOSYS;
	mutex_lock(&syscall_trace_lock);
	if (!sys_refcount_enter)
		ret = register_trace_sys_enter(ftrace_syscall_enter);
	if (ret) {
		pr_info("event trace: Could not activate"
				"syscall entry trace point");
	} else {
		set_bit(num, enabled_enter_syscalls);
		sys_refcount_enter++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void unreg_event_syscall_enter(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return;
	mutex_lock(&syscall_trace_lock);
	sys_refcount_enter--;
	clear_bit(num, enabled_enter_syscalls);
	if (!sys_refcount_enter)
		unregister_trace_sys_enter(ftrace_syscall_enter);
	mutex_unlock(&syscall_trace_lock);
}

int reg_event_syscall_exit(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return -ENOSYS;
	mutex_lock(&syscall_trace_lock);
	if (!sys_refcount_exit)
		ret = register_trace_sys_exit(ftrace_syscall_exit);
	if (ret) {
		pr_info("event trace: Could not activate"
				"syscall exit trace point");
	} else {
		set_bit(num, enabled_exit_syscalls);
		sys_refcount_exit++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void unreg_event_syscall_exit(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return;
	mutex_lock(&syscall_trace_lock);
	sys_refcount_exit--;
	clear_bit(num, enabled_exit_syscalls);
	if (!sys_refcount_exit)
		unregister_trace_sys_exit(ftrace_syscall_exit);
	mutex_unlock(&syscall_trace_lock);
}

int init_syscall_trace(struct ftrace_event_call *call)
{
	int id;

	id = register_ftrace_event(call->event);
	if (!id)
		return -ENODEV;
	call->id = id;
	INIT_LIST_HEAD(&call->fields);
	return 0;
}

int __init init_ftrace_syscalls(void)
{
	struct syscall_metadata *meta;
	unsigned long addr;
	int i;

	syscalls_metadata = kzalloc(sizeof(*syscalls_metadata) *
					NR_syscalls, GFP_KERNEL);
	if (!syscalls_metadata) {
		WARN_ON(1);
		return -ENOMEM;
	}

	for (i = 0; i < NR_syscalls; i++) {
		addr = arch_syscall_addr(i);
		meta = find_syscall_meta(addr);
		if (!meta)
			continue;

		meta->syscall_nr = i;
		syscalls_metadata[i] = meta;
	}

	return 0;
}
core_initcall(init_ftrace_syscalls);

#ifdef CONFIG_EVENT_PROFILE

static DECLARE_BITMAP(enabled_prof_enter_syscalls, NR_syscalls);
static DECLARE_BITMAP(enabled_prof_exit_syscalls, NR_syscalls);
static int sys_prof_refcount_enter;
static int sys_prof_refcount_exit;

static void prof_syscall_enter(struct pt_regs *regs, long id)
{
	struct syscall_metadata *sys_data;
	struct syscall_trace_enter *rec;
	unsigned long flags;
	char *trace_buf;
	char *raw_data;
	int syscall_nr;
	int rctx;
	int size;
	int cpu;

	syscall_nr = syscall_get_nr(current, regs);
	if (!test_bit(syscall_nr, enabled_prof_enter_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	/* get the size after alignment with the u32 buffer size field */
	size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
	size = ALIGN(size + sizeof(u32), sizeof(u64));
	size -= sizeof(u32);

	if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
		      "profile buffer not large enough"))
		return;

	/* Protect the per cpu buffer, begin the rcu read side */
	local_irq_save(flags);

	rctx = perf_swevent_get_recursion_context();
	if (rctx < 0)
		goto end_recursion;

	cpu = smp_processor_id();

	trace_buf = rcu_dereference(perf_trace_buf);

	if (!trace_buf)
		goto end;

	raw_data = per_cpu_ptr(trace_buf, cpu);

	/* zero the dead bytes from align to not leak stack to user */
	*(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL;

	rec = (struct syscall_trace_enter *) raw_data;
	tracing_generic_entry_update(&rec->ent, 0, 0);
	rec->ent.type = sys_data->enter_event->id;
	rec->nr = syscall_nr;
	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
			       (unsigned long *)&rec->args);
	perf_tp_event(sys_data->enter_event->id, 0, 1, rec, size);

end:
	perf_swevent_put_recursion_context(rctx);
end_recursion:
	local_irq_restore(flags);
}

int reg_prof_syscall_enter(char *name)
{
	int ret = 0;
	int num;

	num = syscall_name_to_nr(name);
	if (num < 0 || num >= NR_syscalls)
		return -ENOSYS;

	mutex_lock(&syscall_trace_lock);
	if (!sys_prof_refcount_enter)
		ret = register_trace_sys_enter(prof_syscall_enter);
	if (ret) {
		pr_info("event trace: Could not activate"
				"syscall entry trace point");
	} else {
		set_bit(num, enabled_prof_enter_syscalls);
		sys_prof_refcount_enter++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void unreg_prof_syscall_enter(char *name)
{
	int num;

	num = syscall_name_to_nr(name);
	if (num < 0 || num >= NR_syscalls)
		return;

	mutex_lock(&syscall_trace_lock);
	sys_prof_refcount_enter--;
	clear_bit(num, enabled_prof_enter_syscalls);
	if (!sys_prof_refcount_enter)
		unregister_trace_sys_enter(prof_syscall_enter);
	mutex_unlock(&syscall_trace_lock);
}

static void prof_syscall_exit(struct pt_regs *regs, long ret)
{
	struct syscall_metadata *sys_data;
	struct syscall_trace_exit *rec;
	unsigned long flags;
	int syscall_nr;
	char *trace_buf;
	char *raw_data;
	int rctx;
	int size;
	int cpu;

	syscall_nr = syscall_get_nr(current, regs);
	if (!test_bit(syscall_nr, enabled_prof_exit_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	/* We can probably do that at build time */
	size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
	size -= sizeof(u32);

	/*
	 * Impossible, but be paranoid with the future
	 * How to put this check outside runtime?
	 */
	if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
		"exit event has grown above profile buffer size"))
		return;

	/* Protect the per cpu buffer, begin the rcu read side */
	local_irq_save(flags);

	rctx = perf_swevent_get_recursion_context();
	if (rctx < 0)
		goto end_recursion;

	cpu = smp_processor_id();

	trace_buf = rcu_dereference(perf_trace_buf);

	if (!trace_buf)
		goto end;

	raw_data = per_cpu_ptr(trace_buf, cpu);

	/* zero the dead bytes from align to not leak stack to user */
	*(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL;

	rec = (struct syscall_trace_exit *)raw_data;

	tracing_generic_entry_update(&rec->ent, 0, 0);
	rec->ent.type = sys_data->exit_event->id;
	rec->nr = syscall_nr;
	rec->ret = syscall_get_return_value(current, regs);

	perf_tp_event(sys_data->exit_event->id, 0, 1, rec, size);

end:
	perf_swevent_put_recursion_context(rctx);
end_recursion:
	local_irq_restore(flags);
}

int reg_prof_syscall_exit(char *name)
{
	int ret = 0;
	int num;

	num = syscall_name_to_nr(name);
	if (num < 0 || num >= NR_syscalls)
		return -ENOSYS;

	mutex_lock(&syscall_trace_lock);
	if (!sys_prof_refcount_exit)
		ret = register_trace_sys_exit(prof_syscall_exit);
	if (ret) {
		pr_info("event trace: Could not activate"
				"syscall entry trace point");
	} else {
		set_bit(num, enabled_prof_exit_syscalls);
		sys_prof_refcount_exit++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void unreg_prof_syscall_exit(char *name)
{
	int num;

	num = syscall_name_to_nr(name);
	if (num < 0 || num >= NR_syscalls)
		return;

	mutex_lock(&syscall_trace_lock);
	sys_prof_refcount_exit--;
	clear_bit(num, enabled_prof_exit_syscalls);
	if (!sys_prof_refcount_exit)
		unregister_trace_sys_exit(prof_syscall_exit);
	mutex_unlock(&syscall_trace_lock);
}

#endif
Commit	Line	Data
47788c58	1	#include <trace/syscall.h>
1c569f02	2	#include <trace/events/syscalls.h>
ee08c6ec	3	#include <linux/kernel.h>
fb34a08c	4	#include <linux/ftrace.h>
cdd6c482	5	#include <linux/perf_event.h>
ee08c6ec FW	6	#include <asm/syscall.h>
	7
	8	#include "trace_output.h"
	9	#include "trace.h"
	10
5be71b61	11	static DEFINE_MUTEX(syscall_trace_lock);
fb34a08c JB	12	static int sys_refcount_enter;
fb34a08c JB	13	static int sys_refcount_exit;
57421dbb JB	14	static DECLARE_BITMAP(enabled_enter_syscalls, NR_syscalls);
57421dbb JB	15	static DECLARE_BITMAP(enabled_exit_syscalls, NR_syscalls);
ee08c6ec	16
c44fc770 FW	17	extern unsigned long __start_syscalls_metadata[];
	18	extern unsigned long __stop_syscalls_metadata[];
	19
	20	static struct syscall_metadata **syscalls_metadata;
	21
	22	static struct syscall_metadata *find_syscall_meta(unsigned long syscall)
	23	{
	24	struct syscall_metadata *start;
	25	struct syscall_metadata *stop;
	26	char str[KSYM_SYMBOL_LEN];
	27
	28
	29	start = (struct syscall_metadata *)__start_syscalls_metadata;
	30	stop = (struct syscall_metadata *)__stop_syscalls_metadata;
	31	kallsyms_lookup(syscall, NULL, NULL, NULL, str);
	32
	33	for ( ; start < stop; start++) {
	34	/*
	35	* Only compare after the "sys" prefix. Archs that use
	36	* syscall wrappers may have syscalls symbols aliases prefixed
	37	* with "SyS" instead of "sys", leading to an unwanted
	38	* mismatch.
	39	*/
	40	if (start->name && !strcmp(start->name + 3, str + 3))
	41	return start;
	42	}
	43	return NULL;
	44	}
	45
	46	static struct syscall_metadata *syscall_nr_to_meta(int nr)
	47	{
	48	if (!syscalls_metadata \|\| nr >= NR_syscalls \|\| nr < 0)
	49	return NULL;
	50
	51	return syscalls_metadata[nr];
	52	}
	53
c252f657	54	static int syscall_name_to_nr(const char *name)
c44fc770 FW	55	{
	56	int i;
	57
	58	if (!syscalls_metadata)
	59	return -1;
	60
	61	for (i = 0; i < NR_syscalls; i++) {
	62	if (syscalls_metadata[i]) {
	63	if (!strcmp(syscalls_metadata[i]->name, name))
	64	return i;
	65	}
	66	}
	67	return -1;
	68	}
	69
bed1ffca FW	70	enum print_line_t
	71	print_syscall_enter(struct trace_iterator *iter, int flags)
	72	{
	73	struct trace_seq *s = &iter->seq;
	74	struct trace_entry *ent = iter->ent;
	75	struct syscall_trace_enter *trace;
	76	struct syscall_metadata *entry;
	77	int i, ret, syscall;
	78
64c12e04	79	trace = (typeof(trace))ent;
bed1ffca	80	syscall = trace->nr;
bed1ffca	81	entry = syscall_nr_to_meta(syscall);
64c12e04	82
bed1ffca FW	83	if (!entry)
	84	goto end;
	85
fcc19438	86	if (entry->enter_event->id != ent->type) {
64c12e04 JB	87	WARN_ON_ONCE(1);
	88	goto end;
	89	}
	90
bed1ffca FW	91	ret = trace_seq_printf(s, "%s(", entry->name);
	92	if (!ret)
	93	return TRACE_TYPE_PARTIAL_LINE;
	94
	95	for (i = 0; i < entry->nb_args; i++) {
	96	/* parameter types */
ba8b3a40	97	if (trace_flags & TRACE_ITER_VERBOSE) {
bed1ffca FW	98	ret = trace_seq_printf(s, "%s ", entry->types[i]);
	99	if (!ret)
	100	return TRACE_TYPE_PARTIAL_LINE;
	101	}
	102	/* parameter values */
4539f077	103	ret = trace_seq_printf(s, "%s: %lx%s", entry->args[i],
bed1ffca	104	trace->args[i],
4539f077	105	i == entry->nb_args - 1 ? "" : ", ");
bed1ffca FW	106	if (!ret)
	107	return TRACE_TYPE_PARTIAL_LINE;
	108	}
	109
4539f077 LZ	110	ret = trace_seq_putc(s, ')');
	111	if (!ret)
	112	return TRACE_TYPE_PARTIAL_LINE;
	113
bed1ffca	114	end:
4539f077 LZ	115	ret = trace_seq_putc(s, '\n');
	116	if (!ret)
	117	return TRACE_TYPE_PARTIAL_LINE;
	118
bed1ffca FW	119	return TRACE_TYPE_HANDLED;
	120	}
	121
	122	enum print_line_t
	123	print_syscall_exit(struct trace_iterator *iter, int flags)
	124	{
	125	struct trace_seq *s = &iter->seq;
	126	struct trace_entry *ent = iter->ent;
	127	struct syscall_trace_exit *trace;
	128	int syscall;
	129	struct syscall_metadata *entry;
	130	int ret;
	131
64c12e04	132	trace = (typeof(trace))ent;
bed1ffca	133	syscall = trace->nr;
bed1ffca	134	entry = syscall_nr_to_meta(syscall);
64c12e04	135
bed1ffca FW	136	if (!entry) {
	137	trace_seq_printf(s, "\n");
	138	return TRACE_TYPE_HANDLED;
	139	}
	140
fcc19438	141	if (entry->exit_event->id != ent->type) {
64c12e04 JB	142	WARN_ON_ONCE(1);
	143	return TRACE_TYPE_UNHANDLED;
	144	}
	145
bed1ffca FW	146	ret = trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,
	147	trace->ret);
	148	if (!ret)
	149	return TRACE_TYPE_PARTIAL_LINE;
	150
	151	return TRACE_TYPE_HANDLED;
	152	}
	153
e6971969 LZ	154	extern char *__bad_type_size(void);
	155
	156	#define SYSCALL_FIELD(type, name) \
	157	sizeof(type) != sizeof(trace.name) ? \
	158	__bad_type_size() : \
26a50744 TZ	159	#type, #name, offsetof(typeof(trace), name), \
26a50744 TZ	160	sizeof(trace.name), is_signed_type(type)
e6971969	161
10a5b66f	162	int syscall_enter_format(struct ftrace_event_call call, struct trace_seq s)
dc4ddb4c FW	163	{
dc4ddb4c FW	164	int i;
e6971969	165	int ret;
31c16b13	166	struct syscall_metadata *entry = call->data;
e6971969 LZ	167	struct syscall_trace_enter trace;
e6971969 LZ	168	int offset = offsetof(struct syscall_trace_enter, args);
dc4ddb4c	169
26a50744 TZ	170	ret = trace_seq_printf(s, "\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
26a50744 TZ	171	"\tsigned:%u;\n",
e6971969 LZ	172	SYSCALL_FIELD(int, nr));
	173	if (!ret)
	174	return 0;
dc4ddb4c FW	175
	176	for (i = 0; i < entry->nb_args; i++) {
	177	ret = trace_seq_printf(s, "\tfield:%s %s;", entry->types[i],
	178	entry->args[i]);
	179	if (!ret)
	180	return 0;
26a50744 TZ	181	ret = trace_seq_printf(s, "\toffset:%d;\tsize:%zu;"
	182	"\tsigned:%u;\n", offset,
	183	sizeof(unsigned long),
	184	is_signed_type(unsigned long));
dc4ddb4c FW	185	if (!ret)
	186	return 0;
	187	offset += sizeof(unsigned long);
	188	}
	189
4539f077	190	trace_seq_puts(s, "\nprint fmt: \"");
dc4ddb4c	191	for (i = 0; i < entry->nb_args; i++) {
e6971969	192	ret = trace_seq_printf(s, "%s: 0x%%0%zulx%s", entry->args[i],
dc4ddb4c	193	sizeof(unsigned long),
4539f077	194	i == entry->nb_args - 1 ? "" : ", ");
dc4ddb4c FW	195	if (!ret)
	196	return 0;
	197	}
4539f077	198	trace_seq_putc(s, '"');
dc4ddb4c FW	199
dc4ddb4c FW	200	for (i = 0; i < entry->nb_args; i++) {
4539f077 LZ	201	ret = trace_seq_printf(s, ", ((unsigned long)(REC->%s))",
4539f077 LZ	202	entry->args[i]);
dc4ddb4c FW	203	if (!ret)
	204	return 0;
	205	}
	206
4539f077	207	return trace_seq_putc(s, '\n');
dc4ddb4c FW	208	}
dc4ddb4c FW	209
10a5b66f LZ	210	int syscall_exit_format(struct ftrace_event_call call, struct trace_seq s)
	211	{
	212	int ret;
	213	struct syscall_trace_exit trace;
	214
	215	ret = trace_seq_printf(s,
26a50744 TZ	216	"\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
	217	"\tsigned:%u;\n"
	218	"\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
	219	"\tsigned:%u;\n",
10a5b66f	220	SYSCALL_FIELD(int, nr),
ee949a86	221	SYSCALL_FIELD(long, ret));
10a5b66f LZ	222	if (!ret)
	223	return 0;
	224
	225	return trace_seq_printf(s, "\nprint fmt: \"0x%%lx\", REC->ret\n");
	226	}
	227
540b7b8d LZ	228	int syscall_enter_define_fields(struct ftrace_event_call *call)
	229	{
	230	struct syscall_trace_enter trace;
31c16b13	231	struct syscall_metadata *meta = call->data;
540b7b8d	232	int ret;
540b7b8d LZ	233	int i;
	234	int offset = offsetof(typeof(trace), args);
	235
540b7b8d LZ	236	ret = trace_define_common_fields(call);
	237	if (ret)
	238	return ret;
	239
0f1ef51d LJ	240	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	241	if (ret)
	242	return ret;
	243
540b7b8d	244	for (i = 0; i < meta->nb_args; i++) {
aeaeae11 FW	245	ret = trace_define_field(call, meta->types[i],
aeaeae11 FW	246	meta->args[i], offset,
43b51ead LZ	247	sizeof(unsigned long), 0,
43b51ead LZ	248	FILTER_OTHER);
540b7b8d LZ	249	offset += sizeof(unsigned long);
	250	}
	251
	252	return ret;
	253	}
	254
	255	int syscall_exit_define_fields(struct ftrace_event_call *call)
	256	{
	257	struct syscall_trace_exit trace;
	258	int ret;
	259
	260	ret = trace_define_common_fields(call);
	261	if (ret)
	262	return ret;
	263
0f1ef51d LJ	264	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	265	if (ret)
	266	return ret;
	267
26a50744	268	ret = trace_define_field(call, SYSCALL_FIELD(long, ret),
43b51ead	269	FILTER_OTHER);
540b7b8d LZ	270
	271	return ret;
	272	}
	273
fb34a08c	274	void ftrace_syscall_enter(struct pt_regs *regs, long id)
ee08c6ec	275	{
bed1ffca FW	276	struct syscall_trace_enter *entry;
	277	struct syscall_metadata *sys_data;
	278	struct ring_buffer_event *event;
e77405ad	279	struct ring_buffer *buffer;
bed1ffca	280	int size;
ee08c6ec FW	281	int syscall_nr;
	282
	283	syscall_nr = syscall_get_nr(current, regs);
cd0980fc HB	284	if (syscall_nr < 0)
cd0980fc HB	285	return;
fb34a08c JB	286	if (!test_bit(syscall_nr, enabled_enter_syscalls))
fb34a08c JB	287	return;
ee08c6ec	288
bed1ffca FW	289	sys_data = syscall_nr_to_meta(syscall_nr);
	290	if (!sys_data)
	291	return;
	292
	293	size = sizeof(entry) + sizeof(unsigned long) sys_data->nb_args;
	294
fcc19438 LJ	295	event = trace_current_buffer_lock_reserve(&buffer,
fcc19438 LJ	296	sys_data->enter_event->id, size, 0, 0);
bed1ffca FW	297	if (!event)
	298	return;
	299
	300	entry = ring_buffer_event_data(event);
	301	entry->nr = syscall_nr;
	302	syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args);
	303
e77405ad SR	304	if (!filter_current_check_discard(buffer, sys_data->enter_event,
	305	entry, event))
	306	trace_current_buffer_unlock_commit(buffer, event, 0, 0);
ee08c6ec FW	307	}
ee08c6ec FW	308
fb34a08c	309	void ftrace_syscall_exit(struct pt_regs *regs, long ret)
ee08c6ec	310	{
bed1ffca FW	311	struct syscall_trace_exit *entry;
	312	struct syscall_metadata *sys_data;
	313	struct ring_buffer_event *event;
e77405ad	314	struct ring_buffer *buffer;
ee08c6ec FW	315	int syscall_nr;
	316
	317	syscall_nr = syscall_get_nr(current, regs);
cd0980fc HB	318	if (syscall_nr < 0)
cd0980fc HB	319	return;
fb34a08c JB	320	if (!test_bit(syscall_nr, enabled_exit_syscalls))
fb34a08c JB	321	return;
ee08c6ec	322
bed1ffca FW	323	sys_data = syscall_nr_to_meta(syscall_nr);
	324	if (!sys_data)
	325	return;
	326
fcc19438 LJ	327	event = trace_current_buffer_lock_reserve(&buffer,
fcc19438 LJ	328	sys_data->exit_event->id, sizeof(*entry), 0, 0);
bed1ffca FW	329	if (!event)
	330	return;
	331
	332	entry = ring_buffer_event_data(event);
	333	entry->nr = syscall_nr;
	334	entry->ret = syscall_get_return_value(current, regs);
	335
e77405ad SR	336	if (!filter_current_check_discard(buffer, sys_data->exit_event,
	337	entry, event))
	338	trace_current_buffer_unlock_commit(buffer, event, 0, 0);
ee08c6ec FW	339	}
ee08c6ec FW	340
bd1a5c84	341	int reg_event_syscall_enter(struct ftrace_event_call *call)
ee08c6ec	342	{
fb34a08c JB	343	int ret = 0;
fb34a08c JB	344	int num;
fb34a08c	345
c252f657	346	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	347	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	348	return -ENOSYS;
	349	mutex_lock(&syscall_trace_lock);
	350	if (!sys_refcount_enter)
1c569f02	351	ret = register_trace_sys_enter(ftrace_syscall_enter);
fb34a08c JB	352	if (ret) {
	353	pr_info("event trace: Could not activate"
	354	"syscall entry trace point");
	355	} else {
	356	set_bit(num, enabled_enter_syscalls);
	357	sys_refcount_enter++;
	358	}
	359	mutex_unlock(&syscall_trace_lock);
	360	return ret;
ee08c6ec FW	361	}
ee08c6ec FW	362
bd1a5c84	363	void unreg_event_syscall_enter(struct ftrace_event_call *call)
ee08c6ec	364	{
fb34a08c	365	int num;
ee08c6ec	366
c252f657	367	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	368	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	369	return;
	370	mutex_lock(&syscall_trace_lock);
	371	sys_refcount_enter--;
	372	clear_bit(num, enabled_enter_syscalls);
	373	if (!sys_refcount_enter)
1c569f02	374	unregister_trace_sys_enter(ftrace_syscall_enter);
fb34a08c JB	375	mutex_unlock(&syscall_trace_lock);
fb34a08c JB	376	}
ee08c6ec	377
bd1a5c84	378	int reg_event_syscall_exit(struct ftrace_event_call *call)
ee08c6ec	379	{
fb34a08c JB	380	int ret = 0;
fb34a08c JB	381	int num;
fb34a08c	382
c252f657	383	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	384	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	385	return -ENOSYS;
	386	mutex_lock(&syscall_trace_lock);
	387	if (!sys_refcount_exit)
1c569f02	388	ret = register_trace_sys_exit(ftrace_syscall_exit);
fb34a08c JB	389	if (ret) {
	390	pr_info("event trace: Could not activate"
	391	"syscall exit trace point");
	392	} else {
	393	set_bit(num, enabled_exit_syscalls);
	394	sys_refcount_exit++;
ee08c6ec	395	}
fb34a08c JB	396	mutex_unlock(&syscall_trace_lock);
	397	return ret;
	398	}
ee08c6ec	399
bd1a5c84	400	void unreg_event_syscall_exit(struct ftrace_event_call *call)
fb34a08c JB	401	{
fb34a08c JB	402	int num;
ee08c6ec	403
c252f657	404	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	405	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	406	return;
	407	mutex_lock(&syscall_trace_lock);
	408	sys_refcount_exit--;
	409	clear_bit(num, enabled_exit_syscalls);
	410	if (!sys_refcount_exit)
1c569f02	411	unregister_trace_sys_exit(ftrace_syscall_exit);
fb34a08c	412	mutex_unlock(&syscall_trace_lock);
ee08c6ec	413	}
fb34a08c	414
a1301da0 LJ	415	int init_syscall_trace(struct ftrace_event_call *call)
	416	{
	417	int id;
	418
	419	id = register_ftrace_event(call->event);
	420	if (!id)
	421	return -ENODEV;
	422	call->id = id;
	423	INIT_LIST_HEAD(&call->fields);
	424	return 0;
	425	}
	426
c44fc770 FW	427	int __init init_ftrace_syscalls(void)
	428	{
	429	struct syscall_metadata *meta;
	430	unsigned long addr;
	431	int i;
	432
	433	syscalls_metadata = kzalloc(sizeof(syscalls_metadata)
	434	NR_syscalls, GFP_KERNEL);
	435	if (!syscalls_metadata) {
	436	WARN_ON(1);
	437	return -ENOMEM;
	438	}
	439
	440	for (i = 0; i < NR_syscalls; i++) {
	441	addr = arch_syscall_addr(i);
	442	meta = find_syscall_meta(addr);
c252f657 LJ	443	if (!meta)
	444	continue;
	445
	446	meta->syscall_nr = i;
c44fc770 FW	447	syscalls_metadata[i] = meta;
	448	}
	449
	450	return 0;
	451	}
	452	core_initcall(init_ftrace_syscalls);
	453
f4b5ffcc	454	#ifdef CONFIG_EVENT_PROFILE
19007a67	455
57421dbb JB	456	static DECLARE_BITMAP(enabled_prof_enter_syscalls, NR_syscalls);
57421dbb JB	457	static DECLARE_BITMAP(enabled_prof_exit_syscalls, NR_syscalls);
f4b5ffcc JB	458	static int sys_prof_refcount_enter;
	459	static int sys_prof_refcount_exit;
	460
	461	static void prof_syscall_enter(struct pt_regs *regs, long id)
	462	{
	463	struct syscall_metadata *sys_data;
20ab4425 FW	464	struct syscall_trace_enter *rec;
20ab4425 FW	465	unsigned long flags;
ce71b9df	466	char *trace_buf;
20ab4425	467	char *raw_data;
f4b5ffcc	468	int syscall_nr;
4ed7c92d	469	int rctx;
19007a67	470	int size;
20ab4425	471	int cpu;
f4b5ffcc JB	472
	473	syscall_nr = syscall_get_nr(current, regs);
	474	if (!test_bit(syscall_nr, enabled_prof_enter_syscalls))
	475	return;
	476
	477	sys_data = syscall_nr_to_meta(syscall_nr);
	478	if (!sys_data)
	479	return;
	480
19007a67 FW	481	/* get the size after alignment with the u32 buffer size field */
	482	size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
	483	size = ALIGN(size + sizeof(u32), sizeof(u64));
	484	size -= sizeof(u32);
	485
20ab4425 FW	486	if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
	487	"profile buffer not large enough"))
	488	return;
	489
	490	/* Protect the per cpu buffer, begin the rcu read side */
	491	local_irq_save(flags);
19007a67	492
4ed7c92d PZ	493	rctx = perf_swevent_get_recursion_context();
4ed7c92d PZ	494	if (rctx < 0)
ce71b9df FW	495	goto end_recursion;
ce71b9df FW	496
20ab4425 FW	497	cpu = smp_processor_id();
20ab4425 FW	498
28889bf9	499	trace_buf = rcu_dereference(perf_trace_buf);
20ab4425	500
444a2a3b	501	if (!trace_buf)
20ab4425	502	goto end;
19007a67	503
ce71b9df	504	raw_data = per_cpu_ptr(trace_buf, cpu);
20ab4425 FW	505
	506	/* zero the dead bytes from align to not leak stack to user */
	507	(u64 )(&raw_data[size - sizeof(u64)]) = 0ULL;
	508
	509	rec = (struct syscall_trace_enter *) raw_data;
	510	tracing_generic_entry_update(&rec->ent, 0, 0);
fcc19438	511	rec->ent.type = sys_data->enter_event->id;
20ab4425 FW	512	rec->nr = syscall_nr;
	513	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
	514	(unsigned long *)&rec->args);
fcc19438	515	perf_tp_event(sys_data->enter_event->id, 0, 1, rec, size);
20ab4425 FW	516
20ab4425 FW	517	end:
4ed7c92d	518	perf_swevent_put_recursion_context(rctx);
ce71b9df	519	end_recursion:
20ab4425	520	local_irq_restore(flags);
f4b5ffcc JB	521	}
	522
	523	int reg_prof_syscall_enter(char *name)
	524	{
	525	int ret = 0;
	526	int num;
	527
	528	num = syscall_name_to_nr(name);
57421dbb	529	if (num < 0 \|\| num >= NR_syscalls)
f4b5ffcc JB	530	return -ENOSYS;
	531
	532	mutex_lock(&syscall_trace_lock);
	533	if (!sys_prof_refcount_enter)
1c569f02	534	ret = register_trace_sys_enter(prof_syscall_enter);
f4b5ffcc JB	535	if (ret) {
	536	pr_info("event trace: Could not activate"
	537	"syscall entry trace point");
	538	} else {
	539	set_bit(num, enabled_prof_enter_syscalls);
	540	sys_prof_refcount_enter++;
	541	}
	542	mutex_unlock(&syscall_trace_lock);
	543	return ret;
	544	}
	545
	546	void unreg_prof_syscall_enter(char *name)
	547	{
	548	int num;
	549
	550	num = syscall_name_to_nr(name);
57421dbb	551	if (num < 0 \|\| num >= NR_syscalls)
f4b5ffcc JB	552	return;
	553
	554	mutex_lock(&syscall_trace_lock);
	555	sys_prof_refcount_enter--;
	556	clear_bit(num, enabled_prof_enter_syscalls);
	557	if (!sys_prof_refcount_enter)
1c569f02	558	unregister_trace_sys_enter(prof_syscall_enter);
f4b5ffcc JB	559	mutex_unlock(&syscall_trace_lock);
	560	}
	561
	562	static void prof_syscall_exit(struct pt_regs *regs, long ret)
	563	{
	564	struct syscall_metadata *sys_data;
20ab4425 FW	565	struct syscall_trace_exit *rec;
20ab4425 FW	566	unsigned long flags;
f4b5ffcc	567	int syscall_nr;
ce71b9df	568	char *trace_buf;
20ab4425	569	char *raw_data;
4ed7c92d	570	int rctx;
20ab4425 FW	571	int size;
20ab4425 FW	572	int cpu;
f4b5ffcc JB	573
	574	syscall_nr = syscall_get_nr(current, regs);
	575	if (!test_bit(syscall_nr, enabled_prof_exit_syscalls))
	576	return;
	577
	578	sys_data = syscall_nr_to_meta(syscall_nr);
	579	if (!sys_data)
	580	return;
	581
20ab4425 FW	582	/* We can probably do that at build time */
	583	size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
	584	size -= sizeof(u32);
19007a67	585
20ab4425 FW	586	/*
	587	* Impossible, but be paranoid with the future
	588	* How to put this check outside runtime?
	589	*/
	590	if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
	591	"exit event has grown above profile buffer size"))
	592	return;
	593
	594	/* Protect the per cpu buffer, begin the rcu read side */
	595	local_irq_save(flags);
ce71b9df	596
4ed7c92d PZ	597	rctx = perf_swevent_get_recursion_context();
4ed7c92d PZ	598	if (rctx < 0)
ce71b9df FW	599	goto end_recursion;
ce71b9df FW	600
20ab4425 FW	601	cpu = smp_processor_id();
20ab4425 FW	602
28889bf9	603	trace_buf = rcu_dereference(perf_trace_buf);
20ab4425	604
444a2a3b	605	if (!trace_buf)
20ab4425 FW	606	goto end;
20ab4425 FW	607
ce71b9df	608	raw_data = per_cpu_ptr(trace_buf, cpu);
20ab4425 FW	609
	610	/* zero the dead bytes from align to not leak stack to user */
	611	(u64 )(&raw_data[size - sizeof(u64)]) = 0ULL;
	612
	613	rec = (struct syscall_trace_exit *)raw_data;
	614
	615	tracing_generic_entry_update(&rec->ent, 0, 0);
fcc19438	616	rec->ent.type = sys_data->exit_event->id;
20ab4425 FW	617	rec->nr = syscall_nr;
	618	rec->ret = syscall_get_return_value(current, regs);
	619
fcc19438	620	perf_tp_event(sys_data->exit_event->id, 0, 1, rec, size);
20ab4425 FW	621
20ab4425 FW	622	end:
4ed7c92d	623	perf_swevent_put_recursion_context(rctx);
ce71b9df	624	end_recursion:
20ab4425	625	local_irq_restore(flags);
f4b5ffcc JB	626	}
	627
	628	int reg_prof_syscall_exit(char *name)
	629	{
	630	int ret = 0;
	631	int num;
	632
	633	num = syscall_name_to_nr(name);
57421dbb	634	if (num < 0 \|\| num >= NR_syscalls)
f4b5ffcc JB	635	return -ENOSYS;
	636
	637	mutex_lock(&syscall_trace_lock);
	638	if (!sys_prof_refcount_exit)
1c569f02	639	ret = register_trace_sys_exit(prof_syscall_exit);
f4b5ffcc JB	640	if (ret) {
	641	pr_info("event trace: Could not activate"
	642	"syscall entry trace point");
	643	} else {
	644	set_bit(num, enabled_prof_exit_syscalls);
	645	sys_prof_refcount_exit++;
	646	}
	647	mutex_unlock(&syscall_trace_lock);
	648	return ret;
	649	}
	650
	651	void unreg_prof_syscall_exit(char *name)
	652	{
	653	int num;
	654
	655	num = syscall_name_to_nr(name);
57421dbb	656	if (num < 0 \|\| num >= NR_syscalls)
f4b5ffcc JB	657	return;
	658
	659	mutex_lock(&syscall_trace_lock);
	660	sys_prof_refcount_exit--;
	661	clear_bit(num, enabled_prof_exit_syscalls);
	662	if (!sys_prof_refcount_exit)
1c569f02	663	unregister_trace_sys_exit(prof_syscall_exit);
f4b5ffcc JB	664	mutex_unlock(&syscall_trace_lock);
	665	}
	666
	667	#endif
	668
	669