[net-next-2.6.git] / kernel / trace / kmemtrace.c

/*
 * Memory allocator tracing
 *
 * Copyright (C) 2008 Eduard - Gabriel Munteanu
 * Copyright (C) 2008 Pekka Enberg <penberg@cs.helsinki.fi>
 * Copyright (C) 2008 Frederic Weisbecker <fweisbec@gmail.com>
 */

#include <linux/tracepoint.h>
#include <linux/seq_file.h>
#include <linux/debugfs.h>
#include <linux/dcache.h>
#include <linux/fs.h>

#include <linux/kmemtrace.h>

#include "trace_output.h"
#include "trace.h"

/* Select an alternative, minimalistic output than the original one */
#define TRACE_KMEM_OPT_MINIMAL	0x1

static struct tracer_opt kmem_opts[] = {
	/* Default disable the minimalistic output */
	{ TRACER_OPT(kmem_minimalistic, TRACE_KMEM_OPT_MINIMAL) },
	{ }
};

static struct tracer_flags kmem_tracer_flags = {
	.val			= 0,
	.opts			= kmem_opts
};

static struct trace_array *kmemtrace_array;

/* Trace allocations */
static inline void kmemtrace_alloc(enum kmemtrace_type_id type_id,
				   unsigned long call_site,
				   const void *ptr,
				   size_t bytes_req,
				   size_t bytes_alloc,
				   gfp_t gfp_flags,
				   int node)
{
	struct ftrace_event_call *call = &event_kmem_alloc;
	struct trace_array *tr = kmemtrace_array;
	struct kmemtrace_alloc_entry *entry;
	struct ring_buffer_event *event;

	event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry));
	if (!event)
		return;

	entry = ring_buffer_event_data(event);
	tracing_generic_entry_update(&entry->ent, 0, 0);

	entry->ent.type		= TRACE_KMEM_ALLOC;
	entry->type_id		= type_id;
	entry->call_site	= call_site;
	entry->ptr		= ptr;
	entry->bytes_req	= bytes_req;
	entry->bytes_alloc	= bytes_alloc;
	entry->gfp_flags	= gfp_flags;
	entry->node		= node;

	if (!filter_check_discard(call, entry, tr->buffer, event))
		ring_buffer_unlock_commit(tr->buffer, event);

	trace_wake_up();
}

static inline void kmemtrace_free(enum kmemtrace_type_id type_id,
				  unsigned long call_site,
				  const void *ptr)
{
	struct ftrace_event_call *call = &event_kmem_free;
	struct trace_array *tr = kmemtrace_array;
	struct kmemtrace_free_entry *entry;
	struct ring_buffer_event *event;

	event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry));
	if (!event)
		return;
	entry	= ring_buffer_event_data(event);
	tracing_generic_entry_update(&entry->ent, 0, 0);

	entry->ent.type		= TRACE_KMEM_FREE;
	entry->type_id		= type_id;
	entry->call_site	= call_site;
	entry->ptr		= ptr;

	if (!filter_check_discard(call, entry, tr->buffer, event))
		ring_buffer_unlock_commit(tr->buffer, event);

	trace_wake_up();
}

static void kmemtrace_kmalloc(void *ignore,
			      unsigned long call_site,
			      const void *ptr,
			      size_t bytes_req,
			      size_t bytes_alloc,
			      gfp_t gfp_flags)
{
	kmemtrace_alloc(KMEMTRACE_TYPE_KMALLOC, call_site, ptr,
			bytes_req, bytes_alloc, gfp_flags, -1);
}

static void kmemtrace_kmem_cache_alloc(void *ignore,
				       unsigned long call_site,
				       const void *ptr,
				       size_t bytes_req,
				       size_t bytes_alloc,
				       gfp_t gfp_flags)
{
	kmemtrace_alloc(KMEMTRACE_TYPE_CACHE, call_site, ptr,
			bytes_req, bytes_alloc, gfp_flags, -1);
}

static void kmemtrace_kmalloc_node(void *ignore,
				   unsigned long call_site,
				   const void *ptr,
				   size_t bytes_req,
				   size_t bytes_alloc,
				   gfp_t gfp_flags,
				   int node)
{
	kmemtrace_alloc(KMEMTRACE_TYPE_KMALLOC, call_site, ptr,
			bytes_req, bytes_alloc, gfp_flags, node);
}

static void kmemtrace_kmem_cache_alloc_node(void *ignore,
					    unsigned long call_site,
					    const void *ptr,
					    size_t bytes_req,
					    size_t bytes_alloc,
					    gfp_t gfp_flags,
					    int node)
{
	kmemtrace_alloc(KMEMTRACE_TYPE_CACHE, call_site, ptr,
			bytes_req, bytes_alloc, gfp_flags, node);
}

static void
kmemtrace_kfree(void *ignore, unsigned long call_site, const void *ptr)
{
	kmemtrace_free(KMEMTRACE_TYPE_KMALLOC, call_site, ptr);
}

static void kmemtrace_kmem_cache_free(void *ignore,
				      unsigned long call_site, const void *ptr)
{
	kmemtrace_free(KMEMTRACE_TYPE_CACHE, call_site, ptr);
}

static int kmemtrace_start_probes(void)
{
	int err;

	err = register_trace_kmalloc(kmemtrace_kmalloc, NULL);
	if (err)
		return err;
	err = register_trace_kmem_cache_alloc(kmemtrace_kmem_cache_alloc, NULL);
	if (err)
		return err;
	err = register_trace_kmalloc_node(kmemtrace_kmalloc_node, NULL);
	if (err)
		return err;
	err = register_trace_kmem_cache_alloc_node(kmemtrace_kmem_cache_alloc_node, NULL);
	if (err)
		return err;
	err = register_trace_kfree(kmemtrace_kfree, NULL);
	if (err)
		return err;
	err = register_trace_kmem_cache_free(kmemtrace_kmem_cache_free, NULL);

	return err;
}

static void kmemtrace_stop_probes(void)
{
	unregister_trace_kmalloc(kmemtrace_kmalloc, NULL);
	unregister_trace_kmem_cache_alloc(kmemtrace_kmem_cache_alloc, NULL);
	unregister_trace_kmalloc_node(kmemtrace_kmalloc_node, NULL);
	unregister_trace_kmem_cache_alloc_node(kmemtrace_kmem_cache_alloc_node, NULL);
	unregister_trace_kfree(kmemtrace_kfree, NULL);
	unregister_trace_kmem_cache_free(kmemtrace_kmem_cache_free, NULL);
}

static int kmem_trace_init(struct trace_array *tr)
{
	kmemtrace_array = tr;

	tracing_reset_online_cpus(tr);

	kmemtrace_start_probes();

	return 0;
}

static void kmem_trace_reset(struct trace_array *tr)
{
	kmemtrace_stop_probes();
}

static void kmemtrace_headers(struct seq_file *s)
{
	/* Don't need headers for the original kmemtrace output */
	if (!(kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL))
		return;

	seq_printf(s, "#\n");
	seq_printf(s, "# ALLOC  TYPE  REQ   GIVEN  FLAGS     "
			"      POINTER         NODE    CALLER\n");
	seq_printf(s, "# FREE   |      |     |       |       "
			"       |   |            |        |\n");
	seq_printf(s, "# |\n\n");
}

/*
 * The following functions give the original output from kmemtrace,
 * plus the origin CPU, since reordering occurs in-kernel now.
 */

#define KMEMTRACE_USER_ALLOC	0
#define KMEMTRACE_USER_FREE	1

struct kmemtrace_user_event {
	u8			event_id;
	u8			type_id;
	u16			event_size;
	u32			cpu;
	u64			timestamp;
	unsigned long		call_site;
	unsigned long		ptr;
};

struct kmemtrace_user_event_alloc {
	size_t			bytes_req;
	size_t			bytes_alloc;
	unsigned		gfp_flags;
	int			node;
};

static enum print_line_t
kmemtrace_print_alloc(struct trace_iterator *iter, int flags,
		      struct trace_event *event)
{
	struct trace_seq *s = &iter->seq;
	struct kmemtrace_alloc_entry *entry;
	int ret;

	trace_assign_type(entry, iter->ent);

	ret = trace_seq_printf(s, "type_id %d call_site %pF ptr %lu "
	    "bytes_req %lu bytes_alloc %lu gfp_flags %lu node %d\n",
	    entry->type_id, (void *)entry->call_site, (unsigned long)entry->ptr,
	    (unsigned long)entry->bytes_req, (unsigned long)entry->bytes_alloc,
	    (unsigned long)entry->gfp_flags, entry->node);

	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;
	return TRACE_TYPE_HANDLED;
}

static enum print_line_t
kmemtrace_print_free(struct trace_iterator *iter, int flags,
		     struct trace_event *event)
{
	struct trace_seq *s = &iter->seq;
	struct kmemtrace_free_entry *entry;
	int ret;

	trace_assign_type(entry, iter->ent);

	ret = trace_seq_printf(s, "type_id %d call_site %pF ptr %lu\n",
			       entry->type_id, (void *)entry->call_site,
			       (unsigned long)entry->ptr);

	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;
	return TRACE_TYPE_HANDLED;
}

static enum print_line_t
kmemtrace_print_alloc_user(struct trace_iterator *iter, int flags,
			   struct trace_event *event)
{
	struct trace_seq *s = &iter->seq;
	struct kmemtrace_alloc_entry *entry;
	struct kmemtrace_user_event *ev;
	struct kmemtrace_user_event_alloc *ev_alloc;

	trace_assign_type(entry, iter->ent);

	ev = trace_seq_reserve(s, sizeof(*ev));
	if (!ev)
		return TRACE_TYPE_PARTIAL_LINE;

	ev->event_id		= KMEMTRACE_USER_ALLOC;
	ev->type_id		= entry->type_id;
	ev->event_size		= sizeof(*ev) + sizeof(*ev_alloc);
	ev->cpu			= iter->cpu;
	ev->timestamp		= iter->ts;
	ev->call_site		= entry->call_site;
	ev->ptr			= (unsigned long)entry->ptr;

	ev_alloc = trace_seq_reserve(s, sizeof(*ev_alloc));
	if (!ev_alloc)
		return TRACE_TYPE_PARTIAL_LINE;

	ev_alloc->bytes_req	= entry->bytes_req;
	ev_alloc->bytes_alloc	= entry->bytes_alloc;
	ev_alloc->gfp_flags	= entry->gfp_flags;
	ev_alloc->node		= entry->node;

	return TRACE_TYPE_HANDLED;
}

static enum print_line_t
kmemtrace_print_free_user(struct trace_iterator *iter, int flags,
			  struct trace_event *event)
{
	struct trace_seq *s = &iter->seq;
	struct kmemtrace_free_entry *entry;
	struct kmemtrace_user_event *ev;

	trace_assign_type(entry, iter->ent);

	ev = trace_seq_reserve(s, sizeof(*ev));
	if (!ev)
		return TRACE_TYPE_PARTIAL_LINE;

	ev->event_id		= KMEMTRACE_USER_FREE;
	ev->type_id		= entry->type_id;
	ev->event_size		= sizeof(*ev);
	ev->cpu			= iter->cpu;
	ev->timestamp		= iter->ts;
	ev->call_site		= entry->call_site;
	ev->ptr			= (unsigned long)entry->ptr;

	return TRACE_TYPE_HANDLED;
}

/* The two other following provide a more minimalistic output */
static enum print_line_t
kmemtrace_print_alloc_compress(struct trace_iterator *iter)
{
	struct kmemtrace_alloc_entry *entry;
	struct trace_seq *s = &iter->seq;
	int ret;

	trace_assign_type(entry, iter->ent);

	/* Alloc entry */
	ret = trace_seq_printf(s, "  +      ");
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	/* Type */
	switch (entry->type_id) {
	case KMEMTRACE_TYPE_KMALLOC:
		ret = trace_seq_printf(s, "K   ");
		break;
	case KMEMTRACE_TYPE_CACHE:
		ret = trace_seq_printf(s, "C   ");
		break;
	case KMEMTRACE_TYPE_PAGES:
		ret = trace_seq_printf(s, "P   ");
		break;
	default:
		ret = trace_seq_printf(s, "?   ");
	}

	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	/* Requested */
	ret = trace_seq_printf(s, "%4zu   ", entry->bytes_req);
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	/* Allocated */
	ret = trace_seq_printf(s, "%4zu   ", entry->bytes_alloc);
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	/* Flags
	 * TODO: would be better to see the name of the GFP flag names
	 */
	ret = trace_seq_printf(s, "%08x   ", entry->gfp_flags);
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	/* Pointer to allocated */
	ret = trace_seq_printf(s, "0x%tx   ", (ptrdiff_t)entry->ptr);
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	/* Node and call site*/
	ret = trace_seq_printf(s, "%4d   %pf\n", entry->node,
						 (void *)entry->call_site);
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	return TRACE_TYPE_HANDLED;
}

static enum print_line_t
kmemtrace_print_free_compress(struct trace_iterator *iter)
{
	struct kmemtrace_free_entry *entry;
	struct trace_seq *s = &iter->seq;
	int ret;

	trace_assign_type(entry, iter->ent);

	/* Free entry */
	ret = trace_seq_printf(s, "  -      ");
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	/* Type */
	switch (entry->type_id) {
	case KMEMTRACE_TYPE_KMALLOC:
		ret = trace_seq_printf(s, "K     ");
		break;
	case KMEMTRACE_TYPE_CACHE:
		ret = trace_seq_printf(s, "C     ");
		break;
	case KMEMTRACE_TYPE_PAGES:
		ret = trace_seq_printf(s, "P     ");
		break;
	default:
		ret = trace_seq_printf(s, "?     ");
	}

	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	/* Skip requested/allocated/flags */
	ret = trace_seq_printf(s, "                       ");
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	/* Pointer to allocated */
	ret = trace_seq_printf(s, "0x%tx   ", (ptrdiff_t)entry->ptr);
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	/* Skip node and print call site*/
	ret = trace_seq_printf(s, "       %pf\n", (void *)entry->call_site);
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	return TRACE_TYPE_HANDLED;
}

static enum print_line_t kmemtrace_print_line(struct trace_iterator *iter)
{
	struct trace_entry *entry = iter->ent;

	if (!(kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL))
		return TRACE_TYPE_UNHANDLED;

	switch (entry->type) {
	case TRACE_KMEM_ALLOC:
		return kmemtrace_print_alloc_compress(iter);
	case TRACE_KMEM_FREE:
		return kmemtrace_print_free_compress(iter);
	default:
		return TRACE_TYPE_UNHANDLED;
	}
}

static struct trace_event_functions kmem_trace_alloc_funcs = {
	.trace			= kmemtrace_print_alloc,
	.binary			= kmemtrace_print_alloc_user,
};

static struct trace_event kmem_trace_alloc = {
	.type			= TRACE_KMEM_ALLOC,
	.funcs			= &kmem_trace_alloc_funcs,
};

static struct trace_event_functions kmem_trace_free_funcs = {
	.trace			= kmemtrace_print_free,
	.binary			= kmemtrace_print_free_user,
};

static struct trace_event kmem_trace_free = {
	.type			= TRACE_KMEM_FREE,
	.funcs			= &kmem_trace_free_funcs,
};

static struct tracer kmem_tracer __read_mostly = {
	.name			= "kmemtrace",
	.init			= kmem_trace_init,
	.reset			= kmem_trace_reset,
	.print_line		= kmemtrace_print_line,
	.print_header		= kmemtrace_headers,
	.flags			= &kmem_tracer_flags
};

void kmemtrace_init(void)
{
	/* earliest opportunity to start kmem tracing */
}

static int __init init_kmem_tracer(void)
{
	if (!register_ftrace_event(&kmem_trace_alloc)) {
		pr_warning("Warning: could not register kmem events\n");
		return 1;
	}

	if (!register_ftrace_event(&kmem_trace_free)) {
		pr_warning("Warning: could not register kmem events\n");
		return 1;
	}

	if (register_tracer(&kmem_tracer) != 0) {
		pr_warning("Warning: could not register the kmem tracer\n");
		return 1;
	}

	return 0;
}
device_initcall(init_kmem_tracer);
Commit	Line	Data
36994e58 FW	1	/*
	2	* Memory allocator tracing
	3	*
	4	* Copyright (C) 2008 Eduard - Gabriel Munteanu
	5	* Copyright (C) 2008 Pekka Enberg <penberg@cs.helsinki.fi>
	6	* Copyright (C) 2008 Frederic Weisbecker <fweisbec@gmail.com>
	7	*/
	8
c826e3cd IM	9	#include <linux/tracepoint.h>
c826e3cd IM	10	#include <linux/seq_file.h>
36994e58	11	#include <linux/debugfs.h>
c826e3cd	12	#include <linux/dcache.h>
36994e58	13	#include <linux/fs.h>
c826e3cd	14
02af61bb	15	#include <linux/kmemtrace.h>
36994e58	16
36994e58	17	#include "trace_output.h"
c826e3cd	18	#include "trace.h"
36994e58 FW	19
	20	/* Select an alternative, minimalistic output than the original one */
	21	#define TRACE_KMEM_OPT_MINIMAL 0x1
	22
	23	static struct tracer_opt kmem_opts[] = {
	24	/* Default disable the minimalistic output */
	25	{ TRACER_OPT(kmem_minimalistic, TRACE_KMEM_OPT_MINIMAL) },
	26	{ }
	27	};
	28
	29	static struct tracer_flags kmem_tracer_flags = {
c826e3cd IM	30	.val = 0,
c826e3cd IM	31	.opts = kmem_opts
36994e58 FW	32	};
36994e58 FW	33
36994e58 FW	34	static struct trace_array *kmemtrace_array;
36994e58 FW	35
ca2b84cb EGM	36	/* Trace allocations */
	37	static inline void kmemtrace_alloc(enum kmemtrace_type_id type_id,
	38	unsigned long call_site,
	39	const void *ptr,
	40	size_t bytes_req,
	41	size_t bytes_alloc,
	42	gfp_t gfp_flags,
	43	int node)
	44	{
e1112b4d	45	struct ftrace_event_call *call = &event_kmem_alloc;
ca2b84cb	46	struct trace_array *tr = kmemtrace_array;
c826e3cd IM	47	struct kmemtrace_alloc_entry *entry;
c826e3cd IM	48	struct ring_buffer_event *event;
ca2b84cb EGM	49
	50	event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry));
	51	if (!event)
	52	return;
c826e3cd IM	53
c826e3cd IM	54	entry = ring_buffer_event_data(event);
ca2b84cb EGM	55	tracing_generic_entry_update(&entry->ent, 0, 0);
ca2b84cb EGM	56
c826e3cd IM	57	entry->ent.type = TRACE_KMEM_ALLOC;
	58	entry->type_id = type_id;
	59	entry->call_site = call_site;
	60	entry->ptr = ptr;
	61	entry->bytes_req = bytes_req;
	62	entry->bytes_alloc = bytes_alloc;
	63	entry->gfp_flags = gfp_flags;
	64	entry->node = node;
ca2b84cb	65
eb02ce01 TZ	66	if (!filter_check_discard(call, entry, tr->buffer, event))
eb02ce01 TZ	67	ring_buffer_unlock_commit(tr->buffer, event);
ca2b84cb EGM	68
	69	trace_wake_up();
	70	}
	71
	72	static inline void kmemtrace_free(enum kmemtrace_type_id type_id,
	73	unsigned long call_site,
	74	const void *ptr)
	75	{
e1112b4d	76	struct ftrace_event_call *call = &event_kmem_free;
ca2b84cb	77	struct trace_array *tr = kmemtrace_array;
c826e3cd IM	78	struct kmemtrace_free_entry *entry;
c826e3cd IM	79	struct ring_buffer_event *event;
ca2b84cb EGM	80
	81	event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry));
	82	if (!event)
	83	return;
	84	entry = ring_buffer_event_data(event);
	85	tracing_generic_entry_update(&entry->ent, 0, 0);
	86
c826e3cd IM	87	entry->ent.type = TRACE_KMEM_FREE;
	88	entry->type_id = type_id;
	89	entry->call_site = call_site;
	90	entry->ptr = ptr;
ca2b84cb	91
eb02ce01 TZ	92	if (!filter_check_discard(call, entry, tr->buffer, event))
eb02ce01 TZ	93	ring_buffer_unlock_commit(tr->buffer, event);
ca2b84cb EGM	94
	95	trace_wake_up();
	96	}
	97
38516ab5 SR	98	static void kmemtrace_kmalloc(void *ignore,
38516ab5 SR	99	unsigned long call_site,
ca2b84cb EGM	100	const void *ptr,
	101	size_t bytes_req,
	102	size_t bytes_alloc,
	103	gfp_t gfp_flags)
	104	{
	105	kmemtrace_alloc(KMEMTRACE_TYPE_KMALLOC, call_site, ptr,
	106	bytes_req, bytes_alloc, gfp_flags, -1);
	107	}
	108
38516ab5 SR	109	static void kmemtrace_kmem_cache_alloc(void *ignore,
38516ab5 SR	110	unsigned long call_site,
ca2b84cb EGM	111	const void *ptr,
	112	size_t bytes_req,
	113	size_t bytes_alloc,
	114	gfp_t gfp_flags)
	115	{
	116	kmemtrace_alloc(KMEMTRACE_TYPE_CACHE, call_site, ptr,
	117	bytes_req, bytes_alloc, gfp_flags, -1);
	118	}
	119
38516ab5 SR	120	static void kmemtrace_kmalloc_node(void *ignore,
38516ab5 SR	121	unsigned long call_site,
ca2b84cb EGM	122	const void *ptr,
	123	size_t bytes_req,
	124	size_t bytes_alloc,
	125	gfp_t gfp_flags,
	126	int node)
	127	{
	128	kmemtrace_alloc(KMEMTRACE_TYPE_KMALLOC, call_site, ptr,
	129	bytes_req, bytes_alloc, gfp_flags, node);
	130	}
	131
38516ab5 SR	132	static void kmemtrace_kmem_cache_alloc_node(void *ignore,
38516ab5 SR	133	unsigned long call_site,
ca2b84cb EGM	134	const void *ptr,
	135	size_t bytes_req,
	136	size_t bytes_alloc,
	137	gfp_t gfp_flags,
	138	int node)
	139	{
	140	kmemtrace_alloc(KMEMTRACE_TYPE_CACHE, call_site, ptr,
	141	bytes_req, bytes_alloc, gfp_flags, node);
	142	}
	143
38516ab5 SR	144	static void
38516ab5 SR	145	kmemtrace_kfree(void ignore, unsigned long call_site, const void ptr)
ca2b84cb EGM	146	{
	147	kmemtrace_free(KMEMTRACE_TYPE_KMALLOC, call_site, ptr);
	148	}
	149
38516ab5 SR	150	static void kmemtrace_kmem_cache_free(void *ignore,
38516ab5 SR	151	unsigned long call_site, const void *ptr)
ca2b84cb EGM	152	{
	153	kmemtrace_free(KMEMTRACE_TYPE_CACHE, call_site, ptr);
	154	}
	155
	156	static int kmemtrace_start_probes(void)
	157	{
	158	int err;
	159
38516ab5	160	err = register_trace_kmalloc(kmemtrace_kmalloc, NULL);
ca2b84cb EGM	161	if (err)
ca2b84cb EGM	162	return err;
38516ab5	163	err = register_trace_kmem_cache_alloc(kmemtrace_kmem_cache_alloc, NULL);
ca2b84cb EGM	164	if (err)
ca2b84cb EGM	165	return err;
38516ab5	166	err = register_trace_kmalloc_node(kmemtrace_kmalloc_node, NULL);
ca2b84cb EGM	167	if (err)
ca2b84cb EGM	168	return err;
38516ab5	169	err = register_trace_kmem_cache_alloc_node(kmemtrace_kmem_cache_alloc_node, NULL);
ca2b84cb EGM	170	if (err)
ca2b84cb EGM	171	return err;
38516ab5	172	err = register_trace_kfree(kmemtrace_kfree, NULL);
ca2b84cb EGM	173	if (err)
ca2b84cb EGM	174	return err;
38516ab5	175	err = register_trace_kmem_cache_free(kmemtrace_kmem_cache_free, NULL);
ca2b84cb EGM	176
	177	return err;
	178	}
	179
	180	static void kmemtrace_stop_probes(void)
	181	{
38516ab5 SR	182	unregister_trace_kmalloc(kmemtrace_kmalloc, NULL);
	183	unregister_trace_kmem_cache_alloc(kmemtrace_kmem_cache_alloc, NULL);
	184	unregister_trace_kmalloc_node(kmemtrace_kmalloc_node, NULL);
	185	unregister_trace_kmem_cache_alloc_node(kmemtrace_kmem_cache_alloc_node, NULL);
	186	unregister_trace_kfree(kmemtrace_kfree, NULL);
	187	unregister_trace_kmem_cache_free(kmemtrace_kmem_cache_free, NULL);
ca2b84cb EGM	188	}
ca2b84cb EGM	189
36994e58 FW	190	static int kmem_trace_init(struct trace_array *tr)
36994e58 FW	191	{
36994e58 FW	192	kmemtrace_array = tr;
36994e58 FW	193
76f0d073	194	tracing_reset_online_cpus(tr);
36994e58	195
ca2b84cb	196	kmemtrace_start_probes();
36994e58 FW	197
	198	return 0;
	199	}
	200
	201	static void kmem_trace_reset(struct trace_array *tr)
	202	{
ca2b84cb	203	kmemtrace_stop_probes();
36994e58 FW	204	}
	205
	206	static void kmemtrace_headers(struct seq_file *s)
	207	{
	208	/* Don't need headers for the original kmemtrace output */
	209	if (!(kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL))
	210	return;
	211
	212	seq_printf(s, "#\n");
	213	seq_printf(s, "# ALLOC TYPE REQ GIVEN FLAGS "
	214	" POINTER NODE CALLER\n");
	215	seq_printf(s, "# FREE \| \| \| \| "
	216	" \| \| \| \|\n");
	217	seq_printf(s, "# \|\n\n");
	218	}
	219
	220	/*
42af9054 EGM	221	* The following functions give the original output from kmemtrace,
42af9054 EGM	222	* plus the origin CPU, since reordering occurs in-kernel now.
36994e58	223	*/
42af9054 EGM	224
	225	#define KMEMTRACE_USER_ALLOC 0
	226	#define KMEMTRACE_USER_FREE 1
	227
	228	struct kmemtrace_user_event {
c826e3cd IM	229	u8 event_id;
	230	u8 type_id;
	231	u16 event_size;
	232	u32 cpu;
	233	u64 timestamp;
	234	unsigned long call_site;
	235	unsigned long ptr;
42af9054 EGM	236	};
	237
	238	struct kmemtrace_user_event_alloc {
c826e3cd IM	239	size_t bytes_req;
	240	size_t bytes_alloc;
	241	unsigned gfp_flags;
	242	int node;
42af9054 EGM	243	};
42af9054 EGM	244
36994e58	245	static enum print_line_t
a9a57763 SR	246	kmemtrace_print_alloc(struct trace_iterator *iter, int flags,
a9a57763 SR	247	struct trace_event *event)
ddc1637a LZ	248	{
	249	struct trace_seq *s = &iter->seq;
	250	struct kmemtrace_alloc_entry *entry;
	251	int ret;
	252
	253	trace_assign_type(entry, iter->ent);
	254
	255	ret = trace_seq_printf(s, "type_id %d call_site %pF ptr %lu "
	256	"bytes_req %lu bytes_alloc %lu gfp_flags %lu node %d\n",
	257	entry->type_id, (void *)entry->call_site, (unsigned long)entry->ptr,
	258	(unsigned long)entry->bytes_req, (unsigned long)entry->bytes_alloc,
	259	(unsigned long)entry->gfp_flags, entry->node);
	260
	261	if (!ret)
	262	return TRACE_TYPE_PARTIAL_LINE;
	263	return TRACE_TYPE_HANDLED;
	264	}
	265
	266	static enum print_line_t
a9a57763 SR	267	kmemtrace_print_free(struct trace_iterator *iter, int flags,
a9a57763 SR	268	struct trace_event *event)
36994e58 FW	269	{
36994e58 FW	270	struct trace_seq *s = &iter->seq;
ddc1637a LZ	271	struct kmemtrace_free_entry *entry;
	272	int ret;
	273
	274	trace_assign_type(entry, iter->ent);
	275
	276	ret = trace_seq_printf(s, "type_id %d call_site %pF ptr %lu\n",
	277	entry->type_id, (void *)entry->call_site,
	278	(unsigned long)entry->ptr);
	279
	280	if (!ret)
	281	return TRACE_TYPE_PARTIAL_LINE;
	282	return TRACE_TYPE_HANDLED;
	283	}
	284
	285	static enum print_line_t
a9a57763 SR	286	kmemtrace_print_alloc_user(struct trace_iterator *iter, int flags,
a9a57763 SR	287	struct trace_event *event)
ddc1637a LZ	288	{
	289	struct trace_seq *s = &iter->seq;
	290	struct kmemtrace_alloc_entry *entry;
42af9054	291	struct kmemtrace_user_event *ev;
ddc1637a LZ	292	struct kmemtrace_user_event_alloc *ev_alloc;
	293
	294	trace_assign_type(entry, iter->ent);
36994e58	295
42af9054 EGM	296	ev = trace_seq_reserve(s, sizeof(*ev));
	297	if (!ev)
	298	return TRACE_TYPE_PARTIAL_LINE;
c826e3cd IM	299
	300	ev->event_id = KMEMTRACE_USER_ALLOC;
	301	ev->type_id = entry->type_id;
	302	ev->event_size = sizeof(ev) + sizeof(ev_alloc);
	303	ev->cpu = iter->cpu;
	304	ev->timestamp = iter->ts;
	305	ev->call_site = entry->call_site;
	306	ev->ptr = (unsigned long)entry->ptr;
42af9054 EGM	307
	308	ev_alloc = trace_seq_reserve(s, sizeof(*ev_alloc));
	309	if (!ev_alloc)
36994e58	310	return TRACE_TYPE_PARTIAL_LINE;
c826e3cd IM	311
	312	ev_alloc->bytes_req = entry->bytes_req;
	313	ev_alloc->bytes_alloc = entry->bytes_alloc;
	314	ev_alloc->gfp_flags = entry->gfp_flags;
	315	ev_alloc->node = entry->node;
36994e58 FW	316
	317	return TRACE_TYPE_HANDLED;
	318	}
	319
	320	static enum print_line_t
a9a57763 SR	321	kmemtrace_print_free_user(struct trace_iterator *iter, int flags,
a9a57763 SR	322	struct trace_event *event)
36994e58 FW	323	{
36994e58 FW	324	struct trace_seq *s = &iter->seq;
ddc1637a	325	struct kmemtrace_free_entry *entry;
42af9054	326	struct kmemtrace_user_event *ev;
36994e58	327
ddc1637a LZ	328	trace_assign_type(entry, iter->ent);
ddc1637a LZ	329
42af9054 EGM	330	ev = trace_seq_reserve(s, sizeof(*ev));
42af9054 EGM	331	if (!ev)
36994e58	332	return TRACE_TYPE_PARTIAL_LINE;
c826e3cd IM	333
	334	ev->event_id = KMEMTRACE_USER_FREE;
	335	ev->type_id = entry->type_id;
	336	ev->event_size = sizeof(*ev);
	337	ev->cpu = iter->cpu;
	338	ev->timestamp = iter->ts;
	339	ev->call_site = entry->call_site;
	340	ev->ptr = (unsigned long)entry->ptr;
36994e58 FW	341
	342	return TRACE_TYPE_HANDLED;
	343	}
	344
36994e58 FW	345	/* The two other following provide a more minimalistic output */
36994e58 FW	346	static enum print_line_t
ddc1637a	347	kmemtrace_print_alloc_compress(struct trace_iterator *iter)
36994e58	348	{
ddc1637a	349	struct kmemtrace_alloc_entry *entry;
36994e58 FW	350	struct trace_seq *s = &iter->seq;
	351	int ret;
	352
ddc1637a LZ	353	trace_assign_type(entry, iter->ent);
ddc1637a LZ	354
36994e58 FW	355	/* Alloc entry */
	356	ret = trace_seq_printf(s, " + ");
	357	if (!ret)
	358	return TRACE_TYPE_PARTIAL_LINE;
	359
	360	/* Type */
	361	switch (entry->type_id) {
	362	case KMEMTRACE_TYPE_KMALLOC:
	363	ret = trace_seq_printf(s, "K ");
	364	break;
	365	case KMEMTRACE_TYPE_CACHE:
	366	ret = trace_seq_printf(s, "C ");
	367	break;
	368	case KMEMTRACE_TYPE_PAGES:
	369	ret = trace_seq_printf(s, "P ");
	370	break;
	371	default:
	372	ret = trace_seq_printf(s, "? ");
	373	}
	374
	375	if (!ret)
	376	return TRACE_TYPE_PARTIAL_LINE;
	377
	378	/* Requested */
ecf441b5	379	ret = trace_seq_printf(s, "%4zu ", entry->bytes_req);
36994e58 FW	380	if (!ret)
	381	return TRACE_TYPE_PARTIAL_LINE;
	382
	383	/* Allocated */
ecf441b5	384	ret = trace_seq_printf(s, "%4zu ", entry->bytes_alloc);
36994e58 FW	385	if (!ret)
	386	return TRACE_TYPE_PARTIAL_LINE;
	387
	388	/* Flags
	389	* TODO: would be better to see the name of the GFP flag names
	390	*/
	391	ret = trace_seq_printf(s, "%08x ", entry->gfp_flags);
	392	if (!ret)
	393	return TRACE_TYPE_PARTIAL_LINE;
	394
	395	/* Pointer to allocated */
	396	ret = trace_seq_printf(s, "0x%tx ", (ptrdiff_t)entry->ptr);
	397	if (!ret)
	398	return TRACE_TYPE_PARTIAL_LINE;
	399
6a167c65 FW	400	/* Node and call site*/
	401	ret = trace_seq_printf(s, "%4d %pf\n", entry->node,
	402	(void *)entry->call_site);
36994e58 FW	403	if (!ret)
	404	return TRACE_TYPE_PARTIAL_LINE;
	405
36994e58 FW	406	return TRACE_TYPE_HANDLED;
	407	}
	408
	409	static enum print_line_t
ddc1637a	410	kmemtrace_print_free_compress(struct trace_iterator *iter)
36994e58	411	{
ddc1637a	412	struct kmemtrace_free_entry *entry;
36994e58 FW	413	struct trace_seq *s = &iter->seq;
	414	int ret;
	415
ddc1637a LZ	416	trace_assign_type(entry, iter->ent);
ddc1637a LZ	417
36994e58 FW	418	/* Free entry */
	419	ret = trace_seq_printf(s, " - ");
	420	if (!ret)
	421	return TRACE_TYPE_PARTIAL_LINE;
	422
	423	/* Type */
	424	switch (entry->type_id) {
	425	case KMEMTRACE_TYPE_KMALLOC:
	426	ret = trace_seq_printf(s, "K ");
	427	break;
	428	case KMEMTRACE_TYPE_CACHE:
	429	ret = trace_seq_printf(s, "C ");
	430	break;
	431	case KMEMTRACE_TYPE_PAGES:
	432	ret = trace_seq_printf(s, "P ");
	433	break;
	434	default:
	435	ret = trace_seq_printf(s, "? ");
	436	}
	437
	438	if (!ret)
	439	return TRACE_TYPE_PARTIAL_LINE;
	440
	441	/* Skip requested/allocated/flags */
	442	ret = trace_seq_printf(s, " ");
	443	if (!ret)
	444	return TRACE_TYPE_PARTIAL_LINE;
	445
	446	/* Pointer to allocated */
	447	ret = trace_seq_printf(s, "0x%tx ", (ptrdiff_t)entry->ptr);
	448	if (!ret)
	449	return TRACE_TYPE_PARTIAL_LINE;
	450
6a167c65 FW	451	/* Skip node and print call site*/
6a167c65 FW	452	ret = trace_seq_printf(s, " %pf\n", (void *)entry->call_site);
36994e58 FW	453	if (!ret)
	454	return TRACE_TYPE_PARTIAL_LINE;
	455
36994e58 FW	456	return TRACE_TYPE_HANDLED;
	457	}
	458
	459	static enum print_line_t kmemtrace_print_line(struct trace_iterator *iter)
	460	{
	461	struct trace_entry *entry = iter->ent;
	462
ddc1637a LZ	463	if (!(kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL))
ddc1637a LZ	464	return TRACE_TYPE_UNHANDLED;
36994e58	465
ddc1637a LZ	466	switch (entry->type) {
	467	case TRACE_KMEM_ALLOC:
	468	return kmemtrace_print_alloc_compress(iter);
	469	case TRACE_KMEM_FREE:
	470	return kmemtrace_print_free_compress(iter);
36994e58 FW	471	default:
	472	return TRACE_TYPE_UNHANDLED;
	473	}
	474	}
	475
a9a57763	476	static struct trace_event_functions kmem_trace_alloc_funcs = {
80098c20 LZ	477	.trace = kmemtrace_print_alloc,
80098c20 LZ	478	.binary = kmemtrace_print_alloc_user,
ddc1637a LZ	479	};
ddc1637a LZ	480
a9a57763 SR	481	static struct trace_event kmem_trace_alloc = {
	482	.type = TRACE_KMEM_ALLOC,
	483	.funcs = &kmem_trace_alloc_funcs,
	484	};
	485
	486	static struct trace_event_functions kmem_trace_free_funcs = {
80098c20 LZ	487	.trace = kmemtrace_print_free,
80098c20 LZ	488	.binary = kmemtrace_print_free_user,
ddc1637a LZ	489	};
ddc1637a LZ	490
a9a57763 SR	491	static struct trace_event kmem_trace_free = {
	492	.type = TRACE_KMEM_FREE,
	493	.funcs = &kmem_trace_free_funcs,
	494	};
	495
36994e58	496	static struct tracer kmem_tracer __read_mostly = {
c826e3cd IM	497	.name = "kmemtrace",
	498	.init = kmem_trace_init,
	499	.reset = kmem_trace_reset,
	500	.print_line = kmemtrace_print_line,
	501	.print_header = kmemtrace_headers,
	502	.flags = &kmem_tracer_flags
36994e58 FW	503	};
36994e58 FW	504
3e806802 IM	505	void kmemtrace_init(void)
	506	{
	507	/* earliest opportunity to start kmem tracing */
	508	}
	509
36994e58 FW	510	static int __init init_kmem_tracer(void)
36994e58 FW	511	{
ddc1637a LZ	512	if (!register_ftrace_event(&kmem_trace_alloc)) {
	513	pr_warning("Warning: could not register kmem events\n");
	514	return 1;
	515	}
	516
	517	if (!register_ftrace_event(&kmem_trace_free)) {
	518	pr_warning("Warning: could not register kmem events\n");
	519	return 1;
	520	}
	521
f9ac5a69	522	if (register_tracer(&kmem_tracer) != 0) {
ddc1637a LZ	523	pr_warning("Warning: could not register the kmem tracer\n");
	524	return 1;
	525	}
	526
	527	return 0;
36994e58	528	}
36994e58	529	device_initcall(init_kmem_tracer);