From 51df1142816e469173889fb6d6dc810be9b9e022 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Fri, 20 Aug 2010 12:37:15 -0500 Subject: slub: Dynamically size kmalloc cache allocations kmalloc caches are statically defined and may take up a lot of space just because the sizes of the node array has to be dimensioned for the largest node count supported. This patch makes the size of the kmem_cache structure dynamic throughout by creating a kmem_cache slab cache for the kmem_cache objects. The bootstrap occurs by allocating the initial one or two kmem_cache objects from the page allocator. C2->C3 - Fix various issues indicated by David - Make create kmalloc_cache return a kmem_cache * pointer. Acked-by: David Rientjes Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/slub_def.h | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'include/linux/slub_def.h') diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index 9f63538928c0..a6c43ec6a4a5 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -139,19 +139,16 @@ struct kmem_cache { #ifdef CONFIG_ZONE_DMA #define SLUB_DMA __GFP_DMA -/* Reserve extra caches for potential DMA use */ -#define KMALLOC_CACHES (2 * SLUB_PAGE_SHIFT) #else /* Disable DMA functionality */ #define SLUB_DMA (__force gfp_t)0 -#define KMALLOC_CACHES SLUB_PAGE_SHIFT #endif /* * We keep the general caches in an array of slab caches that are used for * 2^x bytes of allocations. */ -extern struct kmem_cache kmalloc_caches[KMALLOC_CACHES]; +extern struct kmem_cache *kmalloc_caches[SLUB_PAGE_SHIFT]; /* * Sorry that the following has to be that ugly but some versions of GCC @@ -216,7 +213,7 @@ static __always_inline struct kmem_cache *kmalloc_slab(size_t size) if (index == 0) return NULL; - return &kmalloc_caches[index]; + return kmalloc_caches[index]; } void *kmem_cache_alloc(struct kmem_cache *, gfp_t); -- cgit v1.2.3-59-g8ed1b From 7340cc84141d5236c5dd003359ee921513cd9b84 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Tue, 28 Sep 2010 08:10:26 -0500 Subject: slub: reduce differences between SMP and NUMA Reduce the #ifdefs and simplify bootstrap by making SMP and NUMA as much alike as possible. This means that there will be an additional indirection to get to the kmem_cache_node field under SMP. Acked-by: David Rientjes Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/slub_def.h | 5 +---- mm/slub.c | 39 +-------------------------------------- 2 files changed, 2 insertions(+), 42 deletions(-) (limited to 'include/linux/slub_def.h') diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index a6c43ec6a4a5..b33c0f2e61dc 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -96,11 +96,8 @@ struct kmem_cache { * Defragmentation by allocating from a remote node. */ int remote_node_defrag_ratio; - struct kmem_cache_node *node[MAX_NUMNODES]; -#else - /* Avoid an extra cache line for UP */ - struct kmem_cache_node local_node; #endif + struct kmem_cache_node *node[MAX_NUMNODES]; }; /* diff --git a/mm/slub.c b/mm/slub.c index 7e1fe663795a..064bda294af2 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -233,11 +233,7 @@ int slab_is_available(void) static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node) { -#ifdef CONFIG_NUMA return s->node[node]; -#else - return &s->local_node; -#endif } /* Verify that a pointer has an address that is valid within a slab page */ @@ -871,7 +867,7 @@ static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects) * dilemma by deferring the increment of the count during * bootstrap (see early_kmem_cache_node_alloc). */ - if (!NUMA_BUILD || n) { + if (n) { atomic_long_inc(&n->nr_slabs); atomic_long_add(objects, &n->total_objects); } @@ -2112,7 +2108,6 @@ static inline int alloc_kmem_cache_cpus(struct kmem_cache *s) return s->cpu_slab != NULL; } -#ifdef CONFIG_NUMA static struct kmem_cache *kmem_cache_node; /* @@ -2202,17 +2197,6 @@ static int init_kmem_cache_nodes(struct kmem_cache *s) } return 1; } -#else -static void free_kmem_cache_nodes(struct kmem_cache *s) -{ -} - -static int init_kmem_cache_nodes(struct kmem_cache *s) -{ - init_kmem_cache_node(&s->local_node, s); - return 1; -} -#endif static void set_min_partial(struct kmem_cache *s, unsigned long min) { @@ -3023,8 +3007,6 @@ void __init kmem_cache_init(void) int caches = 0; struct kmem_cache *temp_kmem_cache; int order; - -#ifdef CONFIG_NUMA struct kmem_cache *temp_kmem_cache_node; unsigned long kmalloc_size; @@ -3048,12 +3030,6 @@ void __init kmem_cache_init(void) 0, SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL); hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI); -#else - /* Allocate a single kmem_cache from the page allocator */ - kmem_size = sizeof(struct kmem_cache); - order = get_order(kmem_size); - kmem_cache = (void *)__get_free_pages(GFP_NOWAIT, order); -#endif /* Able to allocate the per node structures */ slab_state = PARTIAL; @@ -3064,7 +3040,6 @@ void __init kmem_cache_init(void) kmem_cache = kmem_cache_alloc(kmem_cache, GFP_NOWAIT); memcpy(kmem_cache, temp_kmem_cache, kmem_size); -#ifdef CONFIG_NUMA /* * Allocate kmem_cache_node properly from the kmem_cache slab. * kmem_cache_node is separately allocated so no need to @@ -3078,18 +3053,6 @@ void __init kmem_cache_init(void) kmem_cache_bootstrap_fixup(kmem_cache_node); caches++; -#else - /* - * kmem_cache has kmem_cache_node embedded and we moved it! - * Update the list heads - */ - INIT_LIST_HEAD(&kmem_cache->local_node.partial); - list_splice(&temp_kmem_cache->local_node.partial, &kmem_cache->local_node.partial); -#ifdef CONFIG_SLUB_DEBUG - INIT_LIST_HEAD(&kmem_cache->local_node.full); - list_splice(&temp_kmem_cache->local_node.full, &kmem_cache->local_node.full); -#endif -#endif kmem_cache_bootstrap_fixup(kmem_cache); caches++; /* Free temporary boot structure */ -- cgit v1.2.3-59-g8ed1b From ab4d5ed5eeda4f57c50d14131ce1b1da75d0c938 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Tue, 5 Oct 2010 13:57:26 -0500 Subject: slub: Enable sysfs support for !CONFIG_SLUB_DEBUG Currently disabling CONFIG_SLUB_DEBUG also disabled SYSFS support meaning that the slabs cannot be tuned without DEBUG. Make SYSFS support independent of CONFIG_SLUB_DEBUG Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/slub_def.h | 2 +- lib/Kconfig.debug | 2 +- mm/slub.c | 40 +++++++++++++++++++++++++++++++++++----- 3 files changed, 37 insertions(+), 7 deletions(-) (limited to 'include/linux/slub_def.h') diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index b33c0f2e61dc..e4f5ed180b9b 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -87,7 +87,7 @@ struct kmem_cache { unsigned long min_partial; const char *name; /* Name (only for display!) */ struct list_head list; /* List of slab caches */ -#ifdef CONFIG_SLUB_DEBUG +#ifdef CONFIG_SYSFS struct kobject kobj; /* For sysfs */ #endif diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 1b4afd2e6ca0..b6263651a955 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -353,7 +353,7 @@ config SLUB_DEBUG_ON config SLUB_STATS default n bool "Enable SLUB performance statistics" - depends on SLUB && SLUB_DEBUG && SYSFS + depends on SLUB && SYSFS help SLUB statistics are useful to debug SLUBs allocation behavior in order find ways to optimize the allocator. This should never be diff --git a/mm/slub.c b/mm/slub.c index a018019aa91d..be4d66231c6f 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -198,7 +198,7 @@ struct track { enum track_item { TRACK_ALLOC, TRACK_FREE }; -#ifdef CONFIG_SLUB_DEBUG +#ifdef CONFIG_SYSFS static int sysfs_slab_add(struct kmem_cache *); static int sysfs_slab_alias(struct kmem_cache *, const char *); static void sysfs_slab_remove(struct kmem_cache *); @@ -1102,7 +1102,7 @@ static inline void slab_free_hook(struct kmem_cache *s, void *x) {} static inline void slab_free_hook_irq(struct kmem_cache *s, void *object) {} -#endif +#endif /* CONFIG_SLUB_DEBUG */ /* * Slab allocation and freeing @@ -3373,7 +3373,7 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags, } #endif -#ifdef CONFIG_SLUB_DEBUG +#ifdef CONFIG_SYSFS static int count_inuse(struct page *page) { return page->inuse; @@ -3383,7 +3383,9 @@ static int count_total(struct page *page) { return page->objects; } +#endif +#ifdef CONFIG_SLUB_DEBUG static int validate_slab(struct kmem_cache *s, struct page *page, unsigned long *map) { @@ -3474,6 +3476,7 @@ static long validate_slab_cache(struct kmem_cache *s) kfree(map); return count; } +#endif #ifdef SLUB_RESILIENCY_TEST static void resiliency_test(void) @@ -3532,9 +3535,12 @@ static void resiliency_test(void) validate_slab_cache(kmalloc_caches[9]); } #else +#ifdef CONFIG_SYSFS static void resiliency_test(void) {}; #endif +#endif +#ifdef CONFIG_DEBUG /* * Generate lists of code addresses where slabcache objects are allocated * and freed. @@ -3763,7 +3769,9 @@ static int list_locations(struct kmem_cache *s, char *buf, len += sprintf(buf, "No data\n"); return len; } +#endif +#ifdef CONFIG_SYSFS enum slab_stat_type { SL_ALL, /* All slabs */ SL_PARTIAL, /* Only partially allocated slabs */ @@ -3816,6 +3824,8 @@ static ssize_t show_slab_objects(struct kmem_cache *s, } } + down_read(&slub_lock); +#ifdef CONFIG_SLUB_DEBUG if (flags & SO_ALL) { for_each_node_state(node, N_NORMAL_MEMORY) { struct kmem_cache_node *n = get_node(s, node); @@ -3832,7 +3842,9 @@ static ssize_t show_slab_objects(struct kmem_cache *s, nodes[node] += x; } - } else if (flags & SO_PARTIAL) { + } else +#endif + if (flags & SO_PARTIAL) { for_each_node_state(node, N_NORMAL_MEMORY) { struct kmem_cache_node *n = get_node(s, node); @@ -3857,6 +3869,7 @@ static ssize_t show_slab_objects(struct kmem_cache *s, return x + sprintf(buf + x, "\n"); } +#ifdef CONFIG_SLUB_DEBUG static int any_slab_objects(struct kmem_cache *s) { int node; @@ -3872,6 +3885,7 @@ static int any_slab_objects(struct kmem_cache *s) } return 0; } +#endif #define to_slab_attr(n) container_of(n, struct slab_attribute, attr) #define to_slab(n) container_of(n, struct kmem_cache, kobj); @@ -3973,11 +3987,13 @@ static ssize_t aliases_show(struct kmem_cache *s, char *buf) } SLAB_ATTR_RO(aliases); +#ifdef CONFIG_SLUB_DEBUG static ssize_t slabs_show(struct kmem_cache *s, char *buf) { return show_slab_objects(s, buf, SO_ALL); } SLAB_ATTR_RO(slabs); +#endif static ssize_t partial_show(struct kmem_cache *s, char *buf) { @@ -4003,6 +4019,7 @@ static ssize_t objects_partial_show(struct kmem_cache *s, char *buf) } SLAB_ATTR_RO(objects_partial); +#ifdef CONFIG_SLUB_DEBUG static ssize_t total_objects_show(struct kmem_cache *s, char *buf) { return show_slab_objects(s, buf, SO_ALL|SO_TOTAL); @@ -4055,6 +4072,7 @@ static ssize_t failslab_store(struct kmem_cache *s, const char *buf, } SLAB_ATTR(failslab); #endif +#endif static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf) { @@ -4091,6 +4109,7 @@ static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf) } SLAB_ATTR_RO(destroy_by_rcu); +#ifdef CONFIG_SLUB_DEBUG static ssize_t red_zone_show(struct kmem_cache *s, char *buf) { return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE)); @@ -4166,6 +4185,7 @@ static ssize_t validate_store(struct kmem_cache *s, return ret; } SLAB_ATTR(validate); +#endif static ssize_t shrink_show(struct kmem_cache *s, char *buf) { @@ -4186,6 +4206,7 @@ static ssize_t shrink_store(struct kmem_cache *s, } SLAB_ATTR(shrink); +#ifdef CONFIG_SLUB_DEBUG static ssize_t alloc_calls_show(struct kmem_cache *s, char *buf) { if (!(s->flags & SLAB_STORE_USER)) @@ -4201,6 +4222,7 @@ static ssize_t free_calls_show(struct kmem_cache *s, char *buf) return list_locations(s, buf, TRACK_FREE); } SLAB_ATTR_RO(free_calls); +#endif #ifdef CONFIG_NUMA static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf) @@ -4307,25 +4329,33 @@ static struct attribute *slab_attrs[] = { &min_partial_attr.attr, &objects_attr.attr, &objects_partial_attr.attr, +#ifdef CONFIG_SLUB_DEBUG &total_objects_attr.attr, &slabs_attr.attr, +#endif &partial_attr.attr, &cpu_slabs_attr.attr, &ctor_attr.attr, &aliases_attr.attr, &align_attr.attr, +#ifdef CONFIG_SLUB_DEBUG &sanity_checks_attr.attr, &trace_attr.attr, +#endif &hwcache_align_attr.attr, &reclaim_account_attr.attr, &destroy_by_rcu_attr.attr, +#ifdef CONFIG_SLUB_DEBUG &red_zone_attr.attr, &poison_attr.attr, &store_user_attr.attr, &validate_attr.attr, +#endif &shrink_attr.attr, +#ifdef CONFIG_SLUB_DEBUG &alloc_calls_attr.attr, &free_calls_attr.attr, +#endif #ifdef CONFIG_ZONE_DMA &cache_dma_attr.attr, #endif @@ -4608,7 +4638,7 @@ static int __init slab_sysfs_init(void) } __initcall(slab_sysfs_init); -#endif +#endif /* CONFIG_SYSFS */ /* * The /proc/slabinfo ABI -- cgit v1.2.3-59-g8ed1b