diff options
Diffstat (limited to 'drivers/staging/zram')
| -rw-r--r-- | drivers/staging/zram/Kconfig | 10 | ||||
| -rw-r--r-- | drivers/staging/zram/Makefile | 1 | ||||
| -rw-r--r-- | drivers/staging/zram/xvmalloc.c | 510 | ||||
| -rw-r--r-- | drivers/staging/zram/xvmalloc.h | 30 | ||||
| -rw-r--r-- | drivers/staging/zram/xvmalloc_int.h | 95 | ||||
| -rw-r--r-- | drivers/staging/zram/zram_drv.c | 142 | ||||
| -rw-r--r-- | drivers/staging/zram/zram_drv.h | 12 | ||||
| -rw-r--r-- | drivers/staging/zram/zram_sysfs.c | 4 |
8 files changed, 80 insertions, 724 deletions
diff --git a/drivers/staging/zram/Kconfig b/drivers/staging/zram/Kconfig index 3bec4dba3fe5..9d11a4cb99b7 100644 --- a/drivers/staging/zram/Kconfig +++ b/drivers/staging/zram/Kconfig @@ -1,11 +1,9 @@ -config XVMALLOC - bool - default n - config ZRAM tristate "Compressed RAM block device support" - depends on BLOCK && SYSFS - select XVMALLOC + # X86 dependency is because zsmalloc uses non-portable pte/tlb + # functions + depends on BLOCK && SYSFS && X86 + select ZSMALLOC select LZO_COMPRESS select LZO_DECOMPRESS default n diff --git a/drivers/staging/zram/Makefile b/drivers/staging/zram/Makefile index 2a6d3213a756..7f4a3019e9c4 100644 --- a/drivers/staging/zram/Makefile +++ b/drivers/staging/zram/Makefile @@ -1,4 +1,3 @@ zram-y := zram_drv.o zram_sysfs.o obj-$(CONFIG_ZRAM) += zram.o -obj-$(CONFIG_XVMALLOC) += xvmalloc.o
\ No newline at end of file diff --git a/drivers/staging/zram/xvmalloc.c b/drivers/staging/zram/xvmalloc.c deleted file mode 100644 index 1f9c5082b6d5..000000000000 --- a/drivers/staging/zram/xvmalloc.c +++ /dev/null @@ -1,510 +0,0 @@ -/* - * xvmalloc memory allocator - * - * Copyright (C) 2008, 2009, 2010 Nitin Gupta - * - * This code is released using a dual license strategy: BSD/GPL - * You can choose the licence that better fits your requirements. - * - * Released under the terms of 3-clause BSD License - * Released under the terms of GNU General Public License Version 2.0 - */ - -#ifdef CONFIG_ZRAM_DEBUG -#define DEBUG -#endif - -#include <linux/module.h> -#include <linux/kernel.h> -#include <linux/bitops.h> -#include <linux/errno.h> -#include <linux/highmem.h> -#include <linux/init.h> -#include <linux/string.h> -#include <linux/slab.h> - -#include "xvmalloc.h" -#include "xvmalloc_int.h" - -static void stat_inc(u64 *value) -{ - *value = *value + 1; -} - -static void stat_dec(u64 *value) -{ - *value = *value - 1; -} - -static int test_flag(struct block_header *block, enum blockflags flag) -{ - return block->prev & BIT(flag); -} - -static void set_flag(struct block_header *block, enum blockflags flag) -{ - block->prev |= BIT(flag); -} - -static void clear_flag(struct block_header *block, enum blockflags flag) -{ - block->prev &= ~BIT(flag); -} - -/* - * Given <page, offset> pair, provide a dereferencable pointer. - * This is called from xv_malloc/xv_free path, so it - * needs to be fast. - */ -static void *get_ptr_atomic(struct page *page, u16 offset, enum km_type type) -{ - unsigned char *base; - - base = kmap_atomic(page, type); - return base + offset; -} - -static void put_ptr_atomic(void *ptr, enum km_type type) -{ - kunmap_atomic(ptr, type); -} - -static u32 get_blockprev(struct block_header *block) -{ - return block->prev & PREV_MASK; -} - -static void set_blockprev(struct block_header *block, u16 new_offset) -{ - block->prev = new_offset | (block->prev & FLAGS_MASK); -} - -static struct block_header *BLOCK_NEXT(struct block_header *block) -{ - return (struct block_header *) - ((char *)block + block->size + XV_ALIGN); -} - -/* - * Get index of free list containing blocks of maximum size - * which is less than or equal to given size. - */ -static u32 get_index_for_insert(u32 size) -{ - if (unlikely(size > XV_MAX_ALLOC_SIZE)) - size = XV_MAX_ALLOC_SIZE; - size &= ~FL_DELTA_MASK; - return (size - XV_MIN_ALLOC_SIZE) >> FL_DELTA_SHIFT; -} - -/* - * Get index of free list having blocks of size greater than - * or equal to requested size. - */ -static u32 get_index(u32 size) -{ - if (unlikely(size < XV_MIN_ALLOC_SIZE)) - size = XV_MIN_ALLOC_SIZE; - size = ALIGN(size, FL_DELTA); - return (size - XV_MIN_ALLOC_SIZE) >> FL_DELTA_SHIFT; -} - -/** - * find_block - find block of at least given size - * @pool: memory pool to search from - * @size: size of block required - * @page: page containing required block - * @offset: offset within the page where block is located. - * - * Searches two level bitmap to locate block of at least - * the given size. If such a block is found, it provides - * <page, offset> to identify this block and returns index - * in freelist where we found this block. - * Otherwise, returns 0 and <page, offset> params are not touched. - */ -static u32 find_block(struct xv_pool *pool, u32 size, - struct page **page, u32 *offset) -{ - ulong flbitmap, slbitmap; - u32 flindex, slindex, slbitstart; - - /* There are no free blocks in this pool */ - if (!pool->flbitmap) - return 0; - - /* Get freelist index correspoding to this size */ - slindex = get_index(size); - slbitmap = pool->slbitmap[slindex / BITS_PER_LONG]; - slbitstart = slindex % BITS_PER_LONG; - - /* - * If freelist is not empty at this index, we found the - * block - head of this list. This is approximate best-fit match. - */ - if (test_bit(slbitstart, &slbitmap)) { - *page = pool->freelist[slindex].page; - *offset = pool->freelist[slindex].offset; - return slindex; - } - - /* - * No best-fit found. Search a bit further in bitmap for a free block. - * Second level bitmap consists of series of 32-bit chunks. Search - * further in the chunk where we expected a best-fit, starting from - * index location found above. - */ - slbitstart++; - slbitmap >>= slbitstart; - - /* Skip this search if we were already at end of this bitmap chunk */ - if ((slbitstart != BITS_PER_LONG) && slbitmap) { - slindex += __ffs(slbitmap) + 1; - *page = pool->freelist[slindex].page; - *offset = pool->freelist[slindex].offset; - return slindex; - } - - /* Now do a full two-level bitmap search to find next nearest fit */ - flindex = slindex / BITS_PER_LONG; - - flbitmap = (pool->flbitmap) >> (flindex + 1); - if (!flbitmap) - return 0; - - flindex += __ffs(flbitmap) + 1; - slbitmap = pool->slbitmap[flindex]; - slindex = (flindex * BITS_PER_LONG) + __ffs(slbitmap); - *page = pool->freelist[slindex].page; - *offset = pool->freelist[slindex].offset; - - return slindex; -} - -/* - * Insert block at <page, offset> in freelist of given pool. - * freelist used depends on block size. - */ -static void insert_block(struct xv_pool *pool, struct page *page, u32 offset, - struct block_header *block) -{ - u32 flindex, slindex; - struct block_header *nextblock; - - slindex = get_index_for_insert(block->size); - flindex = slindex / BITS_PER_LONG; - - block->link.prev_page = NULL; - block->link.prev_offset = 0; - block->link.next_page = pool->freelist[slindex].page; - block->link.next_offset = pool->freelist[slindex].offset; - pool->freelist[slindex].page = page; - pool->freelist[slindex].offset = offset; - - if (block->link.next_page) { - nextblock = get_ptr_atomic(block->link.next_page, - block->link.next_offset, KM_USER1); - nextblock->link.prev_page = page; - nextblock->link.prev_offset = offset; - put_ptr_atomic(nextblock, KM_USER1); - /* If there was a next page then the free bits are set. */ - return; - } - - __set_bit(slindex % BITS_PER_LONG, &pool->slbitmap[flindex]); - __set_bit(flindex, &pool->flbitmap); -} - -/* - * Remove block from freelist. Index 'slindex' identifies the freelist. - */ -static void remove_block(struct xv_pool *pool, struct page *page, u32 offset, - struct block_header *block, u32 slindex) -{ - u32 flindex = slindex / BITS_PER_LONG; - struct block_header *tmpblock; - - if (block->link.prev_page) { - tmpblock = get_ptr_atomic(block->link.prev_page, - block->link.prev_offset, KM_USER1); - tmpblock->link.next_page = block->link.next_page; - tmpblock->link.next_offset = block->link.next_offset; - put_ptr_atomic(tmpblock, KM_USER1); - } - - if (block->link.next_page) { - tmpblock = get_ptr_atomic(block->link.next_page, - block->link.next_offset, KM_USER1); - tmpblock->link.prev_page = block->link.prev_page; - tmpblock->link.prev_offset = block->link.prev_offset; - put_ptr_atomic(tmpblock, KM_USER1); - } - - /* Is this block is at the head of the freelist? */ - if (pool->freelist[slindex].page == page - && pool->freelist[slindex].offset == offset) { - - pool->freelist[slindex].page = block->link.next_page; - pool->freelist[slindex].offset = block->link.next_offset; - - if (pool->freelist[slindex].page) { - struct block_header *tmpblock; - tmpblock = get_ptr_atomic(pool->freelist[slindex].page, - pool->freelist[slindex].offset, - KM_USER1); - tmpblock->link.prev_page = NULL; - tmpblock->link.prev_offset = 0; - put_ptr_atomic(tmpblock, KM_USER1); - } else { - /* This freelist bucket is empty */ - __clear_bit(slindex % BITS_PER_LONG, - &pool->slbitmap[flindex]); - if (!pool->slbitmap[flindex]) - __clear_bit(flindex, &pool->flbitmap); - } - } - - block->link.prev_page = NULL; - block->link.prev_offset = 0; - block->link.next_page = NULL; - block->link.next_offset = 0; -} - -/* - * Allocate a page and add it to freelist of given pool. - */ -static int grow_pool(struct xv_pool *pool, gfp_t flags) -{ - struct page *page; - struct block_header *block; - - page = alloc_page(flags); - if (unlikely(!page)) - return -ENOMEM; - - stat_inc(&pool->total_pages); - - spin_lock(&pool->lock); - block = get_ptr_atomic(page, 0, KM_USER0); - - block->size = PAGE_SIZE - XV_ALIGN; - set_flag(block, BLOCK_FREE); - clear_flag(block, PREV_FREE); - set_blockprev(block, 0); - - insert_block(pool, page, 0, block); - - put_ptr_atomic(block, KM_USER0); - spin_unlock(&pool->lock); - - return 0; -} - -/* - * Create a memory pool. Allocates freelist, bitmaps and other - * per-pool metadata. - */ -struct xv_pool *xv_create_pool(void) -{ - u32 ovhd_size; - struct xv_pool *pool; - - ovhd_size = roundup(sizeof(*pool), PAGE_SIZE); - pool = kzalloc(ovhd_size, GFP_KERNEL); - if (!pool) - return NULL; - - spin_lock_init(&pool->lock); - - return pool; -} -EXPORT_SYMBOL_GPL(xv_create_pool); - -void xv_destroy_pool(struct xv_pool *pool) -{ - kfree(pool); -} -EXPORT_SYMBOL_GPL(xv_destroy_pool); - -/** - * xv_malloc - Allocate block of given size from pool. - * @pool: pool to allocate from - * @size: size of block to allocate - * @page: page no. that holds the object - * @offset: location of object within page - * - * On success, <page, offset> identifies block allocated - * and 0 is returned. On failure, <page, offset> is set to - * 0 and -ENOMEM is returned. - * - * Allocation requests with size > XV_MAX_ALLOC_SIZE will fail. - */ -int xv_malloc(struct xv_pool *pool, u32 size, struct page **page, - u32 *offset, gfp_t flags) -{ - int error; - u32 index, tmpsize, origsize, tmpoffset; - struct block_header *block, *tmpblock; - - *page = NULL; - *offset = 0; - origsize = size; - - if (unlikely(!size || size > XV_MAX_ALLOC_SIZE)) - return -ENOMEM; - - size = ALIGN(size, XV_ALIGN); - - spin_lock(&pool->lock); - - index = find_block(pool, size, page, offset); - - if (!*page) { - spin_unlock(&pool->lock); - if (flags & GFP_NOWAIT) - return -ENOMEM; - error = grow_pool(pool, flags); - if (unlikely(error)) - return error; - - spin_lock(&pool->lock); - index = find_block(pool, size, page, offset); - } - - if (!*page) { - spin_unlock(&pool->lock); - return -ENOMEM; - } - - block = get_ptr_atomic(*page, *offset, KM_USER0); - - remove_block(pool, *page, *offset, block, index); - - /* Split the block if required */ - tmpoffset = *offset + size + XV_ALIGN; - tmpsize = block->size - size; - tmpblock = (struct block_header *)((char *)block + size + XV_ALIGN); - if (tmpsize) { - tmpblock->size = tmpsize - XV_ALIGN; - set_flag(tmpblock, BLOCK_FREE); - clear_flag(tmpblock, PREV_FREE); - - set_blockprev(tmpblock, *offset); - if (tmpblock->size >= XV_MIN_ALLOC_SIZE) - insert_block(pool, *page, tmpoffset, tmpblock); - - if (tmpoffset + XV_ALIGN + tmpblock->size != PAGE_SIZE) { - tmpblock = BLOCK_NEXT(tmpblock); - set_blockprev(tmpblock, tmpoffset); - } - } else { - /* This block is exact fit */ - if (tmpoffset != PAGE_SIZE) - clear_flag(tmpblock, PREV_FREE); - } - - block->size = origsize; - clear_flag(block, BLOCK_FREE); - - put_ptr_atomic(block, KM_USER0); - spin_unlock(&pool->lock); - - *offset += XV_ALIGN; - - return 0; -} -EXPORT_SYMBOL_GPL(xv_malloc); - -/* - * Free block identified with <page, offset> - */ -void xv_free(struct xv_pool *pool, struct page *page, u32 offset) -{ - void *page_start; - struct block_header *block, *tmpblock; - - offset -= XV_ALIGN; - - spin_lock(&pool->lock); - - page_start = get_ptr_atomic(page, 0, KM_USER0); - block = (struct block_header *)((char *)page_start + offset); - - /* Catch double free bugs */ - BUG_ON(test_flag(block, BLOCK_FREE)); - - block->size = ALIGN(block->size, XV_ALIGN); - - tmpblock = BLOCK_NEXT(block); - if (offset + block->size + XV_ALIGN == PAGE_SIZE) - tmpblock = NULL; - - /* Merge next block if its free */ - if (tmpblock && test_flag(tmpblock, BLOCK_FREE)) { - /* - * Blocks smaller than XV_MIN_ALLOC_SIZE - * are not inserted in any free list. - */ - if (tmpblock->size >= XV_MIN_ALLOC_SIZE) { - remove_block(pool, page, - offset + block->size + XV_ALIGN, tmpblock, - get_index_for_insert(tmpblock->size)); - } - block->size += tmpblock->size + XV_ALIGN; - } - - /* Merge previous block if its free */ - if (test_flag(block, PREV_FREE)) { - tmpblock = (struct block_header *)((char *)(page_start) + - get_blockprev(block)); - offset = offset - tmpblock->size - XV_ALIGN; - - if (tmpblock->size >= XV_MIN_ALLOC_SIZE) - remove_block(pool, page, offset, tmpblock, - get_index_for_insert(tmpblock->size)); - - tmpblock->size += block->size + XV_ALIGN; - block = tmpblock; - } - - /* No used objects in this page. Free it. */ - if (block->size == PAGE_SIZE - XV_ALIGN) { - put_ptr_atomic(page_start, KM_USER0); - spin_unlock(&pool->lock); - - __free_page(page); - stat_dec(&pool->total_pages); - return; - } - - set_flag(block, BLOCK_FREE); - if (block->size >= XV_MIN_ALLOC_SIZE) - insert_block(pool, page, offset, block); - - if (offset + block->size + XV_ALIGN != PAGE_SIZE) { - tmpblock = BLOCK_NEXT(block); - set_flag(tmpblock, PREV_FREE); - set_blockprev(tmpblock, offset); - } - - put_ptr_atomic(page_start, KM_USER0); - spin_unlock(&pool->lock); -} -EXPORT_SYMBOL_GPL(xv_free); - -u32 xv_get_object_size(void *obj) -{ - struct block_header *blk; - - blk = (struct block_header *)((char *)(obj) - XV_ALIGN); - return blk->size; -} -EXPORT_SYMBOL_GPL(xv_get_object_size); - -/* - * Returns total memory used by allocator (userdata + metadata) - */ -u64 xv_get_total_size_bytes(struct xv_pool *pool) -{ - return pool->total_pages << PAGE_SHIFT; -} -EXPORT_SYMBOL_GPL(xv_get_total_size_bytes); diff --git a/drivers/staging/zram/xvmalloc.h b/drivers/staging/zram/xvmalloc.h deleted file mode 100644 index 5b1a81aa5faf..000000000000 --- a/drivers/staging/zram/xvmalloc.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * xvmalloc memory allocator - * - * Copyright (C) 2008, 2009, 2010 Nitin Gupta - * - * This code is released using a dual license strategy: BSD/GPL - * You can choose the licence that better fits your requirements. - * - * Released under the terms of 3-clause BSD License - * Released under the terms of GNU General Public License Version 2.0 - */ - -#ifndef _XV_MALLOC_H_ -#define _XV_MALLOC_H_ - -#include <linux/types.h> - -struct xv_pool; - -struct xv_pool *xv_create_pool(void); -void xv_destroy_pool(struct xv_pool *pool); - -int xv_malloc(struct xv_pool *pool, u32 size, struct page **page, - u32 *offset, gfp_t flags); -void xv_free(struct xv_pool *pool, struct page *page, u32 offset); - -u32 xv_get_object_size(void *obj); -u64 xv_get_total_size_bytes(struct xv_pool *pool); - -#endif diff --git a/drivers/staging/zram/xvmalloc_int.h b/drivers/staging/zram/xvmalloc_int.h deleted file mode 100644 index b5f1f7febcf6..000000000000 --- a/drivers/staging/zram/xvmalloc_int.h +++ /dev/null @@ -1,95 +0,0 @@ -/* - * xvmalloc memory allocator - * - * Copyright (C) 2008, 2009, 2010 Nitin Gupta - * - * This code is released using a dual license strategy: BSD/GPL - * You can choose the licence that better fits your requirements. - * - * Released under the terms of 3-clause BSD License - * Released under the terms of GNU General Public License Version 2.0 - */ - -#ifndef _XV_MALLOC_INT_H_ -#define _XV_MALLOC_INT_H_ - -#include <linux/kernel.h> -#include <linux/types.h> - -/* User configurable params */ - -/* Must be power of two */ -#ifdef CONFIG_64BIT -#define XV_ALIGN_SHIFT 3 -#else -#define XV_ALIGN_SHIFT 2 -#endif -#define XV_ALIGN (1 << XV_ALIGN_SHIFT) -#define XV_ALIGN_MASK (XV_ALIGN - 1) - -/* This must be greater than sizeof(link_free) */ -#define XV_MIN_ALLOC_SIZE 32 -#define XV_MAX_ALLOC_SIZE (PAGE_SIZE - XV_ALIGN) - -/* - * Free lists are separated by FL_DELTA bytes - * This value is 3 for 4k pages and 4 for 64k pages, for any - * other page size, a conservative (PAGE_SHIFT - 9) is used. - */ -#if PAGE_SHIFT == 16 -#define FL_DELTA_SHIFT 4 -#else -#define FL_DELTA_SHIFT (PAGE_SHIFT - 9) -#endif -#define FL_DELTA (1 << FL_DELTA_SHIFT) -#define FL_DELTA_MASK (FL_DELTA - 1) -#define NUM_FREE_LISTS ((XV_MAX_ALLOC_SIZE - XV_MIN_ALLOC_SIZE) \ - / FL_DELTA + 1) - -#define MAX_FLI DIV_ROUND_UP(NUM_FREE_LISTS, BITS_PER_LONG) - -/* End of user params */ - -enum blockflags { - BLOCK_FREE, - PREV_FREE, - __NR_BLOCKFLAGS, -}; - -#define FLAGS_MASK XV_ALIGN_MASK -#define PREV_MASK (~FLAGS_MASK) - -struct freelist_entry { - struct page *page; - u16 offset; - u16 pad; -}; - -struct link_free { - struct page *prev_page; - struct page *next_page; - u16 prev_offset; - u16 next_offset; -}; - -struct block_header { - union { - /* This common header must be XV_ALIGN bytes */ - u8 common[XV_ALIGN]; - struct { - u16 size; - u16 prev; - }; - }; - struct link_free link; -}; - -struct xv_pool { - ulong flbitmap; - ulong slbitmap[MAX_FLI]; - u64 total_pages; /* stats */ - struct freelist_entry freelist[NUM_FREE_LISTS]; - spinlock_t lock; -}; - -#endif diff --git a/drivers/staging/zram/zram_drv.c b/drivers/staging/zram/zram_drv.c index 2a2a92d389e6..685d612a627b 100644 --- a/drivers/staging/zram/zram_drv.c +++ b/drivers/staging/zram/zram_drv.c @@ -40,7 +40,7 @@ static int zram_major; struct zram *zram_devices; /* Module params (documentation at end) */ -unsigned int zram_num_devices; +static unsigned int num_devices; static void zram_stat_inc(u32 *v) { @@ -135,13 +135,9 @@ static void zram_set_disksize(struct zram *zram, size_t totalram_bytes) static void zram_free_page(struct zram *zram, size_t index) { - u32 clen; - void *obj; + void *handle = zram->table[index].handle; - struct page *page = zram->table[index].page; - u32 offset = zram->table[index].offset; - - if (unlikely(!page)) { + if (unlikely(!handle)) { /* * No memory is allocated for zero filled pages. * Simply clear zero page flag. @@ -154,27 +150,24 @@ static void zram_free_page(struct zram *zram, size_t index) } if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) { - clen = PAGE_SIZE; - __free_page(page); + __free_page(handle); zram_clear_flag(zram, index, ZRAM_UNCOMPRESSED); zram_stat_dec(&zram->stats.pages_expand); goto out; } - obj = kmap_atomic(page, KM_USER0) + offset; - clen = xv_get_object_size(obj) - sizeof(struct zobj_header); - kunmap_atomic(obj, KM_USER0); + zs_free(zram->mem_pool, handle); - xv_free(zram->mem_pool, page, offset); - if (clen <= PAGE_SIZE / 2) + if (zram->table[index].size <= PAGE_SIZE / 2) zram_stat_dec(&zram->stats.good_compress); out: - zram_stat64_sub(zram, &zram->stats.compr_size, clen); + zram_stat64_sub(zram, &zram->stats.compr_size, + zram->table[index].size); zram_stat_dec(&zram->stats.pages_stored); - zram->table[index].page = NULL; - zram->table[index].offset = 0; + zram->table[index].handle = NULL; + zram->table[index].size = 0; } static void handle_zero_page(struct bio_vec *bvec) @@ -182,9 +175,9 @@ static void handle_zero_page(struct bio_vec *bvec) struct page *page = bvec->bv_page; void *user_mem; - user_mem = kmap_atomic(page, KM_USER0); + user_mem = kmap_atomic(page); memset(user_mem + bvec->bv_offset, 0, bvec->bv_len); - kunmap_atomic(user_mem, KM_USER0); + kunmap_atomic(user_mem); flush_dcache_page(page); } @@ -195,12 +188,12 @@ static void handle_uncompressed_page(struct zram *zram, struct bio_vec *bvec, struct page *page = bvec->bv_page; unsigned char *user_mem, *cmem; - user_mem = kmap_atomic(page, KM_USER0); - cmem = kmap_atomic(zram->table[index].page, KM_USER1); + user_mem = kmap_atomic(page); + cmem = kmap_atomic(zram->table[index].handle); memcpy(user_mem + bvec->bv_offset, cmem + offset, bvec->bv_len); - kunmap_atomic(cmem, KM_USER1); - kunmap_atomic(user_mem, KM_USER0); + kunmap_atomic(cmem); + kunmap_atomic(user_mem); flush_dcache_page(page); } @@ -227,7 +220,7 @@ static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, } /* Requested page is not present in compressed area */ - if (unlikely(!zram->table[index].page)) { + if (unlikely(!zram->table[index].handle)) { pr_debug("Read before write: sector=%lu, size=%u", (ulong)(bio->bi_sector), bio->bi_size); handle_zero_page(bvec); @@ -249,16 +242,15 @@ static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, } } - user_mem = kmap_atomic(page, KM_USER0); + user_mem = kmap_atomic(page); if (!is_partial_io(bvec)) uncmem = user_mem; clen = PAGE_SIZE; - cmem = kmap_atomic(zram->table[index].page, KM_USER1) + - zram->table[index].offset; + cmem = zs_map_object(zram->mem_pool, zram->table[index].handle); ret = lzo1x_decompress_safe(cmem + sizeof(*zheader), - xv_get_object_size(cmem) - sizeof(*zheader), + zram->table[index].size, uncmem, &clen); if (is_partial_io(bvec)) { @@ -267,8 +259,8 @@ static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, kfree(uncmem); } - kunmap_atomic(cmem, KM_USER1); - kunmap_atomic(user_mem, KM_USER0); + zs_unmap_object(zram->mem_pool, zram->table[index].handle); + kunmap_atomic(user_mem); /* Should NEVER happen. Return bio error if it does. */ if (unlikely(ret != LZO_E_OK)) { @@ -290,25 +282,24 @@ static int zram_read_before_write(struct zram *zram, char *mem, u32 index) unsigned char *cmem; if (zram_test_flag(zram, index, ZRAM_ZERO) || - !zram->table[index].page) { + !zram->table[index].handle) { memset(mem, 0, PAGE_SIZE); return 0; } - cmem = kmap_atomic(zram->table[index].page, KM_USER0) + - zram->table[index].offset; + cmem = zs_map_object(zram->mem_pool, zram->table[index].handle); /* Page is stored uncompressed since it's incompressible */ if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) { memcpy(mem, cmem, PAGE_SIZE); - kunmap_atomic(cmem, KM_USER0); + kunmap_atomic(cmem); return 0; } ret = lzo1x_decompress_safe(cmem + sizeof(*zheader), - xv_get_object_size(cmem) - sizeof(*zheader), + zram->table[index].size, mem, &clen); - kunmap_atomic(cmem, KM_USER0); + zs_unmap_object(zram->mem_pool, zram->table[index].handle); /* Should NEVER happen. Return bio error if it does. */ if (unlikely(ret != LZO_E_OK)) { @@ -326,6 +317,7 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index, int ret; u32 store_offset; size_t clen; + void *handle; struct zobj_header *zheader; struct page *page, *page_store; unsigned char *user_mem, *cmem, *src, *uncmem = NULL; @@ -355,11 +347,11 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index, * System overwrites unused sectors. Free memory associated * with this sector now. */ - if (zram->table[index].page || + if (zram->table[index].handle || zram_test_flag(zram, index, ZRAM_ZERO)) zram_free_page(zram, index); - user_mem = kmap_atomic(page, KM_USER0); + user_mem = kmap_atomic(page); if (is_partial_io(bvec)) memcpy(uncmem + offset, user_mem + bvec->bv_offset, @@ -368,7 +360,7 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index, uncmem = user_mem; if (page_zero_filled(uncmem)) { - kunmap_atomic(user_mem, KM_USER0); + kunmap_atomic(user_mem); if (is_partial_io(bvec)) kfree(uncmem); zram_stat_inc(&zram->stats.pages_zero); @@ -380,7 +372,7 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index, ret = lzo1x_1_compress(uncmem, PAGE_SIZE, src, &clen, zram->compress_workmem); - kunmap_atomic(user_mem, KM_USER0); + kunmap_atomic(user_mem); if (is_partial_io(bvec)) kfree(uncmem); @@ -407,26 +399,22 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index, store_offset = 0; zram_set_flag(zram, index, ZRAM_UNCOMPRESSED); zram_stat_inc(&zram->stats.pages_expand); - zram->table[index].page = page_store; - src = kmap_atomic(page, KM_USER0); + handle = page_store; + src = kmap_atomic(page); + cmem = kmap_atomic(page_store); goto memstore; } - if (xv_malloc(zram->mem_pool, clen + sizeof(*zheader), - &zram->table[index].page, &store_offset, - GFP_NOIO | __GFP_HIGHMEM)) { + handle = zs_malloc(zram->mem_pool, clen + sizeof(*zheader)); + if (!handle) { pr_info("Error allocating memory for compressed " "page: %u, size=%zu\n", index, clen); ret = -ENOMEM; goto out; } + cmem = zs_map_object(zram->mem_pool, handle); memstore: - zram->table[index].offset = store_offset; - - cmem = kmap_atomic(zram->table[index].page, KM_USER1) + - zram->table[index].offset; - #if 0 /* Back-reference needed for memory defragmentation */ if (!zram_test_flag(zram, index, ZRAM_UNCOMPRESSED)) { @@ -438,9 +426,15 @@ memstore: memcpy(cmem, src, clen); - kunmap_atomic(cmem, KM_USER1); - if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) - kunmap_atomic(src, KM_USER0); + if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) { + kunmap_atomic(cmem); + kunmap_atomic(src); + } else { + zs_unmap_object(zram->mem_pool, handle); + } + + zram->table[index].handle = handle; + zram->table[index].size = clen; /* Update stats */ zram_stat64_add(zram, &zram->stats.compr_size, clen); @@ -598,25 +592,20 @@ void __zram_reset_device(struct zram *zram) /* Free all pages that are still in this zram device */ for (index = 0; index < zram->disksize >> PAGE_SHIFT; index++) { - struct page *page; - u16 offset; - - page = zram->table[index].page; - offset = zram->table[index].offset; - - if (!page) + void *handle = zram->table[index].handle; + if (!handle) continue; if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) - __free_page(page); + __free_page(handle); else - xv_free(zram->mem_pool, page, offset); + zs_free(zram->mem_pool, handle); } vfree(zram->table); zram->table = NULL; - xv_destroy_pool(zram->mem_pool); + zs_destroy_pool(zram->mem_pool); zram->mem_pool = NULL; /* Reset stats */ @@ -674,7 +663,7 @@ int zram_init_device(struct zram *zram) /* zram devices sort of resembles non-rotational disks */ queue_flag_set_unlocked(QUEUE_FLAG_NONROT, zram->disk->queue); - zram->mem_pool = xv_create_pool(); + zram->mem_pool = zs_create_pool("zram", GFP_NOIO | __GFP_HIGHMEM); if (!zram->mem_pool) { pr_err("Error creating memory pool\n"); ret = -ENOMEM; @@ -790,13 +779,18 @@ static void destroy_device(struct zram *zram) blk_cleanup_queue(zram->queue); } +unsigned int zram_get_num_devices(void) +{ + return num_devices; +} + static int __init zram_init(void) { int ret, dev_id; - if (zram_num_devices > max_num_devices) { + if (num_devices > max_num_devices) { pr_warning("Invalid value for num_devices: %u\n", - zram_num_devices); + num_devices); ret = -EINVAL; goto out; } @@ -808,20 +802,20 @@ static int __init zram_init(void) goto out; } - if (!zram_num_devices) { + if (!num_devices) { pr_info("num_devices not specified. Using default: 1\n"); - zram_num_devices = 1; + num_devices = 1; } /* Allocate the device array and initialize each one */ - pr_info("Creating %u devices ...\n", zram_num_devices); - zram_devices = kzalloc(zram_num_devices * sizeof(struct zram), GFP_KERNEL); + pr_info("Creating %u devices ...\n", num_devices); + zram_devices = kzalloc(num_devices * sizeof(struct zram), GFP_KERNEL); if (!zram_devices) { ret = -ENOMEM; goto unregister; } - for (dev_id = 0; dev_id < zram_num_devices; dev_id++) { + for (dev_id = 0; dev_id < num_devices; dev_id++) { ret = create_device(&zram_devices[dev_id], dev_id); if (ret) goto free_devices; @@ -844,7 +838,7 @@ static void __exit zram_exit(void) int i; struct zram *zram; - for (i = 0; i < zram_num_devices; i++) { + for (i = 0; i < num_devices; i++) { zram = &zram_devices[i]; destroy_device(zram); @@ -858,8 +852,8 @@ static void __exit zram_exit(void) pr_debug("Cleanup done!\n"); } -module_param(zram_num_devices, uint, 0); -MODULE_PARM_DESC(zram_num_devices, "Number of zram devices"); +module_param(num_devices, uint, 0); +MODULE_PARM_DESC(num_devices, "Number of zram devices"); module_init(zram_init); module_exit(zram_exit); diff --git a/drivers/staging/zram/zram_drv.h b/drivers/staging/zram/zram_drv.h index e5cd2469b6a0..fbe8ac98704c 100644 --- a/drivers/staging/zram/zram_drv.h +++ b/drivers/staging/zram/zram_drv.h @@ -18,7 +18,7 @@ #include <linux/spinlock.h> #include <linux/mutex.h> -#include "xvmalloc.h" +#include "../zsmalloc/zsmalloc.h" /* * Some arbitrary value. This is just to catch @@ -51,7 +51,7 @@ static const size_t max_zpage_size = PAGE_SIZE / 4 * 3; /* * NOTE: max_zpage_size must be less than or equal to: - * XV_MAX_ALLOC_SIZE - sizeof(struct zobj_header) + * ZS_MAX_ALLOC_SIZE - sizeof(struct zobj_header) * otherwise, xv_malloc() would always return failure. */ @@ -81,8 +81,8 @@ enum zram_pageflags { /* Allocated for each disk page */ struct table { - struct page *page; - u16 offset; + void *handle; + u16 size; /* object size (excluding header) */ u8 count; /* object ref count (not yet used) */ u8 flags; } __attribute__((aligned(4))); @@ -102,7 +102,7 @@ struct zram_stats { }; struct zram { - struct xv_pool *mem_pool; + struct zs_pool *mem_pool; void *compress_workmem; void *compress_buffer; struct table *table; @@ -124,7 +124,7 @@ struct zram { }; extern struct zram *zram_devices; -extern unsigned int zram_num_devices; +unsigned int zram_get_num_devices(void); #ifdef CONFIG_SYSFS extern struct attribute_group zram_disk_attr_group; #endif diff --git a/drivers/staging/zram/zram_sysfs.c b/drivers/staging/zram/zram_sysfs.c index d521122826f6..a7f377175525 100644 --- a/drivers/staging/zram/zram_sysfs.c +++ b/drivers/staging/zram/zram_sysfs.c @@ -34,7 +34,7 @@ static struct zram *dev_to_zram(struct device *dev) int i; struct zram *zram = NULL; - for (i = 0; i < zram_num_devices; i++) { + for (i = 0; i < zram_get_num_devices(); i++) { zram = &zram_devices[i]; if (disk_to_dev(zram->disk) == dev) break; @@ -187,7 +187,7 @@ static ssize_t mem_used_total_show(struct device *dev, struct zram *zram = dev_to_zram(dev); if (zram->init_done) { - val = xv_get_total_size_bytes(zram->mem_pool) + + val = zs_get_total_size_bytes(zram->mem_pool) + ((u64)(zram->stats.pages_expand) << PAGE_SHIFT); } |