diff options
Diffstat (limited to 'drivers/staging/zcache/zcache-main.c')
| -rw-r--r-- | drivers/staging/zcache/zcache-main.c | 1943 |
1 files changed, 0 insertions, 1943 deletions
diff --git a/drivers/staging/zcache/zcache-main.c b/drivers/staging/zcache/zcache-main.c deleted file mode 100644 index 81972fa47beb..000000000000 --- a/drivers/staging/zcache/zcache-main.c +++ /dev/null @@ -1,1943 +0,0 @@ -/* - * zcache.c - * - * Copyright (c) 2010-2012, Dan Magenheimer, Oracle Corp. - * Copyright (c) 2010,2011, Nitin Gupta - * - * Zcache provides an in-kernel "host implementation" for transcendent memory - * ("tmem") and, thus indirectly, for cleancache and frontswap. Zcache uses - * lzo1x compression to improve density and an embedded allocator called - * "zbud" which "buddies" two compressed pages semi-optimally in each physical - * pageframe. Zbud is integrally tied into tmem to allow pageframes to - * be "reclaimed" efficiently. - */ - -#include <linux/module.h> -#include <linux/cpu.h> -#include <linux/highmem.h> -#include <linux/list.h> -#include <linux/slab.h> -#include <linux/spinlock.h> -#include <linux/types.h> -#include <linux/string.h> -#include <linux/atomic.h> -#include <linux/math64.h> -#include <linux/crypto.h> -#include <linux/swap.h> -#include <linux/swapops.h> -#include <linux/pagemap.h> -#include <linux/writeback.h> - -#include <linux/cleancache.h> -#include <linux/frontswap.h> -#include "tmem.h" -#include "zcache.h" -#include "zbud.h" -#include "ramster.h" -#include "debug.h" -#ifdef CONFIG_RAMSTER -static bool ramster_enabled __read_mostly; -static int disable_frontswap_selfshrink; -#else -#define ramster_enabled false -#define disable_frontswap_selfshrink 0 -#endif - -#ifndef __PG_WAS_ACTIVE -static inline bool PageWasActive(struct page *page) -{ - return true; -} - -static inline void SetPageWasActive(struct page *page) -{ -} -#endif - -#ifdef FRONTSWAP_HAS_EXCLUSIVE_GETS -static bool frontswap_has_exclusive_gets __read_mostly = true; -#else -static bool frontswap_has_exclusive_gets __read_mostly; -static inline void frontswap_tmem_exclusive_gets(bool b) -{ -} -#endif - -/* - * mark pampd to special value in order that later - * retrieve will identify zero-filled pages - */ -#define ZERO_FILLED 0x2 - -/* enable (or fix code) when Seth's patches are accepted upstream */ -#define zcache_writeback_enabled 0 - -static bool zcache_enabled __read_mostly; -static bool disable_cleancache __read_mostly; -static bool disable_frontswap __read_mostly; -static bool disable_frontswap_ignore_nonactive __read_mostly; -static bool disable_cleancache_ignore_nonactive __read_mostly; -static char *namestr __read_mostly = "zcache"; - -#define ZCACHE_GFP_MASK \ - (__GFP_FS | __GFP_NORETRY | __GFP_NOWARN | __GFP_NOMEMALLOC) - -/* crypto API for zcache */ -#ifdef CONFIG_ZCACHE_MODULE -static char *zcache_comp_name = "lzo"; -#else -#define ZCACHE_COMP_NAME_SZ CRYPTO_MAX_ALG_NAME -static char zcache_comp_name[ZCACHE_COMP_NAME_SZ] __read_mostly; -#endif -static struct crypto_comp * __percpu *zcache_comp_pcpu_tfms __read_mostly; - -enum comp_op { - ZCACHE_COMPOP_COMPRESS, - ZCACHE_COMPOP_DECOMPRESS -}; - -static inline int zcache_comp_op(enum comp_op op, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen) -{ - struct crypto_comp *tfm; - int ret = -1; - - BUG_ON(!zcache_comp_pcpu_tfms); - tfm = *per_cpu_ptr(zcache_comp_pcpu_tfms, get_cpu()); - BUG_ON(!tfm); - switch (op) { - case ZCACHE_COMPOP_COMPRESS: - ret = crypto_comp_compress(tfm, src, slen, dst, dlen); - break; - case ZCACHE_COMPOP_DECOMPRESS: - ret = crypto_comp_decompress(tfm, src, slen, dst, dlen); - break; - default: - ret = -EINVAL; - } - put_cpu(); - return ret; -} - -/* - * policy parameters - */ - -/* - * byte count defining poor compression; pages with greater zsize will be - * rejected - */ -static unsigned int zbud_max_zsize __read_mostly = (PAGE_SIZE / 8) * 7; -/* - * byte count defining poor *mean* compression; pages with greater zsize - * will be rejected until sufficient better-compressed pages are accepted - * driving the mean below this threshold - */ -static unsigned int zbud_max_mean_zsize __read_mostly = (PAGE_SIZE / 8) * 5; - -/* - * for now, used named slabs so can easily track usage; later can - * either just use kmalloc, or perhaps add a slab-like allocator - * to more carefully manage total memory utilization - */ -static struct kmem_cache *zcache_objnode_cache; -static struct kmem_cache *zcache_obj_cache; - -static DEFINE_PER_CPU(struct zcache_preload, zcache_preloads) = { 0, }; - -/* Used by debug.c */ -ssize_t zcache_pers_zpages; -u64 zcache_pers_zbytes; -ssize_t zcache_eph_pageframes; -ssize_t zcache_pers_pageframes; - -/* Used by this code. */ -ssize_t zcache_last_active_file_pageframes; -ssize_t zcache_last_inactive_file_pageframes; -ssize_t zcache_last_active_anon_pageframes; -ssize_t zcache_last_inactive_anon_pageframes; -#ifdef CONFIG_ZCACHE_WRITEBACK -ssize_t zcache_writtenback_pages; -ssize_t zcache_outstanding_writeback_pages; -#endif -/* - * zcache core code starts here - */ - -static struct zcache_client zcache_host; -static struct zcache_client zcache_clients[MAX_CLIENTS]; - -static inline bool is_local_client(struct zcache_client *cli) -{ - return cli == &zcache_host; -} - -static struct zcache_client *zcache_get_client_by_id(uint16_t cli_id) -{ - struct zcache_client *cli = &zcache_host; - - if (cli_id != LOCAL_CLIENT) { - if (cli_id >= MAX_CLIENTS) - goto out; - cli = &zcache_clients[cli_id]; - } -out: - return cli; -} - -/* - * Tmem operations assume the poolid implies the invoking client. - * Zcache only has one client (the kernel itself): LOCAL_CLIENT. - * RAMster has each client numbered by cluster node, and a KVM version - * of zcache would have one client per guest and each client might - * have a poolid==N. - */ -struct tmem_pool *zcache_get_pool_by_id(uint16_t cli_id, uint16_t poolid) -{ - struct tmem_pool *pool = NULL; - struct zcache_client *cli = NULL; - - cli = zcache_get_client_by_id(cli_id); - if (cli == NULL) - goto out; - if (!is_local_client(cli)) - atomic_inc(&cli->refcount); - if (poolid < MAX_POOLS_PER_CLIENT) { - pool = cli->tmem_pools[poolid]; - if (pool != NULL) - atomic_inc(&pool->refcount); - } -out: - return pool; -} - -void zcache_put_pool(struct tmem_pool *pool) -{ - struct zcache_client *cli = NULL; - - if (pool == NULL) - BUG(); - cli = pool->client; - atomic_dec(&pool->refcount); - if (!is_local_client(cli)) - atomic_dec(&cli->refcount); -} - -int zcache_new_client(uint16_t cli_id) -{ - struct zcache_client *cli; - int ret = -1; - - cli = zcache_get_client_by_id(cli_id); - if (cli == NULL) - goto out; - if (cli->allocated) - goto out; - cli->allocated = 1; - ret = 0; -out: - return ret; -} - -/* - * zcache implementation for tmem host ops - */ - -static struct tmem_objnode *zcache_objnode_alloc(struct tmem_pool *pool) -{ - struct tmem_objnode *objnode = NULL; - struct zcache_preload *kp; - int i; - - kp = &__get_cpu_var(zcache_preloads); - for (i = 0; i < ARRAY_SIZE(kp->objnodes); i++) { - objnode = kp->objnodes[i]; - if (objnode != NULL) { - kp->objnodes[i] = NULL; - break; - } - } - BUG_ON(objnode == NULL); - inc_zcache_objnode_count(); - return objnode; -} - -static void zcache_objnode_free(struct tmem_objnode *objnode, - struct tmem_pool *pool) -{ - dec_zcache_objnode_count(); - kmem_cache_free(zcache_objnode_cache, objnode); -} - -static struct tmem_obj *zcache_obj_alloc(struct tmem_pool *pool) -{ - struct tmem_obj *obj = NULL; - struct zcache_preload *kp; - - kp = &__get_cpu_var(zcache_preloads); - obj = kp->obj; - BUG_ON(obj == NULL); - kp->obj = NULL; - inc_zcache_obj_count(); - return obj; -} - -static void zcache_obj_free(struct tmem_obj *obj, struct tmem_pool *pool) -{ - dec_zcache_obj_count(); - kmem_cache_free(zcache_obj_cache, obj); -} - -/* - * Compressing zero-filled pages will waste memory and introduce - * serious fragmentation, skip it to avoid overhead. - */ -static bool page_is_zero_filled(struct page *p) -{ - unsigned int pos; - char *page; - - page = kmap_atomic(p); - for (pos = 0; pos < PAGE_SIZE / sizeof(*page); pos++) { - if (page[pos]) { - kunmap_atomic(page); - return false; - } - } - kunmap_atomic(page); - - return true; -} - -static void handle_zero_filled_page(void *p) -{ - void *user_mem; - struct page *page = (struct page *)p; - - user_mem = kmap_atomic(page); - memset(user_mem, 0, PAGE_SIZE); - kunmap_atomic(user_mem); - - flush_dcache_page(page); -} - -static struct tmem_hostops zcache_hostops = { - .obj_alloc = zcache_obj_alloc, - .obj_free = zcache_obj_free, - .objnode_alloc = zcache_objnode_alloc, - .objnode_free = zcache_objnode_free, -}; - -static struct page *zcache_alloc_page(void) -{ - struct page *page = alloc_page(ZCACHE_GFP_MASK); - - if (page != NULL) - inc_zcache_pageframes_alloced(); - return page; -} - -static void zcache_free_page(struct page *page) -{ - long curr_pageframes; - static long max_pageframes, min_pageframes; - - if (page == NULL) - BUG(); - __free_page(page); - inc_zcache_pageframes_freed(); - curr_pageframes = curr_pageframes_count(); - if (curr_pageframes > max_pageframes) - max_pageframes = curr_pageframes; - if (curr_pageframes < min_pageframes) - min_pageframes = curr_pageframes; -#ifdef CONFIG_ZCACHE_DEBUG - if (curr_pageframes > 2L || curr_pageframes < -2L) { - /* pr_info here */ - } -#endif -} - -/* - * zcache implementations for PAM page descriptor ops - */ - -/* forward reference */ -static void zcache_compress(struct page *from, - void **out_va, unsigned *out_len); - -static struct page *zcache_evict_eph_pageframe(void); - -static void *zcache_pampd_eph_create(char *data, size_t size, bool raw, - struct tmem_handle *th) -{ - void *pampd = NULL, *cdata = data; - unsigned clen = size; - bool zero_filled = false; - struct page *page = (struct page *)(data), *newpage; - - if (page_is_zero_filled(page)) { - clen = 0; - zero_filled = true; - inc_zcache_zero_filled_pages(); - goto got_pampd; - } - - if (!raw) { - zcache_compress(page, &cdata, &clen); - if (clen > zbud_max_buddy_size()) { - inc_zcache_compress_poor(); - goto out; - } - } else { - BUG_ON(clen > zbud_max_buddy_size()); - } - - /* look for space via an existing match first */ - pampd = (void *)zbud_match_prep(th, true, cdata, clen); - if (pampd != NULL) - goto got_pampd; - - /* no match, now we need to find (or free up) a full page */ - newpage = zcache_alloc_page(); - if (newpage != NULL) - goto create_in_new_page; - - inc_zcache_failed_getfreepages(); - /* can't allocate a page, evict an ephemeral page via LRU */ - newpage = zcache_evict_eph_pageframe(); - if (newpage == NULL) { - inc_zcache_eph_ate_tail_failed(); - goto out; - } - inc_zcache_eph_ate_tail(); - -create_in_new_page: - pampd = (void *)zbud_create_prep(th, true, cdata, clen, newpage); - BUG_ON(pampd == NULL); - inc_zcache_eph_pageframes(); - -got_pampd: - inc_zcache_eph_zbytes(clen); - inc_zcache_eph_zpages(); - if (ramster_enabled && raw && !zero_filled) - ramster_count_foreign_pages(true, 1); - if (zero_filled) - pampd = (void *)ZERO_FILLED; -out: - return pampd; -} - -static void *zcache_pampd_pers_create(char *data, size_t size, bool raw, - struct tmem_handle *th) -{ - void *pampd = NULL, *cdata = data; - unsigned clen = size; - bool zero_filled = false; - struct page *page = (struct page *)(data), *newpage; - unsigned long zbud_mean_zsize; - unsigned long curr_pers_zpages, total_zsize; - - if (data == NULL) { - BUG_ON(!ramster_enabled); - goto create_pampd; - } - - if (page_is_zero_filled(page)) { - clen = 0; - zero_filled = true; - inc_zcache_zero_filled_pages(); - goto got_pampd; - } - - curr_pers_zpages = zcache_pers_zpages; -/* FIXME CONFIG_RAMSTER... subtract atomic remote_pers_pages here? */ - if (!raw) - zcache_compress(page, &cdata, &clen); - /* reject if compression is too poor */ - if (clen > zbud_max_zsize) { - inc_zcache_compress_poor(); - goto out; - } - /* reject if mean compression is too poor */ - if ((clen > zbud_max_mean_zsize) && (curr_pers_zpages > 0)) { - total_zsize = zcache_pers_zbytes; - if ((long)total_zsize < 0) - total_zsize = 0; - zbud_mean_zsize = div_u64(total_zsize, - curr_pers_zpages); - if (zbud_mean_zsize > zbud_max_mean_zsize) { - inc_zcache_mean_compress_poor(); - goto out; - } - } - -create_pampd: - /* look for space via an existing match first */ - pampd = (void *)zbud_match_prep(th, false, cdata, clen); - if (pampd != NULL) - goto got_pampd; - - /* no match, now we need to find (or free up) a full page */ - newpage = zcache_alloc_page(); - if (newpage != NULL) - goto create_in_new_page; - /* - * FIXME do the following only if eph is oversized? - * if (zcache_eph_pageframes > - * (global_page_state(NR_LRU_BASE + LRU_ACTIVE_FILE) + - * global_page_state(NR_LRU_BASE + LRU_INACTIVE_FILE))) - */ - inc_zcache_failed_getfreepages(); - /* can't allocate a page, evict an ephemeral page via LRU */ - newpage = zcache_evict_eph_pageframe(); - if (newpage == NULL) { - inc_zcache_pers_ate_eph_failed(); - goto out; - } - inc_zcache_pers_ate_eph(); - -create_in_new_page: - pampd = (void *)zbud_create_prep(th, false, cdata, clen, newpage); - BUG_ON(pampd == NULL); - inc_zcache_pers_pageframes(); - -got_pampd: - inc_zcache_pers_zpages(); - inc_zcache_pers_zbytes(clen); - if (ramster_enabled && raw && !zero_filled) - ramster_count_foreign_pages(false, 1); - if (zero_filled) - pampd = (void *)ZERO_FILLED; -out: - return pampd; -} - -/* - * This is called directly from zcache_put_page to pre-allocate space - * to store a zpage. - */ -void *zcache_pampd_create(char *data, unsigned int size, bool raw, - int eph, struct tmem_handle *th) -{ - void *pampd = NULL; - struct zcache_preload *kp; - struct tmem_objnode *objnode; - struct tmem_obj *obj; - int i; - - BUG_ON(!irqs_disabled()); - /* pre-allocate per-cpu metadata */ - BUG_ON(zcache_objnode_cache == NULL); - BUG_ON(zcache_obj_cache == NULL); - kp = &__get_cpu_var(zcache_preloads); - for (i = 0; i < ARRAY_SIZE(kp->objnodes); i++) { - objnode = kp->objnodes[i]; - if (objnode == NULL) { - objnode = kmem_cache_alloc(zcache_objnode_cache, - ZCACHE_GFP_MASK); - if (unlikely(objnode == NULL)) { - inc_zcache_failed_alloc(); - goto out; - } - kp->objnodes[i] = objnode; - } - } - if (kp->obj == NULL) { - obj = kmem_cache_alloc(zcache_obj_cache, ZCACHE_GFP_MASK); - kp->obj = obj; - } - if (unlikely(kp->obj == NULL)) { - inc_zcache_failed_alloc(); - goto out; - } - /* - * ok, have all the metadata pre-allocated, now do the data - * but since how we allocate the data is dependent on ephemeral - * or persistent, we split the call here to different sub-functions - */ - if (eph) - pampd = zcache_pampd_eph_create(data, size, raw, th); - else - pampd = zcache_pampd_pers_create(data, size, raw, th); -out: - return pampd; -} - -/* - * This is a pamops called via tmem_put and is necessary to "finish" - * a pampd creation. - */ -void zcache_pampd_create_finish(void *pampd, bool eph) -{ - if (pampd != (void *)ZERO_FILLED) - zbud_create_finish((struct zbudref *)pampd, eph); -} - -/* - * This is passed as a function parameter to zbud_decompress so that - * zbud need not be familiar with the details of crypto. It assumes that - * the bytes from_va and to_va through from_va+size-1 and to_va+size-1 are - * kmapped. It must be successful, else there is a logic bug somewhere. - */ -static void zcache_decompress(char *from_va, unsigned int size, char *to_va) -{ - int ret; - unsigned int outlen = PAGE_SIZE; - - ret = zcache_comp_op(ZCACHE_COMPOP_DECOMPRESS, from_va, size, - to_va, &outlen); - BUG_ON(ret); - BUG_ON(outlen != PAGE_SIZE); -} - -/* - * Decompress from the kernel va to a pageframe - */ -void zcache_decompress_to_page(char *from_va, unsigned int size, - struct page *to_page) -{ - char *to_va = kmap_atomic(to_page); - zcache_decompress(from_va, size, to_va); - kunmap_atomic(to_va); -} - -/* - * fill the pageframe corresponding to the struct page with the data - * from the passed pampd - */ -static int zcache_pampd_get_data(char *data, size_t *sizep, bool raw, - void *pampd, struct tmem_pool *pool, - struct tmem_oid *oid, uint32_t index) -{ - int ret; - bool eph = !is_persistent(pool); - - BUG_ON(preemptible()); - BUG_ON(eph); /* fix later if shared pools get implemented */ - BUG_ON(pampd_is_remote(pampd)); - - if (pampd == (void *)ZERO_FILLED) { - handle_zero_filled_page(data); - if (!raw) - *sizep = PAGE_SIZE; - return 0; - } - - if (raw) - ret = zbud_copy_from_zbud(data, (struct zbudref *)pampd, - sizep, eph); - else { - ret = zbud_decompress((struct page *)(data), - (struct zbudref *)pampd, false, - zcache_decompress); - *sizep = PAGE_SIZE; - } - return ret; -} - -/* - * fill the pageframe corresponding to the struct page with the data - * from the passed pampd - */ -static int zcache_pampd_get_data_and_free(char *data, size_t *sizep, bool raw, - void *pampd, struct tmem_pool *pool, - struct tmem_oid *oid, uint32_t index) -{ - int ret = 0; - bool eph = !is_persistent(pool), zero_filled = false; - struct page *page = NULL; - unsigned int zsize, zpages; - - BUG_ON(preemptible()); - BUG_ON(pampd_is_remote(pampd)); - - if (pampd == (void *)ZERO_FILLED) { - handle_zero_filled_page(data); - zero_filled = true; - zsize = 0; - zpages = 1; - if (!raw) - *sizep = PAGE_SIZE; - dec_zcache_zero_filled_pages(); - goto zero_fill; - } - - if (raw) - ret = zbud_copy_from_zbud(data, (struct zbudref *)pampd, - sizep, eph); - else { - ret = zbud_decompress((struct page *)(data), - (struct zbudref *)pampd, eph, - zcache_decompress); - *sizep = PAGE_SIZE; - } - page = zbud_free_and_delist((struct zbudref *)pampd, eph, - &zsize, &zpages); -zero_fill: - if (eph) { - if (page) - dec_zcache_eph_pageframes(); - dec_zcache_eph_zpages(zpages); - dec_zcache_eph_zbytes(zsize); - } else { - if (page) - dec_zcache_pers_pageframes(); - dec_zcache_pers_zpages(zpages); - dec_zcache_pers_zbytes(zsize); - } - if (!is_local_client(pool->client) && !zero_filled) - ramster_count_foreign_pages(eph, -1); - if (page && !zero_filled) - zcache_free_page(page); - return ret; -} - -/* - * free the pampd and remove it from any zcache lists - * pampd must no longer be pointed to from any tmem data structures! - */ -static void zcache_pampd_free(void *pampd, struct tmem_pool *pool, - struct tmem_oid *oid, uint32_t index, bool acct) -{ - struct page *page = NULL; - unsigned int zsize, zpages; - bool zero_filled = false; - - BUG_ON(preemptible()); - - if (pampd == (void *)ZERO_FILLED) { - zero_filled = true; - zsize = 0; - zpages = 1; - dec_zcache_zero_filled_pages(); - } - - if (pampd_is_remote(pampd) && !zero_filled) { - BUG_ON(!ramster_enabled); - pampd = ramster_pampd_free(pampd, pool, oid, index, acct); - if (pampd == NULL) - return; - } - if (is_ephemeral(pool)) { - if (!zero_filled) - page = zbud_free_and_delist((struct zbudref *)pampd, - true, &zsize, &zpages); - if (page) - dec_zcache_eph_pageframes(); - dec_zcache_eph_zpages(zpages); - dec_zcache_eph_zbytes(zsize); - /* FIXME CONFIG_RAMSTER... check acct parameter? */ - } else { - if (!zero_filled) - page = zbud_free_and_delist((struct zbudref *)pampd, - false, &zsize, &zpages); - if (page) - dec_zcache_pers_pageframes(); - dec_zcache_pers_zpages(zpages); - dec_zcache_pers_zbytes(zsize); - } - if (!is_local_client(pool->client) && !zero_filled) - ramster_count_foreign_pages(is_ephemeral(pool), -1); - if (page && !zero_filled) - zcache_free_page(page); -} - -static struct tmem_pamops zcache_pamops = { - .create_finish = zcache_pampd_create_finish, - .get_data = zcache_pampd_get_data, - .get_data_and_free = zcache_pampd_get_data_and_free, - .free = zcache_pampd_free, -}; - -/* - * zcache compression/decompression and related per-cpu stuff - */ - -static DEFINE_PER_CPU(unsigned char *, zcache_dstmem); -#define ZCACHE_DSTMEM_ORDER 1 - -static void zcache_compress(struct page *from, void **out_va, unsigned *out_len) -{ - int ret; - unsigned char *dmem = __get_cpu_var(zcache_dstmem); - char *from_va; - - BUG_ON(!irqs_disabled()); - /* no buffer or no compressor so can't compress */ - BUG_ON(dmem == NULL); - *out_len = PAGE_SIZE << ZCACHE_DSTMEM_ORDER; - from_va = kmap_atomic(from); - mb(); - ret = zcache_comp_op(ZCACHE_COMPOP_COMPRESS, from_va, PAGE_SIZE, dmem, - out_len); - BUG_ON(ret); - *out_va = dmem; - kunmap_atomic(from_va); -} - -static int zcache_comp_cpu_up(int cpu) -{ - struct crypto_comp *tfm; - - tfm = crypto_alloc_comp(zcache_comp_name, 0, 0); - if (IS_ERR(tfm)) - return NOTIFY_BAD; - *per_cpu_ptr(zcache_comp_pcpu_tfms, cpu) = tfm; - return NOTIFY_OK; -} - -static void zcache_comp_cpu_down(int cpu) -{ - struct crypto_comp *tfm; - - tfm = *per_cpu_ptr(zcache_comp_pcpu_tfms, cpu); - crypto_free_comp(tfm); - *per_cpu_ptr(zcache_comp_pcpu_tfms, cpu) = NULL; -} - -static int zcache_cpu_notifier(struct notifier_block *nb, - unsigned long action, void *pcpu) -{ - int ret, i, cpu = (long)pcpu; - struct zcache_preload *kp; - - switch (action) { - case CPU_UP_PREPARE: - ret = zcache_comp_cpu_up(cpu); - if (ret != NOTIFY_OK) { - pr_err("%s: can't allocate compressor xform\n", - namestr); - return ret; - } - per_cpu(zcache_dstmem, cpu) = (void *)__get_free_pages( - GFP_KERNEL | __GFP_REPEAT, ZCACHE_DSTMEM_ORDER); - if (ramster_enabled) - ramster_cpu_up(cpu); - break; - case CPU_DEAD: - case CPU_UP_CANCELED: - zcache_comp_cpu_down(cpu); - free_pages((unsigned long)per_cpu(zcache_dstmem, cpu), - ZCACHE_DSTMEM_ORDER); - per_cpu(zcache_dstmem, cpu) = NULL; - kp = &per_cpu(zcache_preloads, cpu); - for (i = 0; i < ARRAY_SIZE(kp->objnodes); i++) { - if (kp->objnodes[i]) - kmem_cache_free(zcache_objnode_cache, - kp->objnodes[i]); - } - if (kp->obj) { - kmem_cache_free(zcache_obj_cache, kp->obj); - kp->obj = NULL; - } - if (ramster_enabled) - ramster_cpu_down(cpu); - break; - default: - break; - } - return NOTIFY_OK; -} - -static struct notifier_block zcache_cpu_notifier_block = { - .notifier_call = zcache_cpu_notifier -}; - -/* - * The following code interacts with the zbud eviction and zbud - * zombify code to access LRU pages - */ - -static struct page *zcache_evict_eph_pageframe(void) -{ - struct page *page; - unsigned int zsize = 0, zpages = 0; - - page = zbud_evict_pageframe_lru(&zsize, &zpages); - if (page == NULL) - goto out; - dec_zcache_eph_zbytes(zsize); - dec_zcache_eph_zpages(zpages); - inc_zcache_evicted_eph_zpages(zpages); - dec_zcache_eph_pageframes(); - inc_zcache_evicted_eph_pageframes(); -out: - return page; -} - -#ifdef CONFIG_ZCACHE_WRITEBACK - -static atomic_t zcache_outstanding_writeback_pages_atomic = ATOMIC_INIT(0); - -static inline void inc_zcache_outstanding_writeback_pages(void) -{ - zcache_outstanding_writeback_pages = - atomic_inc_return(&zcache_outstanding_writeback_pages_atomic); -} -static inline void dec_zcache_outstanding_writeback_pages(void) -{ - zcache_outstanding_writeback_pages = - atomic_dec_return(&zcache_outstanding_writeback_pages_atomic); -}; -static void unswiz(struct tmem_oid oid, u32 index, - unsigned *type, pgoff_t *offset); - -/* - * Choose an LRU persistent pageframe and attempt to write it back to - * the backing swap disk by calling frontswap_writeback on both zpages. - * - * This is work-in-progress. - */ - -static void zcache_end_swap_write(struct bio *bio, int err) -{ - end_swap_bio_write(bio, err); - dec_zcache_outstanding_writeback_pages(); - zcache_writtenback_pages++; -} - -/* - * zcache_get_swap_cache_page - * - * This is an adaption of read_swap_cache_async() - * - * If success, page is returned in retpage - * Returns 0 if page was already in the swap cache, page is not locked - * Returns 1 if the new page needs to be populated, page is locked - */ -static int zcache_get_swap_cache_page(int type, pgoff_t offset, - struct page *new_page) -{ - struct page *found_page; - swp_entry_t entry = swp_entry(type, offset); - int err; - - BUG_ON(new_page == NULL); - do { - /* - * First check the swap cache. Since this is normally - * called after lookup_swap_cache() failed, re-calling - * that would confuse statistics. - */ - found_page = find_get_page(&swapper_space, entry.val); - if (found_page) - return 0; - - /* - * call radix_tree_preload() while we can wait. - */ - err = radix_tree_preload(GFP_KERNEL); - if (err) - break; - - /* - * Swap entry may have been freed since our caller observed it. - */ - err = swapcache_prepare(entry); - if (err == -EEXIST) { /* seems racy */ - radix_tree_preload_end(); - continue; - } - if (err) { /* swp entry is obsolete ? */ - radix_tree_preload_end(); - break; - } - - /* May fail (-ENOMEM) if radix-tree node allocation failed. */ - __set_page_locked(new_page); - SetPageSwapBacked(new_page); - err = __add_to_swap_cache(new_page, entry); - if (likely(!err)) { - radix_tree_preload_end(); - lru_cache_add_anon(new_page); - return 1; - } - radix_tree_preload_end(); - ClearPageSwapBacked(new_page); - __clear_page_locked(new_page); - /* - * add_to_swap_cache() doesn't return -EEXIST, so we can safely - * clear SWAP_HAS_CACHE flag. - */ - swapcache_free(entry, NULL); - /* FIXME: is it possible to get here without err==-ENOMEM? - * If not, we can dispense with the do loop, use goto retry */ - } while (err != -ENOMEM); - - return -ENOMEM; -} - -/* - * Given a frontswap zpage in zcache (identified by type/offset) and - * an empty page, put the page into the swap cache, use frontswap - * to get the page from zcache into the empty page, then give it - * to the swap subsystem to send to disk (carefully avoiding the - * possibility that frontswap might snatch it back). - * Returns < 0 if error, 0 if successful, and 1 if successful but - * the newpage passed in not needed and should be freed. - */ -static int zcache_frontswap_writeback_zpage(int type, pgoff_t offset, - struct page *newpage) -{ - struct page *page = newpage; - int ret; - struct writeback_control wbc = { - .sync_mode = WB_SYNC_NONE, - }; - - ret = zcache_get_swap_cache_page(type, offset, page); - if (ret < 0) - return ret; - else if (ret == 0) { - /* more uptodate page is already in swapcache */ - __frontswap_invalidate_page(type, offset); - return 1; - } - - BUG_ON(!frontswap_has_exclusive_gets); /* load must also invalidate */ - /* FIXME: how is it possible to get here when page is unlocked? */ - __frontswap_load(page); - SetPageUptodate(page); /* above does SetPageDirty, is that enough? */ - - /* start writeback */ - SetPageReclaim(page); - /* - * Return value is ignored here because it doesn't change anything - * for us. Page is returned unlocked. - */ - (void)__swap_writepage(page, &wbc, zcache_end_swap_write); - page_cache_release(page); - inc_zcache_outstanding_writeback_pages(); - - return 0; -} - -/* - * The following is still a magic number... we want to allow forward progress - * for writeback because it clears out needed RAM when under pressure, but - * we don't want to allow writeback to absorb and queue too many GFP_KERNEL - * pages if the swap device is very slow. - */ -#define ZCACHE_MAX_OUTSTANDING_WRITEBACK_PAGES 6400 - -/* - * Try to allocate two free pages, first using a non-aggressive alloc, - * then by evicting zcache ephemeral (clean pagecache) pages, and last - * by aggressive GFP_KERNEL alloc. We allow zbud to choose a pageframe - * consisting of 1-2 zbuds/zpages, then call the writeback_zpage helper - * function above for each. - */ -static int zcache_frontswap_writeback(void) -{ - struct tmem_handle th[2]; - int ret = 0; - int nzbuds, writeback_ret; - unsigned type; - struct page *znewpage1 = NULL, *znewpage2 = NULL; - struct page *evictpage1 = NULL, *evictpage2 = NULL; - struct page *newpage1 = NULL, *newpage2 = NULL; - struct page *page1 = NULL, *page2 = NULL; - pgoff_t offset; - - znewpage1 = alloc_page(ZCACHE_GFP_MASK); - znewpage2 = alloc_page(ZCACHE_GFP_MASK); - if (znewpage1 == NULL) - evictpage1 = zcache_evict_eph_pageframe(); - if (znewpage2 == NULL) - evictpage2 = zcache_evict_eph_pageframe(); - - if ((evictpage1 == NULL || evictpage2 == NULL) && - atomic_read(&zcache_outstanding_writeback_pages_atomic) > - ZCACHE_MAX_OUTSTANDING_WRITEBACK_PAGES) { - goto free_and_out; - } - if (znewpage1 == NULL && evictpage1 == NULL) - newpage1 = alloc_page(GFP_KERNEL); - if (znewpage2 == NULL && evictpage2 == NULL) - newpage2 = alloc_page(GFP_KERNEL); - if (newpage1 == NULL || newpage2 == NULL) - goto free_and_out; - - /* ok, we have two pageframes pre-allocated, get a pair of zbuds */ - nzbuds = zbud_make_zombie_lru(&th[0], NULL, NULL, false); - if (nzbuds == 0) { - ret = -ENOENT; - goto free_and_out; - } - - /* process the first zbud */ - unswiz(th[0].oid, th[0].index, &type, &offset); - page1 = (znewpage1 != NULL) ? znewpage1 : - ((newpage1 != NULL) ? newpage1 : evictpage1); - writeback_ret = zcache_frontswap_writeback_zpage(type, offset, page1); - if (writeback_ret < 0) { - ret = -ENOMEM; - goto free_and_out; - } - if (evictpage1 != NULL) - zcache_pageframes_freed = - atomic_inc_return(&zcache_pageframes_freed_atomic); - if (writeback_ret == 0) { - /* zcache_get_swap_cache_page will free, don't double free */ - znewpage1 = NULL; - newpage1 = NULL; - evictpage1 = NULL; - } - if (nzbuds < 2) - goto free_and_out; - - /* if there is a second zbud, process it */ - unswiz(th[1].oid, th[1].index, &type, &offset); - page2 = (znewpage2 != NULL) ? znewpage2 : - ((newpage2 != NULL) ? newpage2 : evictpage2); - writeback_ret = zcache_frontswap_writeback_zpage(type, offset, page2); - if (writeback_ret < 0) { - ret = -ENOMEM; - goto free_and_out; - } - if (evictpage2 != NULL) - zcache_pageframes_freed = - atomic_inc_return(&zcache_pageframes_freed_atomic); - if (writeback_ret == 0) { - znewpage2 = NULL; - newpage2 = NULL; - evictpage2 = NULL; - } - -free_and_out: - if (znewpage1 != NULL) - page_cache_release(znewpage1); - if (znewpage2 != NULL) - page_cache_release(znewpage2); - if (newpage1 != NULL) - page_cache_release(newpage1); - if (newpage2 != NULL) - page_cache_release(newpage2); - if (evictpage1 != NULL) - zcache_free_page(evictpage1); - if (evictpage2 != NULL) - zcache_free_page(evictpage2); - return ret; -} -#endif /* CONFIG_ZCACHE_WRITEBACK */ - -/* - * When zcache is disabled ("frozen"), pools can be created and destroyed, - * but all puts (and thus all other operations that require memory allocation) - * must fail. If zcache is unfrozen, accepts puts, then frozen again, - * data consistency requires all puts while frozen to be converted into - * flushes. - */ -static bool zcache_freeze; - -/* - * This zcache shrinker interface reduces the number of ephemeral pageframes - * used by zcache to approximately the same as the total number of LRU_FILE - * pageframes in use, and now also reduces the number of persistent pageframes - * used by zcache to approximately the same as the total number of LRU_ANON - * pageframes in use. FIXME POLICY: Probably the writeback should only occur - * if the eviction doesn't free enough pages. - */ -static int shrink_zcache_memory(struct shrinker *shrink, - struct shrink_control *sc) -{ - static bool in_progress; - int ret = -1; - int nr = sc->nr_to_scan; - int nr_evict = 0; - int nr_writeback = 0; - struct page *page; - int file_pageframes_inuse, anon_pageframes_inuse; - - if (nr <= 0) - goto skip_evict; - - /* don't allow more than one eviction thread at a time */ - if (in_progress) - goto skip_evict; - - in_progress = true; - - /* we are going to ignore nr, and target a different value */ - zcache_last_active_file_pageframes = - global_page_state(NR_LRU_BASE + LRU_ACTIVE_FILE); - zcache_last_inactive_file_pageframes = - global_page_state(NR_LRU_BASE + LRU_INACTIVE_FILE); - file_pageframes_inuse = zcache_last_active_file_pageframes + - zcache_last_inactive_file_pageframes; - if (zcache_eph_pageframes > file_pageframes_inuse) - nr_evict = zcache_eph_pageframes - file_pageframes_inuse; - else - nr_evict = 0; - while (nr_evict-- > 0) { - page = zcache_evict_eph_pageframe(); - if (page == NULL) - break; - zcache_free_page(page); - } - - zcache_last_active_anon_pageframes = - global_page_state(NR_LRU_BASE + LRU_ACTIVE_ANON); - zcache_last_inactive_anon_pageframes = - global_page_state(NR_LRU_BASE + LRU_INACTIVE_ANON); - anon_pageframes_inuse = zcache_last_active_anon_pageframes + - zcache_last_inactive_anon_pageframes; - if (zcache_pers_pageframes > anon_pageframes_inuse) - nr_writeback = zcache_pers_pageframes - anon_pageframes_inuse; - else - nr_writeback = 0; - while (nr_writeback-- > 0) { -#ifdef CONFIG_ZCACHE_WRITEBACK - int writeback_ret; - writeback_ret = zcache_frontswap_writeback(); - if (writeback_ret == -ENOMEM) -#endif - break; - } - in_progress = false; - -skip_evict: - /* resample: has changed, but maybe not all the way yet */ - zcache_last_active_file_pageframes = - global_page_state(NR_LRU_BASE + LRU_ACTIVE_FILE); - zcache_last_inactive_file_pageframes = - global_page_state(NR_LRU_BASE + LRU_INACTIVE_FILE); - ret = zcache_eph_pageframes - zcache_last_active_file_pageframes + - zcache_last_inactive_file_pageframes; - if (ret < 0) - ret = 0; - return ret; -} - -static struct shrinker zcache_shrinker = { - .shrink = shrink_zcache_memory, - .seeks = DEFAULT_SEEKS, -}; - -/* - * zcache shims between cleancache/frontswap ops and tmem - */ - -/* FIXME rename these core routines to zcache_tmemput etc? */ -int zcache_put_page(int cli_id, int pool_id, struct tmem_oid *oidp, - uint32_t index, void *page, - unsigned int size, bool raw, int ephemeral) -{ - struct tmem_pool *pool; - struct tmem_handle th; - int ret = -1; - void *pampd = NULL; - - BUG_ON(!irqs_disabled()); - pool = zcache_get_pool_by_id(cli_id, pool_id); - if (unlikely(pool == NULL)) - goto out; - if (!zcache_freeze) { - ret = 0; - th.client_id = cli_id; - th.pool_id = pool_id; - th.oid = *oidp; - th.index = index; - pampd = zcache_pampd_create((char *)page, size, raw, - ephemeral, &th); - if (pampd == NULL) { - ret = -ENOMEM; - if (ephemeral) - inc_zcache_failed_eph_puts(); - else - inc_zcache_failed_pers_puts(); - } else { - if (ramster_enabled) - ramster_do_preload_flnode(pool); - ret = tmem_put(pool, oidp, index, 0, pampd); - if (ret < 0) - BUG(); - } - zcache_put_pool(pool); - } else { - inc_zcache_put_to_flush(); - if (ramster_enabled) - ramster_do_preload_flnode(pool); - if (atomic_read(&pool->obj_count) > 0) - /* the put fails whether the flush succeeds or not */ - (void)tmem_flush_page(pool, oidp, index); - zcache_put_pool(pool); - } -out: - return ret; -} - -int zcache_get_page(int cli_id, int pool_id, struct tmem_oid *oidp, - uint32_t index, void *page, - size_t *sizep, bool raw, int get_and_free) -{ - struct tmem_pool *pool; - int ret = -1; - bool eph; - - if (!raw) { - BUG_ON(irqs_disabled()); - BUG_ON(in_softirq()); - } - pool = zcache_get_pool_by_id(cli_id, pool_id); - eph = is_ephemeral(pool); - if (likely(pool != NULL)) { - if (atomic_read(&pool->obj_count) > 0) - ret = tmem_get(pool, oidp, index, (char *)(page), - sizep, raw, get_and_free); - zcache_put_pool(pool); - } - WARN_ONCE((!is_ephemeral(pool) && (ret != 0)), - "zcache_get fails on persistent pool, " - "bad things are very likely to happen soon\n"); -#ifdef RAMSTER_TESTING - if (ret != 0 && ret != -1 && !(ret == -EINVAL && is_ephemeral(pool))) - pr_err("TESTING zcache_get tmem_get returns ret=%d\n", ret); -#endif - return ret; -} - -int zcache_flush_page(int cli_id, int pool_id, - struct tmem_oid *oidp, uint32_t index) -{ - struct tmem_pool *pool; - int ret = -1; - unsigned long flags; - - local_irq_save(flags); - inc_zcache_flush_total(); - pool = zcache_get_pool_by_id(cli_id, pool_id); - if (ramster_enabled) - ramster_do_preload_flnode(pool); - if (likely(pool != NULL)) { - if (atomic_read(&pool->obj_count) > 0) - ret = tmem_flush_page(pool, oidp, index); - zcache_put_pool(pool); - } - if (ret >= 0) - inc_zcache_flush_found(); - local_irq_restore(flags); - return ret; -} - -int zcache_flush_object(int cli_id, int pool_id, - struct tmem_oid *oidp) -{ - struct tmem_pool *pool; - int ret = -1; - unsigned long flags; - - local_irq_save(flags); - inc_zcache_flobj_total(); - pool = zcache_get_pool_by_id(cli_id, pool_id); - if (ramster_enabled) - ramster_do_preload_flnode(pool); - if (likely(pool != NULL)) { - if (atomic_read(&pool->obj_count) > 0) - ret = tmem_flush_object(pool, oidp); - zcache_put_pool(pool); - } - if (ret >= 0) - inc_zcache_flobj_found(); - local_irq_restore(flags); - return ret; -} - -static int zcache_client_destroy_pool(int cli_id, int pool_id) -{ - struct tmem_pool *pool = NULL; - struct zcache_client *cli = NULL; - int ret = -1; - - if (pool_id < 0) - goto out; - if (cli_id == LOCAL_CLIENT) - cli = &zcache_host; - else if ((unsigned int)cli_id < MAX_CLIENTS) - cli = &zcache_clients[cli_id]; - if (cli == NULL) - goto out; - atomic_inc(&cli->refcount); - pool = cli->tmem_pools[pool_id]; - if (pool == NULL) - goto out; - cli->tmem_pools[pool_id] = NULL; - /* wait for pool activity on other cpus to quiesce */ - while (atomic_read(&pool->refcount) != 0) - ; - atomic_dec(&cli->refcount); - local_bh_disable(); - ret = tmem_destroy_pool(pool); - local_bh_enable(); - kfree(pool); - if (cli_id == LOCAL_CLIENT) - pr_info("%s: destroyed local pool id=%d\n", namestr, pool_id); - else - pr_info("%s: destroyed pool id=%d, client=%d\n", - namestr, pool_id, cli_id); -out: - return ret; -} - -int zcache_new_pool(uint16_t cli_id, uint32_t flags) -{ - int poolid = -1; - struct tmem_pool *pool; - struct zcache_client *cli = NULL; - - if (cli_id == LOCAL_CLIENT) - cli = &zcache_host; - else if ((unsigned int)cli_id < MAX_CLIENTS) - cli = &zcache_clients[cli_id]; - if (cli == NULL) - goto out; - atomic_inc(&cli->refcount); - pool = kmalloc(sizeof(struct tmem_pool), GFP_ATOMIC); - if (pool == NULL) - goto out; - - for (poolid = 0; poolid < MAX_POOLS_PER_CLIENT; poolid++) - if (cli->tmem_pools[poolid] == NULL) - break; - if (poolid >= MAX_POOLS_PER_CLIENT) { - pr_info("%s: pool creation failed: max exceeded\n", namestr); - kfree(pool); - poolid = -1; - goto out; - } - atomic_set(&pool->refcount, 0); - pool->client = cli; - pool->pool_id = poolid; - tmem_new_pool(pool, flags); - cli->tmem_pools[poolid] = pool; - if (cli_id == LOCAL_CLIENT) - pr_info("%s: created %s local tmem pool, id=%d\n", namestr, - flags & TMEM_POOL_PERSIST ? "persistent" : "ephemeral", - poolid); - else - pr_info("%s: created %s tmem pool, id=%d, client=%d\n", namestr, - flags & TMEM_POOL_PERSIST ? "persistent" : "ephemeral", - poolid, cli_id); -out: - if (cli != NULL) - atomic_dec(&cli->refcount); - return poolid; -} - -static int zcache_local_new_pool(uint32_t flags) -{ - return zcache_new_pool(LOCAL_CLIENT, flags); -} - -int zcache_autocreate_pool(unsigned int cli_id, unsigned int pool_id, bool eph) -{ - struct tmem_pool *pool; - struct zcache_client *cli = NULL; - uint32_t flags = eph ? 0 : TMEM_POOL_PERSIST; - int ret = -1; - - BUG_ON(!ramster_enabled); - if (cli_id == LOCAL_CLIENT) - goto out; - if (pool_id >= MAX_POOLS_PER_CLIENT) - goto out; - if (cli_id >= MAX_CLIENTS) - goto out; - - cli = &zcache_clients[cli_id]; - if ((eph && disable_cleancache) || (!eph && disable_frontswap)) { - pr_err("zcache_autocreate_pool: pool type disabled\n"); - goto out; - } - if (!cli->allocated) { - if (zcache_new_client(cli_id)) { - pr_err("zcache_autocreate_pool: can't create client\n"); - goto out; - } - cli = &zcache_clients[cli_id]; - } - atomic_inc(&cli->refcount); - pool = cli->tmem_pools[pool_id]; - if (pool != NULL) { - if (pool->persistent && eph) { - pr_err("zcache_autocreate_pool: type mismatch\n"); - goto out; - } - ret = 0; - goto out; - } - pool = kmalloc(sizeof(struct tmem_pool), GFP_KERNEL); - if (pool == NULL) - goto out; - - atomic_set(&pool->refcount, 0); - pool->client = cli; - pool->pool_id = pool_id; - tmem_new_pool(pool, flags); - cli->tmem_pools[pool_id] = pool; - pr_info("%s: AUTOcreated %s tmem poolid=%d, for remote client=%d\n", - namestr, flags & TMEM_POOL_PERSIST ? "persistent" : "ephemeral", - pool_id, cli_id); - ret = 0; -out: - if (cli != NULL) - atomic_dec(&cli->refcount); - return ret; -} - -/********** - * Two kernel functionalities currently can be layered on top of tmem. - * These are "cleancache" which is used as a second-chance cache for clean - * page cache pages; and "frontswap" which is used for swap pages - * to avoid writes to disk. A generic "shim" is provided here for each - * to translate in-kernel semantics to zcache semantics. - */ - -static void zcache_cleancache_put_page(int pool_id, - struct cleancache_filekey key, - pgoff_t index, struct page *page) -{ - u32 ind = (u32) index; - struct tmem_oid oid = *(struct tmem_oid *)&key; - - if (!disable_cleancache_ignore_nonactive && !PageWasActive(page)) { - inc_zcache_eph_nonactive_puts_ignored(); - return; - } - if (likely(ind == index)) - (void)zcache_put_page(LOCAL_CLIENT, pool_id, &oid, index, - page, PAGE_SIZE, false, 1); -} - -static int zcache_cleancache_get_page(int pool_id, - struct cleancache_filekey key, - pgoff_t index, struct page *page) -{ - u32 ind = (u32) index; - struct tmem_oid oid = *(struct tmem_oid *)&key; - size_t size; - int ret = -1; - - if (likely(ind == index)) { - ret = zcache_get_page(LOCAL_CLIENT, pool_id, &oid, index, - page, &size, false, 0); - BUG_ON(ret >= 0 && size != PAGE_SIZE); - if (ret == 0) - SetPageWasActive(page); - } - return ret; -} - -static void zcache_cleancache_flush_page(int pool_id, - struct cleancache_filekey key, - pgoff_t index) -{ - u32 ind = (u32) index; - struct tmem_oid oid = *(struct tmem_oid *)&key; - - if (likely(ind == index)) - (void)zcache_flush_page(LOCAL_CLIENT, pool_id, &oid, ind); -} - -static void zcache_cleancache_flush_inode(int pool_id, - struct cleancache_filekey key) -{ - struct tmem_oid oid = *(struct tmem_oid *)&key; - - (void)zcache_flush_object(LOCAL_CLIENT, pool_id, &oid); -} - -static void zcache_cleancache_flush_fs(int pool_id) -{ - if (pool_id >= 0) - (void)zcache_client_destroy_pool(LOCAL_CLIENT, pool_id); -} - -static int zcache_cleancache_init_fs(size_t pagesize) -{ - BUG_ON(sizeof(struct cleancache_filekey) != - sizeof(struct tmem_oid)); - BUG_ON(pagesize != PAGE_SIZE); - return zcache_local_new_pool(0); -} - -static int zcache_cleancache_init_shared_fs(char *uuid, size_t pagesize) -{ - /* shared pools are unsupported and map to private */ - BUG_ON(sizeof(struct cleancache_filekey) != - sizeof(struct tmem_oid)); - BUG_ON(pagesize != PAGE_SIZE); - return zcache_local_new_pool(0); -} - -static struct cleancache_ops zcache_cleancache_ops = { - .put_page = zcache_cleancache_put_page, - .get_page = zcache_cleancache_get_page, - .invalidate_page = zcache_cleancache_flush_page, - .invalidate_inode = zcache_cleancache_flush_inode, - .invalidate_fs = zcache_cleancache_flush_fs, - .init_shared_fs = zcache_cleancache_init_shared_fs, - .init_fs = zcache_cleancache_init_fs -}; - -struct cleancache_ops *zcache_cleancache_register_ops(void) -{ - struct cleancache_ops *old_ops = - cleancache_register_ops(&zcache_cleancache_ops); - - return old_ops; -} - -/* a single tmem poolid is used for all frontswap "types" (swapfiles) */ -static int zcache_frontswap_poolid __read_mostly = -1; - -/* - * Swizzling increases objects per swaptype, increasing tmem concurrency - * for heavy swaploads. Later, larger nr_cpus -> larger SWIZ_BITS - * Setting SWIZ_BITS to 27 basically reconstructs the swap entry from - * frontswap_get_page(), but has side-effects. Hence using 8. - */ -#define SWIZ_BITS 8 -#define SWIZ_MASK ((1 << SWIZ_BITS) - 1) -#define _oswiz(_type, _ind) ((_type << SWIZ_BITS) | (_ind & SWIZ_MASK)) -#define iswiz(_ind) (_ind >> SWIZ_BITS) - -static inline struct tmem_oid oswiz(unsigned type, u32 ind) -{ - struct tmem_oid oid = { .oid = { 0 } }; - oid.oid[0] = _oswiz(type, ind); - return oid; -} - -#ifdef CONFIG_ZCACHE_WRITEBACK -static void unswiz(struct tmem_oid oid, u32 index, - unsigned *type, pgoff_t *offset) -{ - *type = (unsigned)(oid.oid[0] >> SWIZ_BITS); - *offset = (pgoff_t)((index << SWIZ_BITS) | - (oid.oid[0] & SWIZ_MASK)); -} -#endif - -static int zcache_frontswap_put_page(unsigned type, pgoff_t offset, - struct page *page) -{ - u64 ind64 = (u64)offset; - u32 ind = (u32)offset; - struct tmem_oid oid = oswiz(type, ind); - int ret = -1; - unsigned long flags; - - BUG_ON(!PageLocked(page)); - if (!disable_frontswap_ignore_nonactive && !PageWasActive(page)) { - inc_zcache_pers_nonactive_puts_ignored(); - ret = -ERANGE; - goto out; - } - if (likely(ind64 == ind)) { - local_irq_save(flags); - ret = zcache_put_page(LOCAL_CLIENT, zcache_frontswap_poolid, - &oid, iswiz(ind), - page, PAGE_SIZE, false, 0); - local_irq_restore(flags); - } -out: - return ret; -} - -/* returns 0 if the page was successfully gotten from frontswap, -1 if - * was not present (should never happen!) */ -static int zcache_frontswap_get_page(unsigned type, pgoff_t offset, - struct page *page) -{ - u64 ind64 = (u64)offset; - u32 ind = (u32)offset; - struct tmem_oid oid = oswiz(type, ind); - size_t size; - int ret = -1, get_and_free; - - if (frontswap_has_exclusive_gets) - get_and_free = 1; - else - get_and_free = -1; - BUG_ON(!PageLocked(page)); - if (likely(ind64 == ind)) { - ret = zcache_get_page(LOCAL_CLIENT, zcache_frontswap_poolid, - &oid, iswiz(ind), - page, &size, false, get_and_free); - BUG_ON(ret >= 0 && size != PAGE_SIZE); - } - return ret; -} - -/* flush a single page from frontswap */ -static void zcache_frontswap_flush_page(unsigned type, pgoff_t offset) -{ - u64 ind64 = (u64)offset; - u32 ind = (u32)offset; - struct tmem_oid oid = oswiz(type, ind); - - if (likely(ind64 == ind)) - (void)zcache_flush_page(LOCAL_CLIENT, zcache_frontswap_poolid, - &oid, iswiz(ind)); -} - -/* flush all pages from the passed swaptype */ -static void zcache_frontswap_flush_area(unsigned type) -{ - struct tmem_oid oid; - int ind; - - for (ind = SWIZ_MASK; ind >= 0; ind--) { - oid = oswiz(type, ind); - (void)zcache_flush_object(LOCAL_CLIENT, - zcache_frontswap_poolid, &oid); - } -} - -static void zcache_frontswap_init(unsigned ignored) -{ - /* a single tmem poolid is used for all frontswap "types" (swapfiles) */ - if (zcache_frontswap_poolid < 0) - zcache_frontswap_poolid = - zcache_local_new_pool(TMEM_POOL_PERSIST); -} - -static struct frontswap_ops zcache_frontswap_ops = { - .store = zcache_frontswap_put_page, - .load = zcache_frontswap_get_page, - .invalidate_page = zcache_frontswap_flush_page, - .invalidate_area = zcache_frontswap_flush_area, - .init = zcache_frontswap_init -}; - -struct frontswap_ops *zcache_frontswap_register_ops(void) -{ - struct frontswap_ops *old_ops = - frontswap_register_ops(&zcache_frontswap_ops); - - return old_ops; -} - -/* - * zcache initialization - * NOTE FOR NOW zcache or ramster MUST BE PROVIDED AS A KERNEL BOOT PARAMETER - * OR NOTHING HAPPENS! - */ - -#ifndef CONFIG_ZCACHE_MODULE -static int __init enable_zcache(char *s) -{ - zcache_enabled = true; - return 1; -} -__setup("zcache", enable_zcache); - -static int __init enable_ramster(char *s) -{ - zcache_enabled = true; -#ifdef CONFIG_RAMSTER - ramster_enabled = true; -#endif - return 1; -} -__setup("ramster", enable_ramster); - -/* allow independent dynamic disabling of cleancache and frontswap */ - -static int __init no_cleancache(char *s) -{ - disable_cleancache = true; - return 1; -} - -__setup("nocleancache", no_cleancache); - -static int __init no_frontswap(char *s) -{ - disable_frontswap = true; - return 1; -} - -__setup("nofrontswap", no_frontswap); - -static int __init no_frontswap_exclusive_gets(char *s) -{ - frontswap_has_exclusive_gets = false; - return 1; -} - -__setup("nofrontswapexclusivegets", no_frontswap_exclusive_gets); - -static int __init no_frontswap_ignore_nonactive(char *s) -{ - disable_frontswap_ignore_nonactive = true; - return 1; -} - -__setup("nofrontswapignorenonactive", no_frontswap_ignore_nonactive); - -static int __init no_cleancache_ignore_nonactive(char *s) -{ - disable_cleancache_ignore_nonactive = true; - return 1; -} - -__setup("nocleancacheignorenonactive", no_cleancache_ignore_nonactive); - -static int __init enable_zcache_compressor(char *s) -{ - strlcpy(zcache_comp_name, s, sizeof(zcache_comp_name)); - zcache_enabled = true; - return 1; -} -__setup("zcache=", enable_zcache_compressor); -#endif - - -static int zcache_comp_init(void) -{ - int ret = 0; - - /* check crypto algorithm */ -#ifdef CONFIG_ZCACHE_MODULE - ret = crypto_has_comp(zcache_comp_name, 0, 0); - if (!ret) { - ret = -1; - goto out; - } -#else - if (*zcache_comp_name != '\0') { - ret = crypto_has_comp(zcache_comp_name, 0, 0); - if (!ret) { - pr_info("zcache: %s not supported\n", - zcache_comp_name); - ret = 1; - goto out; - } - } - if (!ret) - strcpy(zcache_comp_name, "lzo"); - ret = crypto_has_comp(zcache_comp_name, 0, 0); - if (!ret) { - ret = 1; - goto out; - } -#endif - pr_info("zcache: using %s compressor\n", zcache_comp_name); - - /* alloc percpu transforms */ - ret = 0; - zcache_comp_pcpu_tfms = alloc_percpu(struct crypto_comp *); - if (!zcache_comp_pcpu_tfms) - ret = 1; -out: - return ret; -} - -static int zcache_init(void) -{ - int ret = 0; - -#ifdef CONFIG_ZCACHE_MODULE - zcache_enabled = 1; -#endif - if (ramster_enabled) { - namestr = "ramster"; - ramster_register_pamops(&zcache_pamops); - } - zcache_debugfs_init(); - if (zcache_enabled) { - unsigned int cpu; - - tmem_register_hostops(&zcache_hostops); - tmem_register_pamops(&zcache_pamops); - ret = register_cpu_notifier(&zcache_cpu_notifier_block); - if (ret) { - pr_err("%s: can't register cpu notifier\n", namestr); - goto out; - } - ret = zcache_comp_init(); - if (ret) { - pr_err("%s: compressor initialization failed\n", - namestr); - goto out; - } - for_each_online_cpu(cpu) { - void *pcpu = (void *)(long)cpu; - zcache_cpu_notifier(&zcache_cpu_notifier_block, - CPU_UP_PREPARE, pcpu); - } - } - zcache_objnode_cache = kmem_cache_create("zcache_objnode", - sizeof(struct tmem_objnode), 0, 0, NULL); - zcache_obj_cache = kmem_cache_create("zcache_obj", - sizeof(struct tmem_obj), 0, 0, NULL); - ret = zcache_new_client(LOCAL_CLIENT); - if (ret) { - pr_err("%s: can't create client\n", namestr); - goto out; - } - zbud_init(); - if (zcache_enabled && !disable_cleancache) { - struct cleancache_ops *old_ops; - - register_shrinker(&zcache_shrinker); - old_ops = zcache_cleancache_register_ops(); - pr_info("%s: cleancache enabled using kernel transcendent " - "memory and compression buddies\n", namestr); -#ifdef CONFIG_ZCACHE_DEBUG - pr_info("%s: cleancache: ignorenonactive = %d\n", - namestr, !disable_cleancache_ignore_nonactive); -#endif - if (old_ops != NULL) - pr_warn("%s: cleancache_ops overridden\n", namestr); - } - if (zcache_enabled && !disable_frontswap) { - struct frontswap_ops *old_ops; - - old_ops = zcache_frontswap_register_ops(); - if (frontswap_has_exclusive_gets) - frontswap_tmem_exclusive_gets(true); - pr_info("%s: frontswap enabled using kernel transcendent " - "memory and compression buddies\n", namestr); -#ifdef CONFIG_ZCACHE_DEBUG - pr_info("%s: frontswap: excl gets = %d active only = %d\n", - namestr, frontswap_has_exclusive_gets, - !disable_frontswap_ignore_nonactive); -#endif - if (IS_ERR(old_ops) || old_ops) { - if (IS_ERR(old_ops)) - return PTR_RET(old_ops); - pr_warn("%s: frontswap_ops overridden\n", namestr); - } - } - if (ramster_enabled) - ramster_init(!disable_cleancache, !disable_frontswap, - frontswap_has_exclusive_gets, - !disable_frontswap_selfshrink); -out: - return ret; -} - -#ifdef CONFIG_ZCACHE_MODULE -#ifdef CONFIG_RAMSTER -module_param(ramster_enabled, bool, S_IRUGO); -module_param(disable_frontswap_selfshrink, int, S_IRUGO); -#endif -module_param(disable_cleancache, bool, S_IRUGO); -module_param(disable_frontswap, bool, S_IRUGO); -#ifdef FRONTSWAP_HAS_EXCLUSIVE_GETS -module_param(frontswap_has_exclusive_gets, bool, S_IRUGO); -#endif -module_param(disable_frontswap_ignore_nonactive, bool, S_IRUGO); -module_param(zcache_comp_name, charp, S_IRUGO); -module_init(zcache_init); -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Dan Magenheimer <dan.magenheimer@oracle.com>"); -MODULE_DESCRIPTION("In-kernel compression of cleancache/frontswap pages"); -#else -late_initcall(zcache_init); -#endif |