aboutsummaryrefslogtreecommitdiffstats
path: root/block/cfq-iosched.c
diff options
context:
space:
mode:
Diffstat (limited to 'block/cfq-iosched.c')
-rw-r--r--block/cfq-iosched.c354
1 files changed, 175 insertions, 179 deletions
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index c8dbe38c81c8..c4a0d5d8d7f0 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -6,21 +6,13 @@
*
* Copyright (C) 2003 Jens Axboe <axboe@suse.de>
*/
-#include <linux/kernel.h>
-#include <linux/fs.h>
-#include <linux/blkdev.h>
-#include <linux/elevator.h>
-#include <linux/bio.h>
#include <linux/config.h>
#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/init.h>
-#include <linux/compiler.h>
+#include <linux/blkdev.h>
+#include <linux/elevator.h>
#include <linux/hash.h>
#include <linux/rbtree.h>
-#include <linux/mempool.h>
#include <linux/ioprio.h>
-#include <linux/writeback.h>
/*
* tunables
@@ -47,6 +39,8 @@ static int cfq_slice_idle = HZ / 100;
*/
static const int cfq_max_depth = 2;
+static DEFINE_RWLOCK(cfq_exit_lock);
+
/*
* for the hash of cfqq inside the cfqd
*/
@@ -89,6 +83,9 @@ static kmem_cache_t *crq_pool;
static kmem_cache_t *cfq_pool;
static kmem_cache_t *cfq_ioc_pool;
+static atomic_t ioc_count = ATOMIC_INIT(0);
+static struct completion *ioc_gone;
+
#define CFQ_PRIO_LISTS IOPRIO_BE_NR
#define cfq_class_idle(cfqq) ((cfqq)->ioprio_class == IOPRIO_CLASS_IDLE)
#define cfq_class_be(cfqq) ((cfqq)->ioprio_class == IOPRIO_CLASS_BE)
@@ -109,7 +106,6 @@ static kmem_cache_t *cfq_ioc_pool;
* Per block device queue structure
*/
struct cfq_data {
- atomic_t ref;
request_queue_t *queue;
/*
@@ -175,6 +171,8 @@ struct cfq_data {
unsigned int cfq_slice_async_rq;
unsigned int cfq_slice_idle;
unsigned int cfq_max_depth;
+
+ struct list_head cic_list;
};
/*
@@ -288,7 +286,7 @@ CFQ_CRQ_FNS(is_sync);
static struct cfq_queue *cfq_find_cfq_hash(struct cfq_data *, unsigned int, unsigned short);
static void cfq_dispatch_insert(request_queue_t *, struct cfq_rq *);
-static void cfq_put_cfqd(struct cfq_data *cfqd);
+static struct cfq_queue *cfq_get_queue(struct cfq_data *cfqd, unsigned int key, struct task_struct *tsk, gfp_t gfp_mask);
#define process_sync(tsk) ((tsk)->flags & PF_SYNCWRITE)
@@ -1160,8 +1158,6 @@ static void cfq_put_queue(struct cfq_queue *cfqq)
if (unlikely(cfqd->active_queue == cfqq))
__cfq_slice_expired(cfqd, cfqq, 0);
- cfq_put_cfqd(cfqq->cfqd);
-
/*
* it's on the empty list and still hashed
*/
@@ -1179,7 +1175,7 @@ __cfq_find_cfq_hash(struct cfq_data *cfqd, unsigned int key, unsigned int prio,
hlist_for_each_safe(entry, next, hash_list) {
struct cfq_queue *__cfqq = list_entry_qhash(entry);
- const unsigned short __p = IOPRIO_PRIO_VALUE(__cfqq->ioprio_class, __cfqq->ioprio);
+ const unsigned short __p = IOPRIO_PRIO_VALUE(__cfqq->org_ioprio_class, __cfqq->org_ioprio);
if (__cfqq->key == key && (__p == prio || prio == CFQ_KEY_ANY))
return __cfqq;
@@ -1198,13 +1194,24 @@ static void cfq_free_io_context(struct cfq_io_context *cic)
{
struct cfq_io_context *__cic;
struct list_head *entry, *next;
+ int freed = 1;
list_for_each_safe(entry, next, &cic->list) {
__cic = list_entry(entry, struct cfq_io_context, list);
kmem_cache_free(cfq_ioc_pool, __cic);
+ freed++;
}
kmem_cache_free(cfq_ioc_pool, cic);
+ if (atomic_sub_and_test(freed, &ioc_count) && ioc_gone)
+ complete(ioc_gone);
+}
+
+static void cfq_trim(struct io_context *ioc)
+{
+ ioc->set_ioprio = NULL;
+ if (ioc->cic)
+ cfq_free_io_context(ioc->cic);
}
/*
@@ -1212,25 +1219,37 @@ static void cfq_free_io_context(struct cfq_io_context *cic)
*/
static void cfq_exit_single_io_context(struct cfq_io_context *cic)
{
- struct cfq_data *cfqd = cic->cfqq->cfqd;
- request_queue_t *q = cfqd->queue;
+ struct cfq_data *cfqd = cic->key;
+ request_queue_t *q;
+
+ if (!cfqd)
+ return;
+
+ q = cfqd->queue;
WARN_ON(!irqs_disabled());
spin_lock(q->queue_lock);
- if (unlikely(cic->cfqq == cfqd->active_queue))
- __cfq_slice_expired(cfqd, cic->cfqq, 0);
+ if (cic->cfqq[ASYNC]) {
+ if (unlikely(cic->cfqq[ASYNC] == cfqd->active_queue))
+ __cfq_slice_expired(cfqd, cic->cfqq[ASYNC], 0);
+ cfq_put_queue(cic->cfqq[ASYNC]);
+ cic->cfqq[ASYNC] = NULL;
+ }
+
+ if (cic->cfqq[SYNC]) {
+ if (unlikely(cic->cfqq[SYNC] == cfqd->active_queue))
+ __cfq_slice_expired(cfqd, cic->cfqq[SYNC], 0);
+ cfq_put_queue(cic->cfqq[SYNC]);
+ cic->cfqq[SYNC] = NULL;
+ }
- cfq_put_queue(cic->cfqq);
- cic->cfqq = NULL;
+ cic->key = NULL;
+ list_del_init(&cic->queue_list);
spin_unlock(q->queue_lock);
}
-/*
- * Another task may update the task cic list, if it is doing a queue lookup
- * on its behalf. cfq_cic_lock excludes such concurrent updates
- */
static void cfq_exit_io_context(struct cfq_io_context *cic)
{
struct cfq_io_context *__cic;
@@ -1242,12 +1261,14 @@ static void cfq_exit_io_context(struct cfq_io_context *cic)
/*
* put the reference this task is holding to the various queues
*/
+ read_lock(&cfq_exit_lock);
list_for_each(entry, &cic->list) {
__cic = list_entry(entry, struct cfq_io_context, list);
cfq_exit_single_io_context(__cic);
}
cfq_exit_single_io_context(cic);
+ read_unlock(&cfq_exit_lock);
local_irq_restore(flags);
}
@@ -1258,7 +1279,8 @@ cfq_alloc_io_context(struct cfq_data *cfqd, gfp_t gfp_mask)
if (cic) {
INIT_LIST_HEAD(&cic->list);
- cic->cfqq = NULL;
+ cic->cfqq[ASYNC] = NULL;
+ cic->cfqq[SYNC] = NULL;
cic->key = NULL;
cic->last_end_request = jiffies;
cic->ttime_total = 0;
@@ -1266,6 +1288,8 @@ cfq_alloc_io_context(struct cfq_data *cfqd, gfp_t gfp_mask)
cic->ttime_mean = 0;
cic->dtor = cfq_free_io_context;
cic->exit = cfq_exit_io_context;
+ INIT_LIST_HEAD(&cic->queue_list);
+ atomic_inc(&ioc_count);
}
return cic;
@@ -1318,14 +1342,27 @@ static void cfq_init_prio_data(struct cfq_queue *cfqq)
cfq_clear_cfqq_prio_changed(cfqq);
}
-static inline void changed_ioprio(struct cfq_queue *cfqq)
+static inline void changed_ioprio(struct cfq_io_context *cic)
{
- if (cfqq) {
- struct cfq_data *cfqd = cfqq->cfqd;
-
+ struct cfq_data *cfqd = cic->key;
+ struct cfq_queue *cfqq;
+ if (cfqd) {
spin_lock(cfqd->queue->queue_lock);
- cfq_mark_cfqq_prio_changed(cfqq);
- cfq_init_prio_data(cfqq);
+ cfqq = cic->cfqq[ASYNC];
+ if (cfqq) {
+ struct cfq_queue *new_cfqq;
+ new_cfqq = cfq_get_queue(cfqd, CFQ_KEY_ASYNC,
+ cic->ioc->task, GFP_ATOMIC);
+ if (new_cfqq) {
+ cic->cfqq[ASYNC] = new_cfqq;
+ cfq_put_queue(cfqq);
+ }
+ }
+ cfqq = cic->cfqq[SYNC];
+ if (cfqq) {
+ cfq_mark_cfqq_prio_changed(cfqq);
+ cfq_init_prio_data(cfqq);
+ }
spin_unlock(cfqd->queue->queue_lock);
}
}
@@ -1335,24 +1372,32 @@ static inline void changed_ioprio(struct cfq_queue *cfqq)
*/
static int cfq_ioc_set_ioprio(struct io_context *ioc, unsigned int ioprio)
{
- struct cfq_io_context *cic = ioc->cic;
+ struct cfq_io_context *cic;
+
+ write_lock(&cfq_exit_lock);
+
+ cic = ioc->cic;
- changed_ioprio(cic->cfqq);
+ changed_ioprio(cic);
list_for_each_entry(cic, &cic->list, list)
- changed_ioprio(cic->cfqq);
+ changed_ioprio(cic);
+
+ write_unlock(&cfq_exit_lock);
return 0;
}
static struct cfq_queue *
-cfq_get_queue(struct cfq_data *cfqd, unsigned int key, unsigned short ioprio,
+cfq_get_queue(struct cfq_data *cfqd, unsigned int key, struct task_struct *tsk,
gfp_t gfp_mask)
{
const int hashval = hash_long(key, CFQ_QHASH_SHIFT);
struct cfq_queue *cfqq, *new_cfqq = NULL;
+ unsigned short ioprio;
retry:
+ ioprio = tsk->ioprio;
cfqq = __cfq_find_cfq_hash(cfqd, key, ioprio, hashval);
if (!cfqq) {
@@ -1381,7 +1426,6 @@ retry:
hlist_add_head(&cfqq->cfq_hash, &cfqd->cfq_hash[hashval]);
atomic_set(&cfqq->ref, 0);
cfqq->cfqd = cfqd;
- atomic_inc(&cfqd->ref);
cfqq->service_last = 0;
/*
* set ->slice_left to allow preemption for a new process
@@ -1419,6 +1463,7 @@ cfq_get_io_context(struct cfq_data *cfqd, pid_t pid, gfp_t gfp_mask)
if (!ioc)
return NULL;
+restart:
if ((cic = ioc->cic) == NULL) {
cic = cfq_alloc_io_context(cfqd, gfp_mask);
@@ -1429,11 +1474,13 @@ cfq_get_io_context(struct cfq_data *cfqd, pid_t pid, gfp_t gfp_mask)
* manually increment generic io_context usage count, it
* cannot go away since we are already holding one ref to it
*/
- ioc->cic = cic;
- ioc->set_ioprio = cfq_ioc_set_ioprio;
cic->ioc = ioc;
cic->key = cfqd;
- atomic_inc(&cfqd->ref);
+ read_lock(&cfq_exit_lock);
+ ioc->set_ioprio = cfq_ioc_set_ioprio;
+ ioc->cic = cic;
+ list_add(&cic->queue_list, &cfqd->cic_list);
+ read_unlock(&cfq_exit_lock);
} else {
struct cfq_io_context *__cic;
@@ -1443,6 +1490,20 @@ cfq_get_io_context(struct cfq_data *cfqd, pid_t pid, gfp_t gfp_mask)
if (cic->key == cfqd)
goto out;
+ if (unlikely(!cic->key)) {
+ read_lock(&cfq_exit_lock);
+ if (list_empty(&cic->list))
+ ioc->cic = NULL;
+ else
+ ioc->cic = list_entry(cic->list.next,
+ struct cfq_io_context,
+ list);
+ read_unlock(&cfq_exit_lock);
+ kmem_cache_free(cfq_ioc_pool, cic);
+ atomic_dec(&ioc_count);
+ goto restart;
+ }
+
/*
* cic exists, check if we already are there. linear search
* should be ok here, the list will usually not be more than
@@ -1457,6 +1518,14 @@ cfq_get_io_context(struct cfq_data *cfqd, pid_t pid, gfp_t gfp_mask)
cic = __cic;
goto out;
}
+ if (unlikely(!__cic->key)) {
+ read_lock(&cfq_exit_lock);
+ list_del(&__cic->list);
+ read_unlock(&cfq_exit_lock);
+ kmem_cache_free(cfq_ioc_pool, __cic);
+ atomic_dec(&ioc_count);
+ goto restart;
+ }
}
/*
@@ -1469,8 +1538,10 @@ cfq_get_io_context(struct cfq_data *cfqd, pid_t pid, gfp_t gfp_mask)
__cic->ioc = ioc;
__cic->key = cfqd;
- atomic_inc(&cfqd->ref);
+ read_lock(&cfq_exit_lock);
list_add(&__cic->list, &cic->list);
+ list_add(&__cic->queue_list, &cfqd->cic_list);
+ read_unlock(&cfq_exit_lock);
cic = __cic;
}
@@ -1890,6 +1961,7 @@ cfq_set_request(request_queue_t *q, struct request *rq, struct bio *bio,
struct cfq_queue *cfqq;
struct cfq_rq *crq;
unsigned long flags;
+ int is_sync = key != CFQ_KEY_ASYNC;
might_sleep_if(gfp_mask & __GFP_WAIT);
@@ -1900,14 +1972,14 @@ cfq_set_request(request_queue_t *q, struct request *rq, struct bio *bio,
if (!cic)
goto queue_fail;
- if (!cic->cfqq) {
- cfqq = cfq_get_queue(cfqd, key, tsk->ioprio, gfp_mask);
+ if (!cic->cfqq[is_sync]) {
+ cfqq = cfq_get_queue(cfqd, key, tsk, gfp_mask);
if (!cfqq)
goto queue_fail;
- cic->cfqq = cfqq;
+ cic->cfqq[is_sync] = cfqq;
} else
- cfqq = cic->cfqq;
+ cfqq = cic->cfqq[is_sync];
cfqq->allocated[rw]++;
cfq_clear_cfqq_must_alloc(cfqq);
@@ -1924,7 +1996,7 @@ cfq_set_request(request_queue_t *q, struct request *rq, struct bio *bio,
crq->cfq_queue = cfqq;
crq->io_context = cic;
- if (rw == READ || process_sync(tsk))
+ if (is_sync)
cfq_mark_crq_is_sync(crq);
else
cfq_clear_crq_is_sync(crq);
@@ -2055,15 +2127,35 @@ static void cfq_shutdown_timer_wq(struct cfq_data *cfqd)
blk_sync_queue(cfqd->queue);
}
-static void cfq_put_cfqd(struct cfq_data *cfqd)
+static void cfq_exit_queue(elevator_t *e)
{
+ struct cfq_data *cfqd = e->elevator_data;
request_queue_t *q = cfqd->queue;
- if (!atomic_dec_and_test(&cfqd->ref))
- return;
+ cfq_shutdown_timer_wq(cfqd);
+ write_lock(&cfq_exit_lock);
+ spin_lock_irq(q->queue_lock);
+ if (cfqd->active_queue)
+ __cfq_slice_expired(cfqd, cfqd->active_queue, 0);
+ while(!list_empty(&cfqd->cic_list)) {
+ struct cfq_io_context *cic = list_entry(cfqd->cic_list.next,
+ struct cfq_io_context,
+ queue_list);
+ if (cic->cfqq[ASYNC]) {
+ cfq_put_queue(cic->cfqq[ASYNC]);
+ cic->cfqq[ASYNC] = NULL;
+ }
+ if (cic->cfqq[SYNC]) {
+ cfq_put_queue(cic->cfqq[SYNC]);
+ cic->cfqq[SYNC] = NULL;
+ }
+ cic->key = NULL;
+ list_del_init(&cic->queue_list);
+ }
+ spin_unlock_irq(q->queue_lock);
+ write_unlock(&cfq_exit_lock);
cfq_shutdown_timer_wq(cfqd);
- blk_put_queue(q);
mempool_destroy(cfqd->crq_pool);
kfree(cfqd->crq_hash);
@@ -2071,14 +2163,6 @@ static void cfq_put_cfqd(struct cfq_data *cfqd)
kfree(cfqd);
}
-static void cfq_exit_queue(elevator_t *e)
-{
- struct cfq_data *cfqd = e->elevator_data;
-
- cfq_shutdown_timer_wq(cfqd);
- cfq_put_cfqd(cfqd);
-}
-
static int cfq_init_queue(request_queue_t *q, elevator_t *e)
{
struct cfq_data *cfqd;
@@ -2097,6 +2181,7 @@ static int cfq_init_queue(request_queue_t *q, elevator_t *e)
INIT_LIST_HEAD(&cfqd->cur_rr);
INIT_LIST_HEAD(&cfqd->idle_rr);
INIT_LIST_HEAD(&cfqd->empty_list);
+ INIT_LIST_HEAD(&cfqd->cic_list);
cfqd->crq_hash = kmalloc(sizeof(struct hlist_head) * CFQ_MHASH_ENTRIES, GFP_KERNEL);
if (!cfqd->crq_hash)
@@ -2118,7 +2203,6 @@ static int cfq_init_queue(request_queue_t *q, elevator_t *e)
e->elevator_data = cfqd;
cfqd->queue = q;
- atomic_inc(&q->refcnt);
cfqd->max_queued = q->nr_requests / 4;
q->nr_batching = cfq_queued;
@@ -2133,8 +2217,6 @@ static int cfq_init_queue(request_queue_t *q, elevator_t *e)
INIT_WORK(&cfqd->unplug_work, cfq_kick_queue, q);
- atomic_set(&cfqd->ref, 1);
-
cfqd->cfq_queued = cfq_queued;
cfqd->cfq_quantum = cfq_quantum;
cfqd->cfq_fifo_expire[0] = cfq_fifo_expire[0];
@@ -2193,11 +2275,6 @@ fail:
/*
* sysfs parts below -->
*/
-struct cfq_fs_entry {
- struct attribute attr;
- ssize_t (*show)(struct cfq_data *, char *);
- ssize_t (*store)(struct cfq_data *, const char *, size_t);
-};
static ssize_t
cfq_var_show(unsigned int var, char *page)
@@ -2215,8 +2292,9 @@ cfq_var_store(unsigned int *var, const char *page, size_t count)
}
#define SHOW_FUNCTION(__FUNC, __VAR, __CONV) \
-static ssize_t __FUNC(struct cfq_data *cfqd, char *page) \
+static ssize_t __FUNC(elevator_t *e, char *page) \
{ \
+ struct cfq_data *cfqd = e->elevator_data; \
unsigned int __data = __VAR; \
if (__CONV) \
__data = jiffies_to_msecs(__data); \
@@ -2226,8 +2304,8 @@ SHOW_FUNCTION(cfq_quantum_show, cfqd->cfq_quantum, 0);
SHOW_FUNCTION(cfq_queued_show, cfqd->cfq_queued, 0);
SHOW_FUNCTION(cfq_fifo_expire_sync_show, cfqd->cfq_fifo_expire[1], 1);
SHOW_FUNCTION(cfq_fifo_expire_async_show, cfqd->cfq_fifo_expire[0], 1);
-SHOW_FUNCTION(cfq_back_max_show, cfqd->cfq_back_max, 0);
-SHOW_FUNCTION(cfq_back_penalty_show, cfqd->cfq_back_penalty, 0);
+SHOW_FUNCTION(cfq_back_seek_max_show, cfqd->cfq_back_max, 0);
+SHOW_FUNCTION(cfq_back_seek_penalty_show, cfqd->cfq_back_penalty, 0);
SHOW_FUNCTION(cfq_slice_idle_show, cfqd->cfq_slice_idle, 1);
SHOW_FUNCTION(cfq_slice_sync_show, cfqd->cfq_slice[1], 1);
SHOW_FUNCTION(cfq_slice_async_show, cfqd->cfq_slice[0], 1);
@@ -2236,8 +2314,9 @@ SHOW_FUNCTION(cfq_max_depth_show, cfqd->cfq_max_depth, 0);
#undef SHOW_FUNCTION
#define STORE_FUNCTION(__FUNC, __PTR, MIN, MAX, __CONV) \
-static ssize_t __FUNC(struct cfq_data *cfqd, const char *page, size_t count) \
+static ssize_t __FUNC(elevator_t *e, const char *page, size_t count) \
{ \
+ struct cfq_data *cfqd = e->elevator_data; \
unsigned int __data; \
int ret = cfq_var_store(&__data, (page), count); \
if (__data < (MIN)) \
@@ -2254,8 +2333,8 @@ STORE_FUNCTION(cfq_quantum_store, &cfqd->cfq_quantum, 1, UINT_MAX, 0);
STORE_FUNCTION(cfq_queued_store, &cfqd->cfq_queued, 1, UINT_MAX, 0);
STORE_FUNCTION(cfq_fifo_expire_sync_store, &cfqd->cfq_fifo_expire[1], 1, UINT_MAX, 1);
STORE_FUNCTION(cfq_fifo_expire_async_store, &cfqd->cfq_fifo_expire[0], 1, UINT_MAX, 1);
-STORE_FUNCTION(cfq_back_max_store, &cfqd->cfq_back_max, 0, UINT_MAX, 0);
-STORE_FUNCTION(cfq_back_penalty_store, &cfqd->cfq_back_penalty, 1, UINT_MAX, 0);
+STORE_FUNCTION(cfq_back_seek_max_store, &cfqd->cfq_back_max, 0, UINT_MAX, 0);
+STORE_FUNCTION(cfq_back_seek_penalty_store, &cfqd->cfq_back_penalty, 1, UINT_MAX, 0);
STORE_FUNCTION(cfq_slice_idle_store, &cfqd->cfq_slice_idle, 0, UINT_MAX, 1);
STORE_FUNCTION(cfq_slice_sync_store, &cfqd->cfq_slice[1], 1, UINT_MAX, 1);
STORE_FUNCTION(cfq_slice_async_store, &cfqd->cfq_slice[0], 1, UINT_MAX, 1);
@@ -2263,112 +2342,22 @@ STORE_FUNCTION(cfq_slice_async_rq_store, &cfqd->cfq_slice_async_rq, 1, UINT_MAX,
STORE_FUNCTION(cfq_max_depth_store, &cfqd->cfq_max_depth, 1, UINT_MAX, 0);
#undef STORE_FUNCTION
-static struct cfq_fs_entry cfq_quantum_entry = {
- .attr = {.name = "quantum", .mode = S_IRUGO | S_IWUSR },
- .show = cfq_quantum_show,
- .store = cfq_quantum_store,
-};
-static struct cfq_fs_entry cfq_queued_entry = {
- .attr = {.name = "queued", .mode = S_IRUGO | S_IWUSR },
- .show = cfq_queued_show,
- .store = cfq_queued_store,
-};
-static struct cfq_fs_entry cfq_fifo_expire_sync_entry = {
- .attr = {.name = "fifo_expire_sync", .mode = S_IRUGO | S_IWUSR },
- .show = cfq_fifo_expire_sync_show,
- .store = cfq_fifo_expire_sync_store,
-};
-static struct cfq_fs_entry cfq_fifo_expire_async_entry = {
- .attr = {.name = "fifo_expire_async", .mode = S_IRUGO | S_IWUSR },
- .show = cfq_fifo_expire_async_show,
- .store = cfq_fifo_expire_async_store,
-};
-static struct cfq_fs_entry cfq_back_max_entry = {
- .attr = {.name = "back_seek_max", .mode = S_IRUGO | S_IWUSR },
- .show = cfq_back_max_show,
- .store = cfq_back_max_store,
-};
-static struct cfq_fs_entry cfq_back_penalty_entry = {
- .attr = {.name = "back_seek_penalty", .mode = S_IRUGO | S_IWUSR },
- .show = cfq_back_penalty_show,
- .store = cfq_back_penalty_store,
-};
-static struct cfq_fs_entry cfq_slice_sync_entry = {
- .attr = {.name = "slice_sync", .mode = S_IRUGO | S_IWUSR },
- .show = cfq_slice_sync_show,
- .store = cfq_slice_sync_store,
-};
-static struct cfq_fs_entry cfq_slice_async_entry = {
- .attr = {.name = "slice_async", .mode = S_IRUGO | S_IWUSR },
- .show = cfq_slice_async_show,
- .store = cfq_slice_async_store,
-};
-static struct cfq_fs_entry cfq_slice_async_rq_entry = {
- .attr = {.name = "slice_async_rq", .mode = S_IRUGO | S_IWUSR },
- .show = cfq_slice_async_rq_show,
- .store = cfq_slice_async_rq_store,
-};
-static struct cfq_fs_entry cfq_slice_idle_entry = {
- .attr = {.name = "slice_idle", .mode = S_IRUGO | S_IWUSR },
- .show = cfq_slice_idle_show,
- .store = cfq_slice_idle_store,
-};
-static struct cfq_fs_entry cfq_max_depth_entry = {
- .attr = {.name = "max_depth", .mode = S_IRUGO | S_IWUSR },
- .show = cfq_max_depth_show,
- .store = cfq_max_depth_store,
-};
-
-static struct attribute *default_attrs[] = {
- &cfq_quantum_entry.attr,
- &cfq_queued_entry.attr,
- &cfq_fifo_expire_sync_entry.attr,
- &cfq_fifo_expire_async_entry.attr,
- &cfq_back_max_entry.attr,
- &cfq_back_penalty_entry.attr,
- &cfq_slice_sync_entry.attr,
- &cfq_slice_async_entry.attr,
- &cfq_slice_async_rq_entry.attr,
- &cfq_slice_idle_entry.attr,
- &cfq_max_depth_entry.attr,
- NULL,
-};
-
-#define to_cfq(atr) container_of((atr), struct cfq_fs_entry, attr)
-
-static ssize_t
-cfq_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
-{
- elevator_t *e = container_of(kobj, elevator_t, kobj);
- struct cfq_fs_entry *entry = to_cfq(attr);
-
- if (!entry->show)
- return -EIO;
-
- return entry->show(e->elevator_data, page);
-}
-
-static ssize_t
-cfq_attr_store(struct kobject *kobj, struct attribute *attr,
- const char *page, size_t length)
-{
- elevator_t *e = container_of(kobj, elevator_t, kobj);
- struct cfq_fs_entry *entry = to_cfq(attr);
-
- if (!entry->store)
- return -EIO;
-
- return entry->store(e->elevator_data, page, length);
-}
-
-static struct sysfs_ops cfq_sysfs_ops = {
- .show = cfq_attr_show,
- .store = cfq_attr_store,
-};
-
-static struct kobj_type cfq_ktype = {
- .sysfs_ops = &cfq_sysfs_ops,
- .default_attrs = default_attrs,
+#define CFQ_ATTR(name) \
+ __ATTR(name, S_IRUGO|S_IWUSR, cfq_##name##_show, cfq_##name##_store)
+
+static struct elv_fs_entry cfq_attrs[] = {
+ CFQ_ATTR(quantum),
+ CFQ_ATTR(queued),
+ CFQ_ATTR(fifo_expire_sync),
+ CFQ_ATTR(fifo_expire_async),
+ CFQ_ATTR(back_seek_max),
+ CFQ_ATTR(back_seek_penalty),
+ CFQ_ATTR(slice_sync),
+ CFQ_ATTR(slice_async),
+ CFQ_ATTR(slice_async_rq),
+ CFQ_ATTR(slice_idle),
+ CFQ_ATTR(max_depth),
+ __ATTR_NULL
};
static struct elevator_type iosched_cfq = {
@@ -2389,8 +2378,9 @@ static struct elevator_type iosched_cfq = {
.elevator_may_queue_fn = cfq_may_queue,
.elevator_init_fn = cfq_init_queue,
.elevator_exit_fn = cfq_exit_queue,
+ .trim = cfq_trim,
},
- .elevator_ktype = &cfq_ktype,
+ .elevator_attrs = cfq_attrs,
.elevator_name = "cfq",
.elevator_owner = THIS_MODULE,
};
@@ -2419,7 +2409,13 @@ static int __init cfq_init(void)
static void __exit cfq_exit(void)
{
+ DECLARE_COMPLETION(all_gone);
elv_unregister(&iosched_cfq);
+ ioc_gone = &all_gone;
+ barrier();
+ if (atomic_read(&ioc_count))
+ complete(ioc_gone);
+ synchronize_rcu();
cfq_slab_kill();
}
Copyright © 1996 – 2023 Jason A. Donenfeld. All Rights Reverse Engineered.