diff options
Diffstat (limited to 'drivers/staging/lustre/lustre/osc')
-rw-r--r-- | drivers/staging/lustre/lustre/osc/lproc_osc.c | 2 | ||||
-rw-r--r-- | drivers/staging/lustre/lustre/osc/osc_cache.c | 4 | ||||
-rw-r--r-- | drivers/staging/lustre/lustre/osc/osc_cl_internal.h | 4 | ||||
-rw-r--r-- | drivers/staging/lustre/lustre/osc/osc_internal.h | 3 | ||||
-rw-r--r-- | drivers/staging/lustre/lustre/osc/osc_io.c | 52 | ||||
-rw-r--r-- | drivers/staging/lustre/lustre/osc/osc_lock.c | 60 | ||||
-rw-r--r-- | drivers/staging/lustre/lustre/osc/osc_object.c | 12 | ||||
-rw-r--r-- | drivers/staging/lustre/lustre/osc/osc_page.c | 77 | ||||
-rw-r--r-- | drivers/staging/lustre/lustre/osc/osc_request.c | 17 |
9 files changed, 145 insertions, 86 deletions
diff --git a/drivers/staging/lustre/lustre/osc/lproc_osc.c b/drivers/staging/lustre/lustre/osc/lproc_osc.c index 575b2969ad83..86f252d6adbd 100644 --- a/drivers/staging/lustre/lustre/osc/lproc_osc.c +++ b/drivers/staging/lustre/lustre/osc/lproc_osc.c @@ -229,7 +229,7 @@ static ssize_t osc_cached_mb_seq_write(struct file *file, rc = atomic_long_read(&cli->cl_lru_in_list) - pages_number; if (rc > 0) { struct lu_env *env; - int refcheck; + u16 refcheck; env = cl_env_get(&refcheck); if (!IS_ERR(env)) { diff --git a/drivers/staging/lustre/lustre/osc/osc_cache.c b/drivers/staging/lustre/lustre/osc/osc_cache.c index 0490478393df..c5ccf568313a 100644 --- a/drivers/staging/lustre/lustre/osc/osc_cache.c +++ b/drivers/staging/lustre/lustre/osc/osc_cache.c @@ -898,6 +898,7 @@ int osc_extent_finish(const struct lu_env *env, struct osc_extent *ext, int offset = last_off & ~PAGE_MASK; int count = last_count + (offset & (blocksize - 1)); int end = (offset + last_count) & (blocksize - 1); + if (end) count += blocksize - end; @@ -988,7 +989,7 @@ static int osc_extent_truncate(struct osc_extent *ext, pgoff_t trunc_index, int grants = 0; int nr_pages = 0; int rc = 0; - int refcheck; + u16 refcheck; LASSERT(sanity_check(ext) == 0); EASSERT(ext->oe_state == OES_TRUNC, ext); @@ -2790,7 +2791,6 @@ again: * We have to wait for this extent because we can't * truncate that page. */ - LASSERT(!ext->oe_hp); OSC_EXTENT_DUMP(D_CACHE, ext, "waiting for busy extent\n"); waiting = osc_extent_get(ext); diff --git a/drivers/staging/lustre/lustre/osc/osc_cl_internal.h b/drivers/staging/lustre/lustre/osc/osc_cl_internal.h index c09ab97d64ae..270212f4e5cf 100644 --- a/drivers/staging/lustre/lustre/osc/osc_cl_internal.h +++ b/drivers/staging/lustre/lustre/osc/osc_cl_internal.h @@ -62,7 +62,9 @@ struct osc_io { /** super class */ struct cl_io_slice oi_cl; /** true if this io is lockless. */ - unsigned int oi_lockless; + unsigned int oi_lockless:1, + /** true if this io is counted as active IO */ + oi_is_active:1; /** how many LRU pages are reserved for this IO */ unsigned long oi_lru_reserved; diff --git a/drivers/staging/lustre/lustre/osc/osc_internal.h b/drivers/staging/lustre/lustre/osc/osc_internal.h index 8abd83f26716..845e795d8795 100644 --- a/drivers/staging/lustre/lustre/osc/osc_internal.h +++ b/drivers/staging/lustre/lustre/osc/osc_internal.h @@ -133,7 +133,8 @@ int osc_build_rpc(const struct lu_env *env, struct client_obd *cli, struct list_head *ext_list, int cmd); long osc_lru_shrink(const struct lu_env *env, struct client_obd *cli, long target, bool force); -long osc_lru_reclaim(struct client_obd *cli, unsigned long npages); +unsigned long osc_lru_reserve(struct client_obd *cli, unsigned long npages); +void osc_lru_unreserve(struct client_obd *cli, unsigned long npages); unsigned long osc_ldlm_weigh_ast(struct ldlm_lock *dlmlock); diff --git a/drivers/staging/lustre/lustre/osc/osc_io.c b/drivers/staging/lustre/lustre/osc/osc_io.c index 0b4cc4283b05..cbab80092442 100644 --- a/drivers/staging/lustre/lustre/osc/osc_io.c +++ b/drivers/staging/lustre/lustre/osc/osc_io.c @@ -216,7 +216,7 @@ static int osc_io_submit(const struct lu_env *env, struct cl_object *obj = ios->cis_obj; cl_object_attr_lock(obj); - attr->cat_mtime = LTIME_S(CURRENT_TIME); + attr->cat_mtime = ktime_get_real_seconds(); attr->cat_ctime = attr->cat_mtime; cl_object_attr_update(env, obj, attr, CAT_MTIME | CAT_CTIME); cl_object_attr_unlock(obj); @@ -256,7 +256,7 @@ static void osc_page_touch_at(const struct lu_env *env, kms > loi->loi_kms ? "" : "not ", loi->loi_kms, kms, loi->loi_lvb.lvb_size); - attr->cat_ctime = LTIME_S(CURRENT_TIME); + attr->cat_ctime = ktime_get_real_seconds(); attr->cat_mtime = attr->cat_ctime; valid = CAT_MTIME | CAT_CTIME; if (kms > loi->loi_kms) { @@ -354,7 +354,10 @@ static int osc_io_iter_init(const struct lu_env *env, spin_lock(&imp->imp_lock); if (likely(!imp->imp_invalid)) { + struct osc_io *oio = osc_env_io(env); + atomic_inc(&osc->oo_nr_ios); + oio->oi_is_active = 1; rc = 0; } spin_unlock(&imp->imp_lock); @@ -368,10 +371,7 @@ static int osc_io_write_iter_init(const struct lu_env *env, struct cl_io *io = ios->cis_io; struct osc_io *oio = osc_env_io(env); struct osc_object *osc = cl2osc(ios->cis_obj); - struct client_obd *cli = osc_cli(osc); - unsigned long c; unsigned long npages; - unsigned long max_pages; if (cl_io_is_append(io)) return osc_io_iter_init(env, ios); @@ -380,31 +380,7 @@ static int osc_io_write_iter_init(const struct lu_env *env, if (io->u.ci_rw.crw_pos & ~PAGE_MASK) ++npages; - max_pages = cli->cl_max_pages_per_rpc * cli->cl_max_rpcs_in_flight; - if (npages > max_pages) - npages = max_pages; - - c = atomic_long_read(cli->cl_lru_left); - if (c < npages && osc_lru_reclaim(cli, npages) > 0) - c = atomic_long_read(cli->cl_lru_left); - while (c >= npages) { - if (c == atomic_long_cmpxchg(cli->cl_lru_left, c, c - npages)) { - oio->oi_lru_reserved = npages; - break; - } - c = atomic_long_read(cli->cl_lru_left); - } - if (atomic_long_read(cli->cl_lru_left) < max_pages) { - /* - * If there aren't enough pages in the per-OSC LRU then - * wake up the LRU thread to try and clear out space, so - * we don't block if pages are being dirtied quickly. - */ - CDEBUG(D_CACHE, "%s: queue LRU, left: %lu/%ld.\n", - cli_name(cli), atomic_long_read(cli->cl_lru_left), - max_pages); - (void)ptlrpcd_queue_work(cli->cl_lru_work); - } + oio->oi_lru_reserved = osc_lru_reserve(osc_cli(osc), npages); return osc_io_iter_init(env, ios); } @@ -412,11 +388,16 @@ static int osc_io_write_iter_init(const struct lu_env *env, static void osc_io_iter_fini(const struct lu_env *env, const struct cl_io_slice *ios) { - struct osc_object *osc = cl2osc(ios->cis_obj); + struct osc_io *oio = osc_env_io(env); + + if (oio->oi_is_active) { + struct osc_object *osc = cl2osc(ios->cis_obj); - LASSERT(atomic_read(&osc->oo_nr_ios) > 0); - if (atomic_dec_and_test(&osc->oo_nr_ios)) - wake_up_all(&osc->oo_io_waitq); + oio->oi_is_active = 0; + LASSERT(atomic_read(&osc->oo_nr_ios) > 0); + if (atomic_dec_and_test(&osc->oo_nr_ios)) + wake_up_all(&osc->oo_io_waitq); + } } static void osc_io_write_iter_fini(const struct lu_env *env, @@ -424,10 +405,9 @@ static void osc_io_write_iter_fini(const struct lu_env *env, { struct osc_io *oio = osc_env_io(env); struct osc_object *osc = cl2osc(ios->cis_obj); - struct client_obd *cli = osc_cli(osc); if (oio->oi_lru_reserved > 0) { - atomic_long_add(oio->oi_lru_reserved, cli->cl_lru_left); + osc_lru_unreserve(osc_cli(osc), oio->oi_lru_reserved); oio->oi_lru_reserved = 0; } oio->oi_write_osclock = NULL; diff --git a/drivers/staging/lustre/lustre/osc/osc_lock.c b/drivers/staging/lustre/lustre/osc/osc_lock.c index 5f799a4c78f9..940c10c1d7a1 100644 --- a/drivers/staging/lustre/lustre/osc/osc_lock.c +++ b/drivers/staging/lustre/lustre/osc/osc_lock.c @@ -167,6 +167,8 @@ static __u64 osc_enq2ldlm_flags(__u32 enqflags) result |= LDLM_FL_AST_DISCARD_DATA; if (enqflags & CEF_PEEK) result |= LDLM_FL_TEST_LOCK; + if (enqflags & CEF_LOCK_MATCH) + result |= LDLM_FL_MATCH_LOCK; return result; } @@ -295,7 +297,7 @@ static int osc_lock_upcall(void *cookie, struct lustre_handle *lockh, struct cl_lock_slice *slice = &oscl->ols_cl; struct lu_env *env; int rc; - int refcheck; + u16 refcheck; env = cl_env_get(&refcheck); /* should never happen, similar to osc_ldlm_blocking_ast(). */ @@ -347,7 +349,7 @@ static int osc_lock_upcall_agl(void *cookie, struct lustre_handle *lockh, struct osc_object *osc = cookie; struct ldlm_lock *dlmlock; struct lu_env *env; - int refcheck; + u16 refcheck; env = cl_env_get(&refcheck); LASSERT(!IS_ERR(env)); @@ -382,7 +384,7 @@ static int osc_lock_flush(struct osc_object *obj, pgoff_t start, pgoff_t end, enum cl_lock_mode mode, int discard) { struct lu_env *env; - int refcheck; + u16 refcheck; int rc = 0; int rc2 = 0; @@ -536,7 +538,7 @@ static int osc_ldlm_blocking_ast(struct ldlm_lock *dlmlock, } case LDLM_CB_CANCELING: { struct lu_env *env; - int refcheck; + u16 refcheck; /* * This can be called in the context of outer IO, e.g., @@ -573,7 +575,7 @@ static int osc_ldlm_glimpse_ast(struct ldlm_lock *dlmlock, void *data) struct req_capsule *cap; struct cl_object *obj = NULL; int result; - int refcheck; + u16 refcheck; LASSERT(lustre_msg_get_opc(req->rq_reqmsg) == LDLM_GL_CALLBACK); @@ -684,7 +686,7 @@ unsigned long osc_ldlm_weigh_ast(struct ldlm_lock *dlmlock) struct osc_lock *oscl; unsigned long weight; bool found = false; - int refcheck; + u16 refcheck; might_sleep(); /* @@ -838,13 +840,14 @@ static void osc_lock_wake_waiters(const struct lu_env *env, spin_unlock(&oscl->ols_lock); } -static void osc_lock_enqueue_wait(const struct lu_env *env, - struct osc_object *obj, - struct osc_lock *oscl) +static int osc_lock_enqueue_wait(const struct lu_env *env, + struct osc_object *obj, + struct osc_lock *oscl) { struct osc_lock *tmp_oscl; struct cl_lock_descr *need = &oscl->ols_cl.cls_lock->cll_descr; struct cl_sync_io *waiter = &osc_env_info(env)->oti_anchor; + int rc = 0; spin_lock(&obj->oo_ol_spin); list_add_tail(&oscl->ols_nextlock_oscobj, &obj->oo_ol_list); @@ -881,13 +884,17 @@ restart: spin_unlock(&tmp_oscl->ols_lock); spin_unlock(&obj->oo_ol_spin); - (void)cl_sync_io_wait(env, waiter, 0); - + rc = cl_sync_io_wait(env, waiter, 0); spin_lock(&obj->oo_ol_spin); + if (rc < 0) + break; + oscl->ols_owner = NULL; goto restart; } spin_unlock(&obj->oo_ol_spin); + + return rc; } /** @@ -935,7 +942,9 @@ static int osc_lock_enqueue(const struct lu_env *env, goto enqueue_base; } - osc_lock_enqueue_wait(env, osc, oscl); + result = osc_lock_enqueue_wait(env, osc, oscl); + if (result < 0) + goto out; /* we can grant lockless lock right after all conflicting locks * are canceled. @@ -960,7 +969,6 @@ enqueue_base: * osc_lock. */ ostid_build_res_name(&osc->oo_oinfo->loi_oi, resname); - osc_lock_build_einfo(env, lock, osc, &oscl->ols_einfo); osc_lock_build_policy(env, lock, policy); if (oscl->ols_agl) { oscl->ols_einfo.ei_cbdata = NULL; @@ -975,18 +983,7 @@ enqueue_base: upcall, cookie, &oscl->ols_einfo, PTLRPCD_SET, async, oscl->ols_agl); - if (result != 0) { - oscl->ols_state = OLS_CANCELLED; - osc_lock_wake_waiters(env, osc, oscl); - - /* hide error for AGL lock. */ - if (oscl->ols_agl) { - cl_object_put(env, osc2cl(osc)); - result = 0; - } - if (anchor) - cl_sync_io_note(env, anchor, result); - } else { + if (!result) { if (osc_lock_is_lockless(oscl)) { oio->oi_lockless = 1; } else if (!async) { @@ -994,6 +991,18 @@ enqueue_base: LASSERT(oscl->ols_hold); LASSERT(oscl->ols_dlmlock); } + } else if (oscl->ols_agl) { + cl_object_put(env, osc2cl(osc)); + result = 0; + } + +out: + if (result < 0) { + oscl->ols_state = OLS_CANCELLED; + osc_lock_wake_waiters(env, osc, oscl); + + if (anchor) + cl_sync_io_note(env, anchor, result); } return result; } @@ -1157,6 +1166,7 @@ int osc_lock_init(const struct lu_env *env, oscl->ols_flags |= LDLM_FL_BLOCK_GRANTED; oscl->ols_glimpse = 1; } + osc_lock_build_einfo(env, lock, cl2osc(obj), &oscl->ols_einfo); cl_lock_slice_add(lock, &oscl->ols_cl, obj, &osc_lock_ops); diff --git a/drivers/staging/lustre/lustre/osc/osc_object.c b/drivers/staging/lustre/lustre/osc/osc_object.c index d3e5ca7db7b2..fa621bda1ffe 100644 --- a/drivers/staging/lustre/lustre/osc/osc_object.c +++ b/drivers/staging/lustre/lustre/osc/osc_object.c @@ -200,10 +200,6 @@ static int osc_object_prune(const struct lu_env *env, struct cl_object *obj) struct osc_object *osc = cl2osc(obj); struct ldlm_res_id *resname = &osc_env_info(env)->oti_resname; - LASSERTF(osc->oo_npages == 0, - DFID "still have %lu pages, obj: %p, osc: %p\n", - PFID(lu_object_fid(&obj->co_lu)), osc->oo_npages, obj, osc); - /* DLM locks don't hold a reference of osc_object so we have to * clear it before the object is being destroyed. */ @@ -457,9 +453,15 @@ int osc_object_invalidate(const struct lu_env *env, struct osc_object *osc) l_wait_event(osc->oo_io_waitq, !atomic_read(&osc->oo_nr_ios), &lwi); - /* Discard all pages of this object. */ + /* Discard all dirty pages of this object. */ osc_cache_truncate_start(env, osc, 0, NULL); + /* Discard all caching pages */ + osc_lock_discard_pages(env, osc, 0, CL_PAGE_EOF, CLM_WRITE); + + /* Clear ast data of dlm lock. Do this after discarding all pages */ + osc_object_prune(env, osc2cl(osc)); + return 0; } diff --git a/drivers/staging/lustre/lustre/osc/osc_page.c b/drivers/staging/lustre/lustre/osc/osc_page.c index ab9d0d7bb943..ed8a0dc18ee5 100644 --- a/drivers/staging/lustre/lustre/osc/osc_page.c +++ b/drivers/staging/lustre/lustre/osc/osc_page.c @@ -42,8 +42,8 @@ static void osc_lru_del(struct client_obd *cli, struct osc_page *opg); static void osc_lru_use(struct client_obd *cli, struct osc_page *opg); -static int osc_lru_reserve(const struct lu_env *env, struct osc_object *obj, - struct osc_page *opg); +static int osc_lru_alloc(const struct lu_env *env, struct client_obd *cli, + struct osc_page *opg); /** \addtogroup osc * @{ @@ -273,7 +273,7 @@ int osc_page_init(const struct lu_env *env, struct cl_object *obj, /* reserve an LRU space for this page */ if (page->cp_type == CPT_CACHEABLE && result == 0) { - result = osc_lru_reserve(env, osc, opg); + result = osc_lru_alloc(env, osc_cli(osc), opg); if (result == 0) { spin_lock(&osc->oo_tree_lock); result = radix_tree_insert(&osc->oo_tree, index, opg); @@ -676,12 +676,12 @@ long osc_lru_shrink(const struct lu_env *env, struct client_obd *cli, * LRU pages in batch. Therefore, the actual number is adjusted at least * max_pages_per_rpc. */ -long osc_lru_reclaim(struct client_obd *cli, unsigned long npages) +static long osc_lru_reclaim(struct client_obd *cli, unsigned long npages) { struct lu_env *env; struct cl_client_cache *cache = cli->cl_cache; int max_scans; - int refcheck; + u16 refcheck; long rc = 0; LASSERT(cache); @@ -749,18 +749,17 @@ out: } /** - * osc_lru_reserve() is called to reserve an LRU slot for a cl_page. + * osc_lru_alloc() is called to reserve an LRU slot for a cl_page. * * Usually the LRU slots are reserved in osc_io_iter_rw_init(). * Only in the case that the LRU slots are in extreme shortage, it should * have reserved enough slots for an IO. */ -static int osc_lru_reserve(const struct lu_env *env, struct osc_object *obj, - struct osc_page *opg) +static int osc_lru_alloc(const struct lu_env *env, struct client_obd *cli, + struct osc_page *opg) { struct l_wait_info lwi = LWI_INTR(LWI_ON_SIGNAL_NOOP, NULL); struct osc_io *oio = osc_env_io(env); - struct client_obd *cli = osc_cli(obj); int rc = 0; if (!cli->cl_cache) /* shall not be in LRU */ @@ -801,6 +800,64 @@ out: } /** + * osc_lru_reserve() is called to reserve enough LRU slots for I/O. + * + * The benefit of doing this is to reduce contention against atomic counter + * cl_lru_left by changing it from per-page access to per-IO access. + */ +unsigned long osc_lru_reserve(struct client_obd *cli, unsigned long npages) +{ + unsigned long reserved = 0; + unsigned long max_pages; + unsigned long c; + + /* + * reserve a full RPC window at most to avoid that a thread accidentally + * consumes too many LRU slots + */ + max_pages = cli->cl_max_pages_per_rpc * cli->cl_max_rpcs_in_flight; + if (npages > max_pages) + npages = max_pages; + + c = atomic_long_read(cli->cl_lru_left); + if (c < npages && osc_lru_reclaim(cli, npages) > 0) + c = atomic_long_read(cli->cl_lru_left); + while (c >= npages) { + if (c == atomic_long_cmpxchg(cli->cl_lru_left, c, c - npages)) { + reserved = npages; + break; + } + c = atomic_long_read(cli->cl_lru_left); + } + if (atomic_long_read(cli->cl_lru_left) < max_pages) { + /* + * If there aren't enough pages in the per-OSC LRU then + * wake up the LRU thread to try and clear out space, so + * we don't block if pages are being dirtied quickly. + */ + CDEBUG(D_CACHE, "%s: queue LRU, left: %lu/%ld.\n", + cli_name(cli), atomic_long_read(cli->cl_lru_left), + max_pages); + (void)ptlrpcd_queue_work(cli->cl_lru_work); + } + + return reserved; +} + +/** + * osc_lru_unreserve() is called to unreserve LRU slots. + * + * LRU slots reserved by osc_lru_reserve() may have entries left due to several + * reasons such as page already existing or I/O error. Those reserved slots + * should be freed by calling this function. + */ +void osc_lru_unreserve(struct client_obd *cli, unsigned long npages) +{ + atomic_long_add(npages, cli->cl_lru_left); + wake_up_all(&osc_lru_waitq); +} + +/** * Atomic operations are expensive. We accumulate the accounting for the * same page pgdat to get better performance. * In practice this can work pretty good because the pages in the same RPC @@ -988,7 +1045,7 @@ unsigned long osc_cache_shrink_scan(struct shrinker *sk, struct client_obd *cli; struct lu_env *env; long shrank = 0; - int refcheck; + u16 refcheck; int rc; if (!sc->nr_to_scan) diff --git a/drivers/staging/lustre/lustre/osc/osc_request.c b/drivers/staging/lustre/lustre/osc/osc_request.c index c4cfe18c3294..d8aa3fb468c7 100644 --- a/drivers/staging/lustre/lustre/osc/osc_request.c +++ b/drivers/staging/lustre/lustre/osc/osc_request.c @@ -1195,7 +1195,8 @@ static int osc_brw_prep_request(int cmd, struct client_obd *cli, return rc; } -static int check_write_checksum(struct obdo *oa, const lnet_process_id_t *peer, +static int check_write_checksum(struct obdo *oa, + const struct lnet_process_id *peer, __u32 client_cksum, __u32 server_cksum, int nob, u32 page_count, struct brw_page **pga, enum cksum_type client_cksum_type) @@ -1245,7 +1246,7 @@ static int check_write_checksum(struct obdo *oa, const lnet_process_id_t *peer, static int osc_brw_fini_request(struct ptlrpc_request *req, int rc) { struct osc_brw_async_args *aa = (void *)&req->rq_async_args; - const lnet_process_id_t *peer = + const struct lnet_process_id *peer = &req->rq_import->imp_connection->c_peer; struct client_obd *cli = aa->aa_cli; struct ost_body *body; @@ -2011,7 +2012,7 @@ int osc_enqueue_base(struct obd_export *exp, struct ldlm_res_id *res_id, } no_match: - if (*flags & LDLM_FL_TEST_LOCK) + if (*flags & (LDLM_FL_TEST_LOCK | LDLM_FL_MATCH_LOCK)) return -ENOLCK; if (intent) { req = ptlrpc_request_alloc(class_exp2cliimp(exp), @@ -2495,7 +2496,13 @@ static int osc_ldlm_resource_invalidate(struct cfs_hash *hs, osc = lock->l_ast_data; cl_object_get(osc2cl(osc)); } - lock->l_ast_data = NULL; + + /* + * clear LDLM_FL_CLEANED flag to make sure it will be canceled + * by the 2nd round of ldlm_namespace_clean() call in + * osc_import_event(). + */ + ldlm_clear_cleaned(lock); } unlock_res(res); @@ -2532,7 +2539,7 @@ static int osc_import_event(struct obd_device *obd, case IMP_EVENT_INVALIDATE: { struct ldlm_namespace *ns = obd->obd_namespace; struct lu_env *env; - int refcheck; + u16 refcheck; ldlm_namespace_cleanup(ns, LDLM_FL_LOCAL_ONLY); |