aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/staging/lustre/lnet/lnet/lib-ptl.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/staging/lustre/lnet/lnet/lib-ptl.c')
-rw-r--r--drivers/staging/lustre/lnet/lnet/lib-ptl.c258
1 files changed, 154 insertions, 104 deletions
diff --git a/drivers/staging/lustre/lnet/lnet/lib-ptl.c b/drivers/staging/lustre/lnet/lnet/lib-ptl.c
index bd7b071b2873..3947e8b711c0 100644
--- a/drivers/staging/lustre/lnet/lnet/lib-ptl.c
+++ b/drivers/staging/lustre/lnet/lnet/lib-ptl.c
@@ -13,11 +13,6 @@
* General Public License version 2 for more details (a copy is included
* in the LICENSE file that accompanied this code).
*
- * You should have received a copy of the GNU General Public License
- * version 2 along with this program; if not, write to the
- * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
- * Boston, MA 021110-1307, USA
- *
* GPL HEADER END
*/
/*
@@ -50,7 +45,7 @@ lnet_ptl_match_type(unsigned int index, lnet_process_id_t match_id,
struct lnet_portal *ptl = the_lnet.ln_portals[index];
int unique;
- unique = ignore_bits == 0 &&
+ unique = !ignore_bits &&
match_id.nid != LNET_NID_ANY &&
match_id.pid != LNET_PID_ANY;
@@ -139,8 +134,10 @@ static int
lnet_try_match_md(lnet_libmd_t *md,
struct lnet_match_info *info, struct lnet_msg *msg)
{
- /* ALWAYS called holding the lnet_res_lock, and can't lnet_res_unlock;
- * lnet_match_blocked_msg() relies on this to avoid races */
+ /*
+ * ALWAYS called holding the lnet_res_lock, and can't lnet_res_unlock;
+ * lnet_match_blocked_msg() relies on this to avoid races
+ */
unsigned int offset;
unsigned int mlength;
lnet_me_t *me = md->md_me;
@@ -150,7 +147,7 @@ lnet_try_match_md(lnet_libmd_t *md,
return LNET_MATCHMD_NONE | LNET_MATCHMD_EXHAUSTED;
/* mismatched MD op */
- if ((md->md_options & info->mi_opc) == 0)
+ if (!(md->md_options & info->mi_opc))
return LNET_MATCHMD_NONE;
/* mismatched ME nid/pid? */
@@ -163,17 +160,17 @@ lnet_try_match_md(lnet_libmd_t *md,
return LNET_MATCHMD_NONE;
/* mismatched ME matchbits? */
- if (((me->me_match_bits ^ info->mi_mbits) & ~me->me_ignore_bits) != 0)
+ if ((me->me_match_bits ^ info->mi_mbits) & ~me->me_ignore_bits)
return LNET_MATCHMD_NONE;
/* Hurrah! This _is_ a match; check it out... */
- if ((md->md_options & LNET_MD_MANAGE_REMOTE) == 0)
+ if (!(md->md_options & LNET_MD_MANAGE_REMOTE))
offset = md->md_offset;
else
offset = info->mi_roffset;
- if ((md->md_options & LNET_MD_MAX_SIZE) != 0) {
+ if (md->md_options & LNET_MD_MAX_SIZE) {
mlength = md->md_max_size;
LASSERT(md->md_offset + mlength <= md->md_length);
} else {
@@ -182,7 +179,7 @@ lnet_try_match_md(lnet_libmd_t *md,
if (info->mi_rlength <= mlength) { /* fits in allowed space */
mlength = info->mi_rlength;
- } else if ((md->md_options & LNET_MD_TRUNCATE) == 0) {
+ } else if (!(md->md_options & LNET_MD_TRUNCATE)) {
/* this packet _really_ is too big */
CERROR("Matching packet from %s, match %llu length %d too big: %d left, %d allowed\n",
libcfs_id2str(info->mi_id), info->mi_mbits,
@@ -203,10 +200,12 @@ lnet_try_match_md(lnet_libmd_t *md,
if (!lnet_md_exhausted(md))
return LNET_MATCHMD_OK;
- /* Auto-unlink NOW, so the ME gets unlinked if required.
+ /*
+ * Auto-unlink NOW, so the ME gets unlinked if required.
* We bumped md->md_refcount above so the MD just gets flagged
- * for unlink when it is finalized. */
- if ((md->md_flags & LNET_MD_FLAG_AUTO_UNLINK) != 0)
+ * for unlink when it is finalized.
+ */
+ if (md->md_flags & LNET_MD_FLAG_AUTO_UNLINK)
lnet_md_unlink(md);
return LNET_MATCHMD_OK | LNET_MATCHMD_EXHAUSTED;
@@ -239,7 +238,7 @@ lnet_mt_of_attach(unsigned int index, lnet_process_id_t id,
ptl = the_lnet.ln_portals[index];
mtable = lnet_match2mt(ptl, id, mbits);
- if (mtable != NULL) /* unique portal or only one match-table */
+ if (mtable) /* unique portal or only one match-table */
return mtable;
/* it's a wildcard portal */
@@ -248,8 +247,10 @@ lnet_mt_of_attach(unsigned int index, lnet_process_id_t id,
return NULL;
case LNET_INS_BEFORE:
case LNET_INS_AFTER:
- /* posted by no affinity thread, always hash to specific
- * match-table to avoid buffer stealing which is heavy */
+ /*
+ * posted by no affinity thread, always hash to specific
+ * match-table to avoid buffer stealing which is heavy
+ */
return ptl->ptl_mtables[ptl->ptl_index % LNET_CPT_NUMBER];
case LNET_INS_LOCAL:
/* posted by cpu-affinity thread */
@@ -274,7 +275,7 @@ lnet_mt_of_match(struct lnet_match_info *info, struct lnet_msg *msg)
LASSERT(lnet_ptl_is_wildcard(ptl) || lnet_ptl_is_unique(ptl));
mtable = lnet_match2mt(ptl, info->mi_id, info->mi_mbits);
- if (mtable != NULL)
+ if (mtable)
return mtable;
/* it's a wildcard portal */
@@ -298,10 +299,12 @@ lnet_mt_of_match(struct lnet_match_info *info, struct lnet_msg *msg)
/* is there any active entry for this portal? */
nmaps = ptl->ptl_mt_nmaps;
/* map to an active mtable to avoid heavy "stealing" */
- if (nmaps != 0) {
- /* NB: there is possibility that ptl_mt_maps is being
+ if (nmaps) {
+ /*
+ * NB: there is possibility that ptl_mt_maps is being
* changed because we are not under protection of
- * lnet_ptl_lock, but it shouldn't hurt anything */
+ * lnet_ptl_lock, but it shouldn't hurt anything
+ */
cpt = ptl->ptl_mt_maps[rotor % nmaps];
}
}
@@ -331,7 +334,7 @@ lnet_mt_test_exhausted(struct lnet_match_table *mtable, int pos)
bmap = &mtable->mt_exhausted[pos >> LNET_MT_BITS_U64];
pos &= (1 << LNET_MT_BITS_U64) - 1;
- return ((*bmap) & (1ULL << pos)) != 0;
+ return (*bmap & (1ULL << pos));
}
static void
@@ -357,16 +360,15 @@ lnet_mt_match_head(struct lnet_match_table *mtable,
lnet_process_id_t id, __u64 mbits)
{
struct lnet_portal *ptl = the_lnet.ln_portals[mtable->mt_portal];
+ unsigned long hash = mbits;
- if (lnet_ptl_is_wildcard(ptl)) {
- return &mtable->mt_mhash[mbits & LNET_MT_HASH_MASK];
- } else {
- unsigned long hash = mbits + id.nid + id.pid;
+ if (!lnet_ptl_is_wildcard(ptl)) {
+ hash += id.nid + id.pid;
LASSERT(lnet_ptl_is_unique(ptl));
hash = hash_long(hash, LNET_MT_HASH_BITS);
- return &mtable->mt_mhash[hash];
}
+ return &mtable->mt_mhash[hash & LNET_MT_HASH_MASK];
}
int
@@ -391,18 +393,20 @@ lnet_mt_match_md(struct lnet_match_table *mtable,
list_for_each_entry_safe(me, tmp, head, me_list) {
/* ME attached but MD not attached yet */
- if (me->me_md == NULL)
+ if (!me->me_md)
continue;
LASSERT(me == me->me_md->md_me);
rc = lnet_try_match_md(me->me_md, info, msg);
- if ((rc & LNET_MATCHMD_EXHAUSTED) == 0)
+ if (!(rc & LNET_MATCHMD_EXHAUSTED))
exhausted = 0; /* mlist is not empty */
- if ((rc & LNET_MATCHMD_FINISH) != 0) {
- /* don't return EXHAUSTED bit because we don't know
- * whether the mlist is empty or not */
+ if (rc & LNET_MATCHMD_FINISH) {
+ /*
+ * don't return EXHAUSTED bit because we don't know
+ * whether the mlist is empty or not
+ */
return rc & ~LNET_MATCHMD_EXHAUSTED;
}
}
@@ -413,7 +417,7 @@ lnet_mt_match_md(struct lnet_match_table *mtable,
exhausted = 0;
}
- if (exhausted == 0 && head == &mtable->mt_mhash[LNET_MT_HASH_IGNORE]) {
+ if (!exhausted && head == &mtable->mt_mhash[LNET_MT_HASH_IGNORE]) {
head = lnet_mt_match_head(mtable, info->mi_id, info->mi_mbits);
goto again; /* re-check MEs w/o ignore-bits */
}
@@ -430,8 +434,10 @@ lnet_ptl_match_early(struct lnet_portal *ptl, struct lnet_msg *msg)
{
int rc;
- /* message arrived before any buffer posting on this portal,
- * simply delay or drop this message */
+ /*
+ * message arrived before any buffer posting on this portal,
+ * simply delay or drop this message
+ */
if (likely(lnet_ptl_is_wildcard(ptl) || lnet_ptl_is_unique(ptl)))
return 0;
@@ -446,7 +452,7 @@ lnet_ptl_match_early(struct lnet_portal *ptl, struct lnet_msg *msg)
if (msg->msg_rx_ready_delay) {
msg->msg_rx_delayed = 1;
list_add_tail(&msg->msg_list,
- &ptl->ptl_msg_delayed);
+ &ptl->ptl_msg_delayed);
}
rc = LNET_MATCHMD_NONE;
} else {
@@ -465,9 +471,13 @@ lnet_ptl_match_delay(struct lnet_portal *ptl,
int rc = 0;
int i;
- /* steal buffer from other CPTs, and delay it if nothing to steal,
- * this function is more expensive than a regular match, but we
- * don't expect it can happen a lot */
+ /**
+ * Steal buffer from other CPTs, and delay msg if nothing to
+ * steal. This function is more expensive than a regular
+ * match, but we don't expect it can happen a lot. The return
+ * code contains one of LNET_MATCHMD_OK, LNET_MATCHMD_DROP, or
+ * LNET_MATCHMD_NONE.
+ */
LASSERT(lnet_ptl_is_wildcard(ptl));
for (i = 0; i < LNET_CPT_NUMBER; i++) {
@@ -476,56 +486,77 @@ lnet_ptl_match_delay(struct lnet_portal *ptl,
cpt = (first + i) % LNET_CPT_NUMBER;
mtable = ptl->ptl_mtables[cpt];
- if (i != 0 && i != LNET_CPT_NUMBER - 1 && !mtable->mt_enabled)
+ if (i && i != LNET_CPT_NUMBER - 1 && !mtable->mt_enabled)
continue;
lnet_res_lock(cpt);
lnet_ptl_lock(ptl);
- if (i == 0) { /* the first try, attach on stealing list */
+ if (!i) {
+ /* The first try, add to stealing list. */
list_add_tail(&msg->msg_list,
- &ptl->ptl_msg_stealing);
+ &ptl->ptl_msg_stealing);
}
- if (!list_empty(&msg->msg_list)) { /* on stealing list */
+ if (!list_empty(&msg->msg_list)) {
+ /* On stealing list. */
rc = lnet_mt_match_md(mtable, info, msg);
- if ((rc & LNET_MATCHMD_EXHAUSTED) != 0 &&
+ if ((rc & LNET_MATCHMD_EXHAUSTED) &&
mtable->mt_enabled)
lnet_ptl_disable_mt(ptl, cpt);
- if ((rc & LNET_MATCHMD_FINISH) != 0)
+ if (rc & LNET_MATCHMD_FINISH) {
+ /* Match found, remove from stealing list. */
+ list_del_init(&msg->msg_list);
+ } else if (i == LNET_CPT_NUMBER - 1 || /* (1) */
+ !ptl->ptl_mt_nmaps || /* (2) */
+ (ptl->ptl_mt_nmaps == 1 && /* (3) */
+ ptl->ptl_mt_maps[0] == cpt)) {
+ /**
+ * No match found, and this is either
+ * (1) the last cpt to check, or
+ * (2) there is no active cpt, or
+ * (3) this is the only active cpt.
+ * There is nothing to steal: delay or
+ * drop the message.
+ */
list_del_init(&msg->msg_list);
- } else {
- /* could be matched by lnet_ptl_attach_md()
- * which is called by another thread */
- rc = msg->msg_md == NULL ?
- LNET_MATCHMD_DROP : LNET_MATCHMD_OK;
- }
-
- if (!list_empty(&msg->msg_list) && /* not matched yet */
- (i == LNET_CPT_NUMBER - 1 || /* the last CPT */
- ptl->ptl_mt_nmaps == 0 || /* no active CPT */
- (ptl->ptl_mt_nmaps == 1 && /* the only active CPT */
- ptl->ptl_mt_maps[0] == cpt))) {
- /* nothing to steal, delay or drop */
- list_del_init(&msg->msg_list);
-
- if (lnet_ptl_is_lazy(ptl)) {
- msg->msg_rx_delayed = 1;
- list_add_tail(&msg->msg_list,
- &ptl->ptl_msg_delayed);
- rc = LNET_MATCHMD_NONE;
+ if (lnet_ptl_is_lazy(ptl)) {
+ msg->msg_rx_delayed = 1;
+ list_add_tail(&msg->msg_list,
+ &ptl->ptl_msg_delayed);
+ rc = LNET_MATCHMD_NONE;
+ } else {
+ rc = LNET_MATCHMD_DROP;
+ }
} else {
- rc = LNET_MATCHMD_DROP;
+ /* Do another iteration. */
+ rc = 0;
}
+ } else {
+ /**
+ * No longer on stealing list: another thread
+ * matched the message in lnet_ptl_attach_md().
+ * We are now expected to handle the message.
+ */
+ rc = !msg->msg_md ?
+ LNET_MATCHMD_DROP : LNET_MATCHMD_OK;
}
lnet_ptl_unlock(ptl);
lnet_res_unlock(cpt);
- if ((rc & LNET_MATCHMD_FINISH) != 0 || msg->msg_rx_delayed)
+ /**
+ * Note that test (1) above ensures that we always
+ * exit the loop through this break statement.
+ *
+ * LNET_MATCHMD_NONE means msg was added to the
+ * delayed queue, and we may no longer reference it
+ * after lnet_ptl_unlock() and lnet_res_unlock().
+ */
+ if (rc & (LNET_MATCHMD_FINISH | LNET_MATCHMD_NONE))
break;
}
@@ -551,7 +582,7 @@ lnet_ptl_match_md(struct lnet_match_info *info, struct lnet_msg *msg)
ptl = the_lnet.ln_portals[info->mi_portal];
rc = lnet_ptl_match_early(ptl, msg);
- if (rc != 0) /* matched or delayed early message */
+ if (rc) /* matched or delayed early message */
return rc;
mtable = lnet_mt_of_match(info, msg);
@@ -563,13 +594,13 @@ lnet_ptl_match_md(struct lnet_match_info *info, struct lnet_msg *msg)
}
rc = lnet_mt_match_md(mtable, info, msg);
- if ((rc & LNET_MATCHMD_EXHAUSTED) != 0 && mtable->mt_enabled) {
+ if ((rc & LNET_MATCHMD_EXHAUSTED) && mtable->mt_enabled) {
lnet_ptl_lock(ptl);
lnet_ptl_disable_mt(ptl, mtable->mt_cpt);
lnet_ptl_unlock(ptl);
}
- if ((rc & LNET_MATCHMD_FINISH) != 0) /* matched or dropping */
+ if (rc & LNET_MATCHMD_FINISH) /* matched or dropping */
goto out1;
if (!msg->msg_rx_ready_delay)
@@ -587,13 +618,14 @@ lnet_ptl_match_md(struct lnet_match_info *info, struct lnet_msg *msg)
lnet_ptl_unlock(ptl);
lnet_res_unlock(mtable->mt_cpt);
-
+ rc = LNET_MATCHMD_NONE;
} else {
lnet_res_unlock(mtable->mt_cpt);
rc = lnet_ptl_match_delay(ptl, info, msg);
}
- if (msg->msg_rx_delayed) {
+ /* LNET_MATCHMD_NONE means msg was added to the delay queue */
+ if (rc & LNET_MATCHMD_NONE) {
CDEBUG(D_NET,
"Delaying %s from %s ptl %d MB %#llx off %d len %d\n",
info->mi_opc == LNET_MD_OP_PUT ? "PUT" : "GET",
@@ -630,7 +662,7 @@ lnet_ptl_attach_md(lnet_me_t *me, lnet_libmd_t *md,
int exhausted = 0;
int cpt;
- LASSERT(md->md_refcount == 0); /* a brand new MD */
+ LASSERT(!md->md_refcount); /* a brand new MD */
me->me_md = md;
md->md_me = me;
@@ -664,15 +696,15 @@ lnet_ptl_attach_md(lnet_me_t *me, lnet_libmd_t *md,
rc = lnet_try_match_md(md, &info, msg);
- exhausted = (rc & LNET_MATCHMD_EXHAUSTED) != 0;
- if ((rc & LNET_MATCHMD_NONE) != 0) {
+ exhausted = (rc & LNET_MATCHMD_EXHAUSTED);
+ if (rc & LNET_MATCHMD_NONE) {
if (exhausted)
break;
continue;
}
/* Hurrah! This _is_ a match */
- LASSERT((rc & LNET_MATCHMD_FINISH) != 0);
+ LASSERT(rc & LNET_MATCHMD_FINISH);
list_del_init(&msg->msg_list);
if (head == &ptl->ptl_msg_stealing) {
@@ -682,7 +714,7 @@ lnet_ptl_attach_md(lnet_me_t *me, lnet_libmd_t *md,
continue;
}
- if ((rc & LNET_MATCHMD_OK) != 0) {
+ if (rc & LNET_MATCHMD_OK) {
list_add_tail(&msg->msg_list, matches);
CDEBUG(D_NET, "Resuming delayed PUT from %s portal %d match %llu offset %d length %d.\n",
@@ -717,7 +749,7 @@ lnet_ptl_cleanup(struct lnet_portal *ptl)
struct lnet_match_table *mtable;
int i;
- if (ptl->ptl_mtables == NULL) /* uninitialized portal */
+ if (!ptl->ptl_mtables) /* uninitialized portal */
return;
LASSERT(list_empty(&ptl->ptl_msg_delayed));
@@ -727,7 +759,7 @@ lnet_ptl_cleanup(struct lnet_portal *ptl)
lnet_me_t *me;
int j;
- if (mtable->mt_mhash == NULL) /* uninitialized match-table */
+ if (!mtable->mt_mhash) /* uninitialized match-table */
continue;
mhash = mtable->mt_mhash;
@@ -735,7 +767,7 @@ lnet_ptl_cleanup(struct lnet_portal *ptl)
for (j = 0; j < LNET_MT_HASH_SIZE + 1; j++) {
while (!list_empty(&mhash[j])) {
me = list_entry(mhash[j].next,
- lnet_me_t, me_list);
+ lnet_me_t, me_list);
CERROR("Active ME %p on exit\n", me);
list_del(&me->me_list);
lnet_me_free(me);
@@ -759,7 +791,7 @@ lnet_ptl_setup(struct lnet_portal *ptl, int index)
ptl->ptl_mtables = cfs_percpt_alloc(lnet_cpt_table(),
sizeof(struct lnet_match_table));
- if (ptl->ptl_mtables == NULL) {
+ if (!ptl->ptl_mtables) {
CERROR("Failed to create match table for portal %d\n", index);
return -ENOMEM;
}
@@ -772,7 +804,7 @@ lnet_ptl_setup(struct lnet_portal *ptl, int index)
/* the extra entry is for MEs with ignore bits */
LIBCFS_CPT_ALLOC(mhash, lnet_cpt_table(), i,
sizeof(*mhash) * (LNET_MT_HASH_SIZE + 1));
- if (mhash == NULL) {
+ if (!mhash) {
CERROR("Failed to create match hash for portal %d\n",
index);
goto failed;
@@ -800,7 +832,7 @@ lnet_portals_destroy(void)
{
int i;
- if (the_lnet.ln_portals == NULL)
+ if (!the_lnet.ln_portals)
return;
for (i = 0; i < the_lnet.ln_nportals; i++)
@@ -820,7 +852,7 @@ lnet_portals_create(void)
the_lnet.ln_nportals = MAX_PORTALS;
the_lnet.ln_portals = cfs_array_alloc(the_lnet.ln_nportals, size);
- if (the_lnet.ln_portals == NULL) {
+ if (!the_lnet.ln_portals) {
CERROR("Failed to allocate portals table\n");
return -ENOMEM;
}
@@ -886,17 +918,8 @@ LNetSetLazyPortal(int portal)
}
EXPORT_SYMBOL(LNetSetLazyPortal);
-/**
- * Turn off the lazy portal attribute. Delayed requests on the portal,
- * if any, will be all dropped when this function returns.
- *
- * \param portal Index of the portal to disable the lazy attribute on.
- *
- * \retval 0 On success.
- * \retval -EINVAL If \a portal is not a valid index.
- */
int
-LNetClearLazyPortal(int portal)
+lnet_clear_lazy_portal(struct lnet_ni *ni, int portal, char *reason)
{
struct lnet_portal *ptl;
LIST_HEAD(zombies);
@@ -915,21 +938,48 @@ LNetClearLazyPortal(int portal)
return 0;
}
- if (the_lnet.ln_shutdown)
- CWARN("Active lazy portal %d on exit\n", portal);
- else
- CDEBUG(D_NET, "clearing portal %d lazy\n", portal);
+ if (ni) {
+ struct lnet_msg *msg, *tmp;
+
+ /* grab all messages which are on the NI passed in */
+ list_for_each_entry_safe(msg, tmp, &ptl->ptl_msg_delayed,
+ msg_list) {
+ if (msg->msg_rxpeer->lp_ni == ni)
+ list_move(&msg->msg_list, &zombies);
+ }
+ } else {
+ if (the_lnet.ln_shutdown)
+ CWARN("Active lazy portal %d on exit\n", portal);
+ else
+ CDEBUG(D_NET, "clearing portal %d lazy\n", portal);
- /* grab all the blocked messages atomically */
- list_splice_init(&ptl->ptl_msg_delayed, &zombies);
+ /* grab all the blocked messages atomically */
+ list_splice_init(&ptl->ptl_msg_delayed, &zombies);
- lnet_ptl_unsetopt(ptl, LNET_PTL_LAZY);
+ lnet_ptl_unsetopt(ptl, LNET_PTL_LAZY);
+ }
lnet_ptl_unlock(ptl);
lnet_res_unlock(LNET_LOCK_EX);
- lnet_drop_delayed_msg_list(&zombies, "Clearing lazy portal attr");
+ lnet_drop_delayed_msg_list(&zombies, reason);
return 0;
}
+
+/**
+ * Turn off the lazy portal attribute. Delayed requests on the portal,
+ * if any, will be all dropped when this function returns.
+ *
+ * \param portal Index of the portal to disable the lazy attribute on.
+ *
+ * \retval 0 On success.
+ * \retval -EINVAL If \a portal is not a valid index.
+ */
+int
+LNetClearLazyPortal(int portal)
+{
+ return lnet_clear_lazy_portal(NULL, portal,
+ "Clearing lazy portal attr");
+}
EXPORT_SYMBOL(LNetClearLazyPortal);