diff options
Diffstat (limited to 'drivers/block/drbd/drbd_nl.c')
| -rw-r--r-- | drivers/block/drbd/drbd_nl.c | 1361 |
1 files changed, 1247 insertions, 114 deletions
diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c index e80cbefbc2b5..c055c5e12f24 100644 --- a/drivers/block/drbd/drbd_nl.c +++ b/drivers/block/drbd/drbd_nl.c @@ -36,6 +36,7 @@ #include "drbd_int.h" #include "drbd_protocol.h" #include "drbd_req.h" +#include "drbd_state_change.h" #include <asm/unaligned.h> #include <linux/drbd_limits.h> #include <linux/kthread.h> @@ -75,11 +76,24 @@ int drbd_adm_get_status(struct sk_buff *skb, struct genl_info *info); int drbd_adm_get_timeout_type(struct sk_buff *skb, struct genl_info *info); /* .dumpit */ int drbd_adm_get_status_all(struct sk_buff *skb, struct netlink_callback *cb); +int drbd_adm_dump_resources(struct sk_buff *skb, struct netlink_callback *cb); +int drbd_adm_dump_devices(struct sk_buff *skb, struct netlink_callback *cb); +int drbd_adm_dump_devices_done(struct netlink_callback *cb); +int drbd_adm_dump_connections(struct sk_buff *skb, struct netlink_callback *cb); +int drbd_adm_dump_connections_done(struct netlink_callback *cb); +int drbd_adm_dump_peer_devices(struct sk_buff *skb, struct netlink_callback *cb); +int drbd_adm_dump_peer_devices_done(struct netlink_callback *cb); +int drbd_adm_get_initial_state(struct sk_buff *skb, struct netlink_callback *cb); #include <linux/drbd_genl_api.h> #include "drbd_nla.h" #include <linux/genl_magic_func.h> +static atomic_t drbd_genl_seq = ATOMIC_INIT(2); /* two. */ +static atomic_t notify_genl_seq = ATOMIC_INIT(2); /* two. */ + +DEFINE_MUTEX(notification_mutex); + /* used blkdev_get_by_path, to claim our meta data device(s) */ static char *drbd_m_holder = "Hands off! this is DRBD's meta data device."; @@ -349,6 +363,7 @@ int drbd_khelper(struct drbd_device *device, char *cmd) sib.sib_reason = SIB_HELPER_PRE; sib.helper_name = cmd; drbd_bcast_event(device, &sib); + notify_helper(NOTIFY_CALL, device, connection, cmd, 0); ret = call_usermodehelper(usermode_helper, argv, envp, UMH_WAIT_PROC); if (ret) drbd_warn(device, "helper command: %s %s %s exit code %u (0x%x)\n", @@ -361,6 +376,7 @@ int drbd_khelper(struct drbd_device *device, char *cmd) sib.sib_reason = SIB_HELPER_POST; sib.helper_exit_code = ret; drbd_bcast_event(device, &sib); + notify_helper(NOTIFY_RESPONSE, device, connection, cmd, ret); if (current == connection->worker.task) clear_bit(CALLBACK_PENDING, &connection->flags); @@ -388,6 +404,7 @@ static int conn_khelper(struct drbd_connection *connection, char *cmd) drbd_info(connection, "helper command: %s %s %s\n", usermode_helper, cmd, resource_name); /* TODO: conn_bcast_event() ?? */ + notify_helper(NOTIFY_CALL, NULL, connection, cmd, 0); ret = call_usermodehelper(usermode_helper, argv, envp, UMH_WAIT_PROC); if (ret) @@ -399,6 +416,7 @@ static int conn_khelper(struct drbd_connection *connection, char *cmd) usermode_helper, cmd, resource_name, (ret >> 8) & 0xff, ret); /* TODO: conn_bcast_event() ?? */ + notify_helper(NOTIFY_RESPONSE, NULL, connection, cmd, ret); if (ret < 0) /* Ignore any ERRNOs we got. */ ret = 0; @@ -847,9 +865,11 @@ char *ppsize(char *buf, unsigned long long size) * and can be long lived. * This changes an device->flag, is triggered by drbd internals, * and should be short-lived. */ +/* It needs to be a counter, since multiple threads might + independently suspend and resume IO. */ void drbd_suspend_io(struct drbd_device *device) { - set_bit(SUSPEND_IO, &device->flags); + atomic_inc(&device->suspend_cnt); if (drbd_suspended(device)) return; wait_event(device->misc_wait, !atomic_read(&device->ap_bio_cnt)); @@ -857,8 +877,8 @@ void drbd_suspend_io(struct drbd_device *device) void drbd_resume_io(struct drbd_device *device) { - clear_bit(SUSPEND_IO, &device->flags); - wake_up(&device->misc_wait); + if (atomic_dec_and_test(&device->suspend_cnt)) + wake_up(&device->misc_wait); } /** @@ -871,27 +891,32 @@ void drbd_resume_io(struct drbd_device *device) enum determine_dev_size drbd_determine_dev_size(struct drbd_device *device, enum dds_flags flags, struct resize_parms *rs) __must_hold(local) { - sector_t prev_first_sect, prev_size; /* previous meta location */ - sector_t la_size_sect, u_size; + struct md_offsets_and_sizes { + u64 last_agreed_sect; + u64 md_offset; + s32 al_offset; + s32 bm_offset; + u32 md_size_sect; + + u32 al_stripes; + u32 al_stripe_size_4k; + } prev; + sector_t u_size, size; struct drbd_md *md = &device->ldev->md; - u32 prev_al_stripe_size_4k; - u32 prev_al_stripes; - sector_t size; char ppb[10]; void *buffer; int md_moved, la_size_changed; enum determine_dev_size rv = DS_UNCHANGED; - /* race: - * application request passes inc_ap_bio, - * but then cannot get an AL-reference. - * this function later may wait on ap_bio_cnt == 0. -> deadlock. + /* We may change the on-disk offsets of our meta data below. Lock out + * anything that may cause meta data IO, to avoid acting on incomplete + * layout changes or scribbling over meta data that is in the process + * of being moved. * - * to avoid that: - * Suspend IO right here. - * still lock the act_log to not trigger ASSERTs there. - */ + * Move is not exactly correct, btw, currently we have all our meta + * data in core memory, to "move" it we just write it all out, there + * are no reads. */ drbd_suspend_io(device); buffer = drbd_md_get_buffer(device, __func__); /* Lock meta-data IO */ if (!buffer) { @@ -899,19 +924,17 @@ drbd_determine_dev_size(struct drbd_device *device, enum dds_flags flags, struct return DS_ERROR; } - /* no wait necessary anymore, actually we could assert that */ - wait_event(device->al_wait, lc_try_lock(device->act_log)); - - prev_first_sect = drbd_md_first_sector(device->ldev); - prev_size = device->ldev->md.md_size_sect; - la_size_sect = device->ldev->md.la_size_sect; + /* remember current offset and sizes */ + prev.last_agreed_sect = md->la_size_sect; + prev.md_offset = md->md_offset; + prev.al_offset = md->al_offset; + prev.bm_offset = md->bm_offset; + prev.md_size_sect = md->md_size_sect; + prev.al_stripes = md->al_stripes; + prev.al_stripe_size_4k = md->al_stripe_size_4k; if (rs) { /* rs is non NULL if we should change the AL layout only */ - - prev_al_stripes = md->al_stripes; - prev_al_stripe_size_4k = md->al_stripe_size_4k; - md->al_stripes = rs->al_stripes; md->al_stripe_size_4k = rs->al_stripe_size / 4; md->al_size_4k = (u64)rs->al_stripes * rs->al_stripe_size / 4; @@ -924,7 +947,7 @@ drbd_determine_dev_size(struct drbd_device *device, enum dds_flags flags, struct rcu_read_unlock(); size = drbd_new_dev_size(device, device->ldev, u_size, flags & DDSF_FORCED); - if (size < la_size_sect) { + if (size < prev.last_agreed_sect) { if (rs && u_size == 0) { /* Remove "rs &&" later. This check should always be active, but right now the receiver expects the permissive behavior */ @@ -945,30 +968,29 @@ drbd_determine_dev_size(struct drbd_device *device, enum dds_flags flags, struct err = drbd_bm_resize(device, size, !(flags & DDSF_NO_RESYNC)); if (unlikely(err)) { /* currently there is only one error: ENOMEM! */ - size = drbd_bm_capacity(device)>>1; + size = drbd_bm_capacity(device); if (size == 0) { drbd_err(device, "OUT OF MEMORY! " "Could not allocate bitmap!\n"); } else { drbd_err(device, "BM resizing failed. " - "Leaving size unchanged at size = %lu KB\n", - (unsigned long)size); + "Leaving size unchanged\n"); } rv = DS_ERROR; } /* racy, see comments above. */ drbd_set_my_capacity(device, size); - device->ldev->md.la_size_sect = size; + md->la_size_sect = size; drbd_info(device, "size = %s (%llu KB)\n", ppsize(ppb, size>>1), (unsigned long long)size>>1); } if (rv <= DS_ERROR) goto err_out; - la_size_changed = (la_size_sect != device->ldev->md.la_size_sect); + la_size_changed = (prev.last_agreed_sect != md->la_size_sect); - md_moved = prev_first_sect != drbd_md_first_sector(device->ldev) - || prev_size != device->ldev->md.md_size_sect; + md_moved = prev.md_offset != md->md_offset + || prev.md_size_sect != md->md_size_sect; if (la_size_changed || md_moved || rs) { u32 prev_flags; @@ -977,20 +999,29 @@ drbd_determine_dev_size(struct drbd_device *device, enum dds_flags flags, struct * Clear the timer, to avoid scary "timer expired!" messages, * "Superblock" is written out at least twice below, anyways. */ del_timer(&device->md_sync_timer); - drbd_al_shrink(device); /* All extents inactive. */ + /* We won't change the "al-extents" setting, we just may need + * to move the on-disk location of the activity log ringbuffer. + * Lock for transaction is good enough, it may well be "dirty" + * or even "starving". */ + wait_event(device->al_wait, lc_try_lock_for_transaction(device->act_log)); + + /* mark current on-disk bitmap and activity log as unreliable */ prev_flags = md->flags; - md->flags &= ~MDF_PRIMARY_IND; + md->flags |= MDF_FULL_SYNC | MDF_AL_DISABLED; drbd_md_write(device, buffer); + drbd_al_initialize(device, buffer); + drbd_info(device, "Writing the whole bitmap, %s\n", la_size_changed && md_moved ? "size changed and md moved" : la_size_changed ? "size changed" : "md moved"); /* next line implicitly does drbd_suspend_io()+drbd_resume_io() */ drbd_bitmap_io(device, md_moved ? &drbd_bm_write_all : &drbd_bm_write, "size changed", BM_LOCKED_MASK); - drbd_initialize_al(device, buffer); + /* on-disk bitmap and activity log is authoritative again + * (unless there was an IO error meanwhile...) */ md->flags = prev_flags; drbd_md_write(device, buffer); @@ -999,20 +1030,22 @@ drbd_determine_dev_size(struct drbd_device *device, enum dds_flags flags, struct md->al_stripes, md->al_stripe_size_4k * 4); } - if (size > la_size_sect) - rv = la_size_sect ? DS_GREW : DS_GREW_FROM_ZERO; - if (size < la_size_sect) + if (size > prev.last_agreed_sect) + rv = prev.last_agreed_sect ? DS_GREW : DS_GREW_FROM_ZERO; + if (size < prev.last_agreed_sect) rv = DS_SHRUNK; if (0) { err_out: - if (rs) { - md->al_stripes = prev_al_stripes; - md->al_stripe_size_4k = prev_al_stripe_size_4k; - md->al_size_4k = (u64)prev_al_stripes * prev_al_stripe_size_4k; - - drbd_md_set_sector_offsets(device, device->ldev); - } + /* restore previous offset and sizes */ + md->la_size_sect = prev.last_agreed_sect; + md->md_offset = prev.md_offset; + md->al_offset = prev.al_offset; + md->bm_offset = prev.bm_offset; + md->md_size_sect = prev.md_size_sect; + md->al_stripes = prev.al_stripes; + md->al_stripe_size_4k = prev.al_stripe_size_4k; + md->al_size_4k = (u64)prev.al_stripes * prev.al_stripe_size_4k; } lc_unlock(device->act_log); wake_up(&device->al_wait); @@ -1115,8 +1148,7 @@ static int drbd_check_al_size(struct drbd_device *device, struct disk_conf *dc) lc_destroy(n); return -EBUSY; } else { - if (t) - lc_destroy(t); + lc_destroy(t); } drbd_md_mark_dirty(device); /* we changed device->act_log->nr_elemens */ return 0; @@ -1151,21 +1183,20 @@ static void drbd_setup_queue_param(struct drbd_device *device, struct drbd_backi if (b) { struct drbd_connection *connection = first_peer_device(device)->connection; + blk_queue_max_discard_sectors(q, DRBD_MAX_DISCARD_SECTORS); + if (blk_queue_discard(b) && (connection->cstate < C_CONNECTED || connection->agreed_features & FF_TRIM)) { - /* For now, don't allow more than one activity log extent worth of data - * to be discarded in one go. We may need to rework drbd_al_begin_io() - * to allow for even larger discard ranges */ - blk_queue_max_discard_sectors(q, DRBD_MAX_DISCARD_SECTORS); - + /* We don't care, stacking below should fix it for the local device. + * Whether or not it is a suitable granularity on the remote device + * is not our problem, really. If you care, you need to + * use devices with similar topology on all peers. */ + q->limits.discard_granularity = 512; queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); - /* REALLY? Is stacking secdiscard "legal"? */ - if (blk_queue_secdiscard(b)) - queue_flag_set_unlocked(QUEUE_FLAG_SECDISCARD, q); } else { blk_queue_max_discard_sectors(q, 0); queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q); - queue_flag_clear_unlocked(QUEUE_FLAG_SECDISCARD, q); + q->limits.discard_granularity = 0; } blk_queue_stack_limits(q, b); @@ -1177,6 +1208,12 @@ static void drbd_setup_queue_param(struct drbd_device *device, struct drbd_backi q->backing_dev_info.ra_pages = b->backing_dev_info.ra_pages; } } + /* To avoid confusion, if this queue does not support discard, clear + * max_discard_sectors, which is what lsblk -D reports to the user. */ + if (!blk_queue_discard(q)) { + blk_queue_max_discard_sectors(q, 0); + q->limits.discard_granularity = 0; + } } void drbd_reconsider_max_bio_size(struct drbd_device *device, struct drbd_backing_dev *bdev) @@ -1241,8 +1278,8 @@ static void conn_reconfig_done(struct drbd_connection *connection) connection->cstate == C_STANDALONE; spin_unlock_irq(&connection->resource->req_lock); if (stop_threads) { - /* asender is implicitly stopped by receiver - * in conn_disconnect() */ + /* ack_receiver thread and ack_sender workqueue are implicitly + * stopped by receiver in conn_disconnect() */ drbd_thread_stop(&connection->receiver); drbd_thread_stop(&connection->worker); } @@ -1389,13 +1426,13 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info) goto fail_unlock; } - write_lock_irq(&global_state_lock); + lock_all_resources(); retcode = drbd_resync_after_valid(device, new_disk_conf->resync_after); if (retcode == NO_ERROR) { rcu_assign_pointer(device->ldev->disk_conf, new_disk_conf); drbd_resync_after_changed(device); } - write_unlock_irq(&global_state_lock); + unlock_all_resources(); if (retcode != NO_ERROR) goto fail_unlock; @@ -1418,7 +1455,7 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info) set_bit(MD_NO_FUA, &device->flags); if (write_ordering_changed(old_disk_conf, new_disk_conf)) - drbd_bump_write_ordering(device->resource, NULL, WO_bdev_flush); + drbd_bump_write_ordering(device->resource, NULL, WO_BDEV_FLUSH); drbd_md_sync(device); @@ -1449,6 +1486,88 @@ success: return 0; } +static struct block_device *open_backing_dev(struct drbd_device *device, + const char *bdev_path, void *claim_ptr, bool do_bd_link) +{ + struct block_device *bdev; + int err = 0; + + bdev = blkdev_get_by_path(bdev_path, + FMODE_READ | FMODE_WRITE | FMODE_EXCL, claim_ptr); + if (IS_ERR(bdev)) { + drbd_err(device, "open(\"%s\") failed with %ld\n", + bdev_path, PTR_ERR(bdev)); + return bdev; + } + + if (!do_bd_link) + return bdev; + + err = bd_link_disk_holder(bdev, device->vdisk); + if (err) { + blkdev_put(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL); + drbd_err(device, "bd_link_disk_holder(\"%s\", ...) failed with %d\n", + bdev_path, err); + bdev = ERR_PTR(err); + } + return bdev; +} + +static int open_backing_devices(struct drbd_device *device, + struct disk_conf *new_disk_conf, + struct drbd_backing_dev *nbc) +{ + struct block_device *bdev; + + bdev = open_backing_dev(device, new_disk_conf->backing_dev, device, true); + if (IS_ERR(bdev)) + return ERR_OPEN_DISK; + nbc->backing_bdev = bdev; + + /* + * meta_dev_idx >= 0: external fixed size, possibly multiple + * drbd sharing one meta device. TODO in that case, paranoia + * check that [md_bdev, meta_dev_idx] is not yet used by some + * other drbd minor! (if you use drbd.conf + drbdadm, that + * should check it for you already; but if you don't, or + * someone fooled it, we need to double check here) + */ + bdev = open_backing_dev(device, new_disk_conf->meta_dev, + /* claim ptr: device, if claimed exclusively; shared drbd_m_holder, + * if potentially shared with other drbd minors */ + (new_disk_conf->meta_dev_idx < 0) ? (void*)device : (void*)drbd_m_holder, + /* avoid double bd_claim_by_disk() for the same (source,target) tuple, + * as would happen with internal metadata. */ + (new_disk_conf->meta_dev_idx != DRBD_MD_INDEX_FLEX_INT && + new_disk_conf->meta_dev_idx != DRBD_MD_INDEX_INTERNAL)); + if (IS_ERR(bdev)) + return ERR_OPEN_MD_DISK; + nbc->md_bdev = bdev; + return NO_ERROR; +} + +static void close_backing_dev(struct drbd_device *device, struct block_device *bdev, + bool do_bd_unlink) +{ + if (!bdev) + return; + if (do_bd_unlink) + bd_unlink_disk_holder(bdev, device->vdisk); + blkdev_put(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL); +} + +void drbd_backing_dev_free(struct drbd_device *device, struct drbd_backing_dev *ldev) +{ + if (ldev == NULL) + return; + + close_backing_dev(device, ldev->md_bdev, ldev->md_bdev != ldev->backing_bdev); + close_backing_dev(device, ldev->backing_bdev, true); + + kfree(ldev->disk_conf); + kfree(ldev); +} + int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info) { struct drbd_config_context adm_ctx; @@ -1462,7 +1581,6 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info) sector_t min_md_device_sectors; struct drbd_backing_dev *nbc = NULL; /* new_backing_conf */ struct disk_conf *new_disk_conf = NULL; - struct block_device *bdev; struct lru_cache *resync_lru = NULL; struct fifo_buffer *new_plan = NULL; union drbd_state ns, os; @@ -1478,7 +1596,7 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info) device = adm_ctx.device; mutex_lock(&adm_ctx.resource->adm_mutex); peer_device = first_peer_device(device); - connection = peer_device ? peer_device->connection : NULL; + connection = peer_device->connection; conn_reconfig_start(connection); /* if you want to reconfigure, please tear down first */ @@ -1539,12 +1657,6 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info) goto fail; } - write_lock_irq(&global_state_lock); - retcode = drbd_resync_after_valid(device, new_disk_conf->resync_after); - write_unlock_irq(&global_state_lock); - if (retcode != NO_ERROR) - goto fail; - rcu_read_lock(); nc = rcu_dereference(connection->net_conf); if (nc) { @@ -1556,35 +1668,9 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info) } rcu_read_unlock(); - bdev = blkdev_get_by_path(new_disk_conf->backing_dev, - FMODE_READ | FMODE_WRITE | FMODE_EXCL, device); - if (IS_ERR(bdev)) { - drbd_err(device, "open(\"%s\") failed with %ld\n", new_disk_conf->backing_dev, - PTR_ERR(bdev)); - retcode = ERR_OPEN_DISK; - goto fail; - } - nbc->backing_bdev = bdev; - - /* - * meta_dev_idx >= 0: external fixed size, possibly multiple - * drbd sharing one meta device. TODO in that case, paranoia - * check that [md_bdev, meta_dev_idx] is not yet used by some - * other drbd minor! (if you use drbd.conf + drbdadm, that - * should check it for you already; but if you don't, or - * someone fooled it, we need to double check here) - */ - bdev = blkdev_get_by_path(new_disk_conf->meta_dev, - FMODE_READ | FMODE_WRITE | FMODE_EXCL, - (new_disk_conf->meta_dev_idx < 0) ? - (void *)device : (void *)drbd_m_holder); - if (IS_ERR(bdev)) { - drbd_err(device, "open(\"%s\") failed with %ld\n", new_disk_conf->meta_dev, - PTR_ERR(bdev)); - retcode = ERR_OPEN_MD_DISK; + retcode = open_backing_devices(device, new_disk_conf, nbc); + if (retcode != NO_ERROR) goto fail; - } - nbc->md_bdev = bdev; if ((nbc->backing_bdev == nbc->md_bdev) != (new_disk_conf->meta_dev_idx == DRBD_MD_INDEX_INTERNAL || @@ -1707,6 +1793,13 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info) goto force_diskless_dec; } + lock_all_resources(); + retcode = drbd_resync_after_valid(device, new_disk_conf->resync_after); + if (retcode != NO_ERROR) { + unlock_all_resources(); + goto force_diskless_dec; + } + /* Reset the "barriers don't work" bits here, then force meta data to * be written, to ensure we determine if barriers are supported. */ if (new_disk_conf->md_flushes) @@ -1727,7 +1820,9 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info) new_disk_conf = NULL; new_plan = NULL; - drbd_bump_write_ordering(device->resource, device->ldev, WO_bdev_flush); + drbd_resync_after_changed(device); + drbd_bump_write_ordering(device->resource, device->ldev, WO_BDEV_FLUSH); + unlock_all_resources(); if (drbd_md_test_flag(device->ldev, MDF_CRASHED_PRIMARY)) set_bit(CRASHED_PRIMARY, &device->flags); @@ -1875,12 +1970,8 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info) fail: conn_reconfig_done(connection); if (nbc) { - if (nbc->backing_bdev) - blkdev_put(nbc->backing_bdev, - FMODE_READ | FMODE_WRITE | FMODE_EXCL); - if (nbc->md_bdev) - blkdev_put(nbc->md_bdev, - FMODE_READ | FMODE_WRITE | FMODE_EXCL); + close_backing_dev(device, nbc->md_bdev, nbc->md_bdev != nbc->backing_bdev); + close_backing_dev(device, nbc->backing_bdev, true); kfree(nbc); } kfree(new_disk_conf); @@ -1895,6 +1986,7 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info) static int adm_detach(struct drbd_device *device, int force) { enum drbd_state_rv retcode; + void *buffer; int ret; if (force) { @@ -1905,13 +1997,16 @@ static int adm_detach(struct drbd_device *device, int force) } drbd_suspend_io(device); /* so no-one is stuck in drbd_al_begin_io */ - drbd_md_get_buffer(device, __func__); /* make sure there is no in-flight meta-data IO */ - retcode = drbd_request_state(device, NS(disk, D_FAILED)); - drbd_md_put_buffer(device); + buffer = drbd_md_get_buffer(device, __func__); /* make sure there is no in-flight meta-data IO */ + if (buffer) { + retcode = drbd_request_state(device, NS(disk, D_FAILED)); + drbd_md_put_buffer(device); + } else /* already <= D_FAILED */ + retcode = SS_NOTHING_TO_DO; /* D_FAILED will transition to DISKLESS. */ + drbd_resume_io(device); ret = wait_event_interruptible(device->misc_wait, device->state.disk != D_FAILED); - drbd_resume_io(device); if ((int)retcode == (int)SS_IS_DISKLESS) retcode = SS_NOTHING_TO_DO; if (ret) @@ -2245,8 +2340,31 @@ int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info) return 0; } +static void connection_to_info(struct connection_info *info, + struct drbd_connection *connection) +{ + info->conn_connection_state = connection->cstate; + info->conn_role = conn_highest_peer(connection); +} + +static void peer_device_to_info(struct peer_device_info *info, + struct drbd_peer_device *peer_device) +{ + struct drbd_device *device = peer_device->device; + + info->peer_repl_state = + max_t(enum drbd_conns, C_WF_REPORT_PARAMS, device->state.conn); + info->peer_disk_state = device->state.pdsk; + info->peer_resync_susp_user = device->state.user_isp; + info->peer_resync_susp_peer = device->state.peer_isp; + info->peer_resync_susp_dependency = device->state.aftr_isp; +} + int drbd_adm_connect(struct sk_buff *skb, struct genl_info *info) { + struct connection_info connection_info; + enum drbd_notification_type flags; + unsigned int peer_devices = 0; struct drbd_config_context adm_ctx; struct drbd_peer_device *peer_device; struct net_conf *old_net_conf, *new_net_conf = NULL; @@ -2347,6 +2465,22 @@ int drbd_adm_connect(struct sk_buff *skb, struct genl_info *info) connection->peer_addr_len = nla_len(adm_ctx.peer_addr); memcpy(&connection->peer_addr, nla_data(adm_ctx.peer_addr), connection->peer_addr_len); + idr_for_each_entry(&connection->peer_devices, peer_device, i) { + peer_devices++; + } + + connection_to_info(&connection_info, connection); + flags = (peer_devices--) ? NOTIFY_CONTINUES : 0; + mutex_lock(¬ification_mutex); + notify_connection_state(NULL, 0, connection, &connection_info, NOTIFY_CREATE | flags); + idr_for_each_entry(&connection->peer_devices, peer_device, i) { + struct peer_device_info peer_device_info; + + peer_device_to_info(&peer_device_info, peer_device); + flags = (peer_devices--) ? NOTIFY_CONTINUES : 0; + notify_peer_device_state(NULL, 0, peer_device, &peer_device_info, NOTIFY_CREATE | flags); + } + mutex_unlock(¬ification_mutex); mutex_unlock(&adm_ctx.resource->conf_update); rcu_read_lock(); @@ -2428,6 +2562,8 @@ static enum drbd_state_rv conn_try_disconnect(struct drbd_connection *connection drbd_err(connection, "unexpected rv2=%d in conn_try_disconnect()\n", rv2); + /* Unlike in DRBD 9, the state engine has generated + * NOTIFY_DESTROY events before clearing connection->net_conf. */ } return rv; } @@ -2585,6 +2721,7 @@ int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info) mutex_unlock(&device->resource->conf_update); synchronize_rcu(); kfree(old_disk_conf); + new_disk_conf = NULL; } ddsf = (rs.resize_force ? DDSF_FORCED : 0) | (rs.no_resync ? DDSF_NO_RESYNC : 0); @@ -2618,6 +2755,7 @@ int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info) fail_ldev: put_ldev(device); + kfree(new_disk_conf); goto fail; } @@ -2855,7 +2993,30 @@ int drbd_adm_resume_io(struct sk_buff *skb, struct genl_info *info) mutex_lock(&adm_ctx.resource->adm_mutex); device = adm_ctx.device; if (test_bit(NEW_CUR_UUID, &device->flags)) { - drbd_uuid_new_current(device); + if (get_ldev_if_state(device, D_ATTACHING)) { + drbd_uuid_new_current(device); + put_ldev(device); + } else { + /* This is effectively a multi-stage "forced down". + * The NEW_CUR_UUID bit is supposedly only set, if we + * lost the replication connection, and are configured + * to freeze IO and wait for some fence-peer handler. + * So we still don't have a replication connection. + * And now we don't have a local disk either. After + * resume, we will fail all pending and new IO, because + * we don't have any data anymore. Which means we will + * eventually be able to terminate all users of this + * device, and then take it down. By bumping the + * "effective" data uuid, we make sure that you really + * need to tear down before you reconfigure, we will + * the refuse to re-connect or re-attach (because no + * matching real data uuid exists). + */ + u64 val; + get_random_bytes(&val, sizeof(u64)); + drbd_set_ed_uuid(device, val); + drbd_warn(device, "Resumed without access to data; please tear down before attempting to re-configure.\n"); + } clear_bit(NEW_CUR_UUID, &device->flags); } drbd_suspend_io(device); @@ -2910,6 +3071,486 @@ nla_put_failure: } /* + * The generic netlink dump callbacks are called outside the genl_lock(), so + * they cannot use the simple attribute parsing code which uses global + * attribute tables. + */ +static struct nlattr *find_cfg_context_attr(const struct nlmsghdr *nlh, int attr) +{ + const unsigned hdrlen = GENL_HDRLEN + GENL_MAGIC_FAMILY_HDRSZ; + const int maxtype = ARRAY_SIZE(drbd_cfg_context_nl_policy) - 1; + struct nlattr *nla; + + nla = nla_find(nlmsg_attrdata(nlh, hdrlen), nlmsg_attrlen(nlh, hdrlen), + DRBD_NLA_CFG_CONTEXT); + if (!nla) + return NULL; + return drbd_nla_find_nested(maxtype, nla, __nla_type(attr)); +} + +static void resource_to_info(struct resource_info *, struct drbd_resource *); + +int drbd_adm_dump_resources(struct sk_buff *skb, struct netlink_callback *cb) +{ + struct drbd_genlmsghdr *dh; + struct drbd_resource *resource; + struct resource_info resource_info; + struct resource_statistics resource_statistics; + int err; + + rcu_read_lock(); + if (cb->args[0]) { + for_each_resource_rcu(resource, &drbd_resources) + if (resource == (struct drbd_resource *)cb->args[0]) + goto found_resource; + err = 0; /* resource was probably deleted */ + goto out; + } + resource = list_entry(&drbd_resources, + struct drbd_resource, resources); + +found_resource: + list_for_each_entry_continue_rcu(resource, &drbd_resources, resources) { + goto put_result; + } + err = 0; + goto out; + +put_result: + dh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, + cb->nlh->nlmsg_seq, &drbd_genl_family, + NLM_F_MULTI, DRBD_ADM_GET_RESOURCES); + err = -ENOMEM; + if (!dh) + goto out; + dh->minor = -1U; + dh->ret_code = NO_ERROR; + err = nla_put_drbd_cfg_context(skb, resource, NULL, NULL); + if (err) + goto out; + err = res_opts_to_skb(skb, &resource->res_opts, !capable(CAP_SYS_ADMIN)); + if (err) + goto out; + resource_to_info(&resource_info, resource); + err = resource_info_to_skb(skb, &resource_info, !capable(CAP_SYS_ADMIN)); + if (err) + goto out; + resource_statistics.res_stat_write_ordering = resource->write_ordering; + err = resource_statistics_to_skb(skb, &resource_statistics, !capable(CAP_SYS_ADMIN)); + if (err) + goto out; + cb->args[0] = (long)resource; + genlmsg_end(skb, dh); + err = 0; + +out: + rcu_read_unlock(); + if (err) + return err; + return skb->len; +} + +static void device_to_statistics(struct device_statistics *s, + struct drbd_device *device) +{ + memset(s, 0, sizeof(*s)); + s->dev_upper_blocked = !may_inc_ap_bio(device); + if (get_ldev(device)) { + struct drbd_md *md = &device->ldev->md; + u64 *history_uuids = (u64 *)s->history_uuids; + struct request_queue *q; + int n; + + spin_lock_irq(&md->uuid_lock); + s->dev_current_uuid = md->uuid[UI_CURRENT]; + BUILD_BUG_ON(sizeof(s->history_uuids) < UI_HISTORY_END - UI_HISTORY_START + 1); + for (n = 0; n < UI_HISTORY_END - UI_HISTORY_START + 1; n++) + history_uuids[n] = md->uuid[UI_HISTORY_START + n]; + for (; n < HISTORY_UUIDS; n++) + history_uuids[n] = 0; + s->history_uuids_len = HISTORY_UUIDS; + spin_unlock_irq(&md->uuid_lock); + + s->dev_disk_flags = md->flags; + q = bdev_get_queue(device->ldev->backing_bdev); + s->dev_lower_blocked = + bdi_congested(&q->backing_dev_info, + (1 << WB_async_congested) | + (1 << WB_sync_congested)); + put_ldev(device); + } + s->dev_size = drbd_get_capacity(device->this_bdev); + s->dev_read = device->read_cnt; + s->dev_write = device->writ_cnt; + s->dev_al_writes = device->al_writ_cnt; + s->dev_bm_writes = device->bm_writ_cnt; + s->dev_upper_pending = atomic_read(&device->ap_bio_cnt); + s->dev_lower_pending = atomic_read(&device->local_cnt); + s->dev_al_suspended = test_bit(AL_SUSPENDED, &device->flags); + s->dev_exposed_data_uuid = device->ed_uuid; +} + +static int put_resource_in_arg0(struct netlink_callback *cb, int holder_nr) +{ + if (cb->args[0]) { + struct drbd_resource *resource = + (struct drbd_resource *)cb->args[0]; + kref_put(&resource->kref, drbd_destroy_resource); + } + + return 0; +} + +int drbd_adm_dump_devices_done(struct netlink_callback *cb) { + return put_resource_in_arg0(cb, 7); +} + +static void device_to_info(struct device_info *, struct drbd_device *); + +int drbd_adm_dump_devices(struct sk_buff *skb, struct netlink_callback *cb) +{ + struct nlattr *resource_filter; + struct drbd_resource *resource; + struct drbd_device *uninitialized_var(device); + int minor, err, retcode; + struct drbd_genlmsghdr *dh; + struct device_info device_info; + struct device_statistics device_statistics; + struct idr *idr_to_search; + + resource = (struct drbd_resource *)cb->args[0]; + if (!cb->args[0] && !cb->args[1]) { + resource_filter = find_cfg_context_attr(cb->nlh, T_ctx_resource_name); + if (resource_filter) { + retcode = ERR_RES_NOT_KNOWN; + resource = drbd_find_resource(nla_data(resource_filter)); + if (!resource) + goto put_result; + cb->args[0] = (long)resource; + } + } + + rcu_read_lock(); + minor = cb->args[1]; + idr_to_search = resource ? &resource->devices : &drbd_devices; + device = idr_get_next(idr_to_search, &minor); + if (!device) { + err = 0; + goto out; + } + idr_for_each_entry_continue(idr_to_search, device, minor) { + retcode = NO_ERROR; + goto put_result; /* only one iteration */ + } + err = 0; + goto out; /* no more devices */ + +put_result: + dh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, + cb->nlh->nlmsg_seq, &drbd_genl_family, + NLM_F_MULTI, DRBD_ADM_GET_DEVICES); + err = -ENOMEM; + if (!dh) + goto out; + dh->ret_code = retcode; + dh->minor = -1U; + if (retcode == NO_ERROR) { + dh->minor = device->minor; + err = nla_put_drbd_cfg_context(skb, device->resource, NULL, device); + if (err) + goto out; + if (get_ldev(device)) { + struct disk_conf *disk_conf = + rcu_dereference(device->ldev->disk_conf); + + err = disk_conf_to_skb(skb, disk_conf, !capable(CAP_SYS_ADMIN)); + put_ldev(device); + if (err) + goto out; + } + device_to_info(&device_info, device); + err = device_info_to_skb(skb, &device_info, !capable(CAP_SYS_ADMIN)); + if (err) + goto out; + + device_to_statistics(&device_statistics, device); + err = device_statistics_to_skb(skb, &device_statistics, !capable(CAP_SYS_ADMIN)); + if (err) + goto out; + cb->args[1] = minor + 1; + } + genlmsg_end(skb, dh); + err = 0; + +out: + rcu_read_unlock(); + if (err) + return err; + return skb->len; +} + +int drbd_adm_dump_connections_done(struct netlink_callback *cb) +{ + return put_resource_in_arg0(cb, 6); +} + +enum { SINGLE_RESOURCE, ITERATE_RESOURCES }; + +int drbd_adm_dump_connections(struct sk_buff *skb, struct netlink_callback *cb) +{ + struct nlattr *resource_filter; + struct drbd_resource *resource = NULL, *next_resource; + struct drbd_connection *uninitialized_var(connection); + int err = 0, retcode; + struct drbd_genlmsghdr *dh; + struct connection_info connection_info; + struct connection_statistics connection_statistics; + + rcu_read_lock(); + resource = (struct drbd_resource *)cb->args[0]; + if (!cb->args[0]) { + resource_filter = find_cfg_context_attr(cb->nlh, T_ctx_resource_name); + if (resource_filter) { + retcode = ERR_RES_NOT_KNOWN; + resource = drbd_find_resource(nla_data(resource_filter)); + if (!resource) + goto put_result; + cb->args[0] = (long)resource; + cb->args[1] = SINGLE_RESOURCE; + } + } + if (!resource) { + if (list_empty(&drbd_resources)) + goto out; + resource = list_first_entry(&drbd_resources, struct drbd_resource, resources); + kref_get(&resource->kref); + cb->args[0] = (long)resource; + cb->args[1] = ITERATE_RESOURCES; + } + + next_resource: + rcu_read_unlock(); + mutex_lock(&resource->conf_update); + rcu_read_lock(); + if (cb->args[2]) { + for_each_connection_rcu(connection, resource) + if (connection == (struct drbd_connection *)cb->args[2]) + goto found_connection; + /* connection was probably deleted */ + goto no_more_connections; + } + connection = list_entry(&resource->connections, struct drbd_connection, connections); + +found_connection: + list_for_each_entry_continue_rcu(connection, &resource->connections, connections) { + if (!has_net_conf(connection)) + continue; + retcode = NO_ERROR; + goto put_result; /* only one iteration */ + } + +no_more_connections: + if (cb->args[1] == ITERATE_RESOURCES) { + for_each_resource_rcu(next_resource, &drbd_resources) { + if (next_resource == resource) + goto found_resource; + } + /* resource was probably deleted */ + } + goto out; + +found_resource: + list_for_each_entry_continue_rcu(next_resource, &drbd_resources, resources) { + mutex_unlock(&resource->conf_update); + kref_put(&resource->kref, drbd_destroy_resource); + resource = next_resource; + kref_get(&resource->kref); + cb->args[0] = (long)resource; + cb->args[2] = 0; + goto next_resource; + } + goto out; /* no more resources */ + +put_result: + dh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, + cb->nlh->nlmsg_seq, &drbd_genl_family, + NLM_F_MULTI, DRBD_ADM_GET_CONNECTIONS); + err = -ENOMEM; + if (!dh) + goto out; + dh->ret_code = retcode; + dh->minor = -1U; + if (retcode == NO_ERROR) { + struct net_conf *net_conf; + + err = nla_put_drbd_cfg_context(skb, resource, connection, NULL); + if (err) + goto out; + net_conf = rcu_dereference(connection->net_conf); + if (net_conf) { + err = net_conf_to_skb(skb, net_conf, !capable(CAP_SYS_ADMIN)); + if (err) + goto out; + } + connection_to_info(&connection_info, connection); + err = connection_info_to_skb(skb, &connection_info, !capable(CAP_SYS_ADMIN)); + if (err) + goto out; + connection_statistics.conn_congested = test_bit(NET_CONGESTED, &connection->flags); + err = connection_statistics_to_skb(skb, &connection_statistics, !capable(CAP_SYS_ADMIN)); + if (err) + goto out; + cb->args[2] = (long)connection; + } + genlmsg_end(skb, dh); + err = 0; + +out: + rcu_read_unlock(); + if (resource) + mutex_unlock(&resource->conf_update); + if (err) + return err; + return skb->len; +} + +enum mdf_peer_flag { + MDF_PEER_CONNECTED = 1 << 0, + MDF_PEER_OUTDATED = 1 << 1, + MDF_PEER_FENCING = 1 << 2, + MDF_PEER_FULL_SYNC = 1 << 3, +}; + +static void peer_device_to_statistics(struct peer_device_statistics *s, + struct drbd_peer_device *peer_device) +{ + struct drbd_device *device = peer_device->device; + + memset(s, 0, sizeof(*s)); + s->peer_dev_received = device->recv_cnt; + s->peer_dev_sent = device->send_cnt; + s->peer_dev_pending = atomic_read(&device->ap_pending_cnt) + + atomic_read(&device->rs_pending_cnt); + s->peer_dev_unacked = atomic_read(&device->unacked_cnt); + s->peer_dev_out_of_sync = drbd_bm_total_weight(device) << (BM_BLOCK_SHIFT - 9); + s->peer_dev_resync_failed = device->rs_failed << (BM_BLOCK_SHIFT - 9); + if (get_ldev(device)) { + struct drbd_md *md = &device->ldev->md; + + spin_lock_irq(&md->uuid_lock); + s->peer_dev_bitmap_uuid = md->uuid[UI_BITMAP]; + spin_unlock_irq(&md->uuid_lock); + s->peer_dev_flags = + (drbd_md_test_flag(device->ldev, MDF_CONNECTED_IND) ? + MDF_PEER_CONNECTED : 0) + + (drbd_md_test_flag(device->ldev, MDF_CONSISTENT) && + !drbd_md_test_flag(device->ldev, MDF_WAS_UP_TO_DATE) ? + MDF_PEER_OUTDATED : 0) + + /* FIXME: MDF_PEER_FENCING? */ + (drbd_md_test_flag(device->ldev, MDF_FULL_SYNC) ? + MDF_PEER_FULL_SYNC : 0); + put_ldev(device); + } +} + +int drbd_adm_dump_peer_devices_done(struct netlink_callback *cb) +{ + return put_resource_in_arg0(cb, 9); +} + +int drbd_adm_dump_peer_devices(struct sk_buff *skb, struct netlink_callback *cb) +{ + struct nlattr *resource_filter; + struct drbd_resource *resource; + struct drbd_device *uninitialized_var(device); + struct drbd_peer_device *peer_device = NULL; + int minor, err, retcode; + struct drbd_genlmsghdr *dh; + struct idr *idr_to_search; + + resource = (struct drbd_resource *)cb->args[0]; + if (!cb->args[0] && !cb->args[1]) { + resource_filter = find_cfg_context_attr(cb->nlh, T_ctx_resource_name); + if (resource_filter) { + retcode = ERR_RES_NOT_KNOWN; + resource = drbd_find_resource(nla_data(resource_filter)); + if (!resource) + goto put_result; + } + cb->args[0] = (long)resource; + } + + rcu_read_lock(); + minor = cb->args[1]; + idr_to_search = resource ? &resource->devices : &drbd_devices; + device = idr_find(idr_to_search, minor); + if (!device) { +next_device: + minor++; + cb->args[2] = 0; + device = idr_get_next(idr_to_search, &minor); + if (!device) { + err = 0; + goto out; + } + } + if (cb->args[2]) { + for_each_peer_device(peer_device, device) + if (peer_device == (struct drbd_peer_device *)cb->args[2]) + goto found_peer_device; + /* peer device was probably deleted */ + goto next_device; + } + /* Make peer_device point to the list head (not the first entry). */ + peer_device = list_entry(&device->peer_devices, struct drbd_peer_device, peer_devices); + +found_peer_device: + list_for_each_entry_continue_rcu(peer_device, &device->peer_devices, peer_devices) { + if (!has_net_conf(peer_device->connection)) + continue; + retcode = NO_ERROR; + goto put_result; /* only one iteration */ + } + goto next_device; + +put_result: + dh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, + cb->nlh->nlmsg_seq, &drbd_genl_family, + NLM_F_MULTI, DRBD_ADM_GET_PEER_DEVICES); + err = -ENOMEM; + if (!dh) + goto out; + dh->ret_code = retcode; + dh->minor = -1U; + if (retcode == NO_ERROR) { + struct peer_device_info peer_device_info; + struct peer_device_statistics peer_device_statistics; + + dh->minor = minor; + err = nla_put_drbd_cfg_context(skb, device->resource, peer_device->connection, device); + if (err) + goto out; + peer_device_to_info(&peer_device_info, peer_device); + err = peer_device_info_to_skb(skb, &peer_device_info, !capable(CAP_SYS_ADMIN)); + if (err) + goto out; + peer_device_to_statistics(&peer_device_statistics, peer_device); + err = peer_device_statistics_to_skb(skb, &peer_device_statistics, !capable(CAP_SYS_ADMIN)); + if (err) + goto out; + cb->args[1] = minor; + cb->args[2] = (long)peer_device; + } + genlmsg_end(skb, dh); + err = 0; + +out: + rcu_read_unlock(); + if (err) + return err; + return skb->len; +} +/* * Return the connection of @resource if @resource has exactly one connection. */ static struct drbd_connection *the_only_connection(struct drbd_resource *resource) @@ -3414,8 +4055,18 @@ drbd_check_resource_name(struct drbd_config_context *adm_ctx) return NO_ERROR; } +static void resource_to_info(struct resource_info *info, + struct drbd_resource *resource) +{ + info->res_role = conn_highest_role(first_connection(resource)); + info->res_susp = resource->susp; + info->res_susp_nod = resource->susp_nod; + info->res_susp_fen = resource->susp_fen; +} + int drbd_adm_new_resource(struct sk_buff *skb, struct genl_info *info) { + struct drbd_connection *connection; struct drbd_config_context adm_ctx; enum drbd_ret_code retcode; struct res_opts res_opts; @@ -3449,13 +4100,33 @@ int drbd_adm_new_resource(struct sk_buff *skb, struct genl_info *info) } /* not yet safe for genl_family.parallel_ops */ - if (!conn_create(adm_ctx.resource_name, &res_opts)) + mutex_lock(&resources_mutex); + connection = conn_create(adm_ctx.resource_name, &res_opts); + mutex_unlock(&resources_mutex); + + if (connection) { + struct resource_info resource_info; + + mutex_lock(¬ification_mutex); + resource_to_info(&resource_info, connection->resource); + notify_resource_state(NULL, 0, connection->resource, + &resource_info, NOTIFY_CREATE); + mutex_unlock(¬ification_mutex); + } else retcode = ERR_NOMEM; + out: drbd_adm_finish(&adm_ctx, info, retcode); return 0; } +static void device_to_info(struct device_info *info, + struct drbd_device *device) +{ + info->dev_disk_state = device->state.disk; +} + + int drbd_adm_new_minor(struct sk_buff *skb, struct genl_info *info) { struct drbd_config_context adm_ctx; @@ -3490,6 +4161,36 @@ int drbd_adm_new_minor(struct sk_buff *skb, struct genl_info *info) mutex_lock(&adm_ctx.resource->adm_mutex); retcode = drbd_create_device(&adm_ctx, dh->minor); + if (retcode == NO_ERROR) { + struct drbd_device *device; + struct drbd_peer_device *peer_device; + struct device_info info; + unsigned int peer_devices = 0; + enum drbd_notification_type flags; + + device = minor_to_device(dh->minor); + for_each_peer_device(peer_device, device) { + if (!has_net_conf(peer_device->connection)) + continue; + peer_devices++; + } + + device_to_info(&info, device); + mutex_lock(¬ification_mutex); + flags = (peer_devices--) ? NOTIFY_CONTINUES : 0; + notify_device_state(NULL, 0, device, &info, NOTIFY_CREATE | flags); + for_each_peer_device(peer_device, device) { + struct peer_device_info peer_device_info; + + if (!has_net_conf(peer_device->connection)) + continue; + peer_device_to_info(&peer_device_info, peer_device); + flags = (peer_devices--) ? NOTIFY_CONTINUES : 0; + notify_peer_device_state(NULL, 0, peer_device, &peer_device_info, + NOTIFY_CREATE | flags); + } + mutex_unlock(¬ification_mutex); + } mutex_unlock(&adm_ctx.resource->adm_mutex); out: drbd_adm_finish(&adm_ctx, info, retcode); @@ -3498,13 +4199,35 @@ out: static enum drbd_ret_code adm_del_minor(struct drbd_device *device) { + struct drbd_peer_device *peer_device; + if (device->state.disk == D_DISKLESS && /* no need to be device->state.conn == C_STANDALONE && * we may want to delete a minor from a live replication group. */ device->state.role == R_SECONDARY) { + struct drbd_connection *connection = + first_connection(device->resource); + _drbd_request_state(device, NS(conn, C_WF_REPORT_PARAMS), CS_VERBOSE + CS_WAIT_COMPLETE); + + /* If the state engine hasn't stopped the sender thread yet, we + * need to flush the sender work queue before generating the + * DESTROY events here. */ + if (get_t_state(&connection->worker) == RUNNING) + drbd_flush_workqueue(&connection->sender_work); + + mutex_lock(¬ification_mutex); + for_each_peer_device(peer_device, device) { + if (!has_net_conf(peer_device->connection)) + continue; + notify_peer_device_state(NULL, 0, peer_device, NULL, + NOTIFY_DESTROY | NOTIFY_CONTINUES); + } + notify_device_state(NULL, 0, device, NULL, NOTIFY_DESTROY); + mutex_unlock(¬ification_mutex); + drbd_delete_device(device); return NO_ERROR; } else @@ -3541,7 +4264,16 @@ static int adm_del_resource(struct drbd_resource *resource) if (!idr_is_empty(&resource->devices)) return ERR_RES_IN_USE; + /* The state engine has stopped the sender thread, so we don't + * need to flush the sender work queue before generating the + * DESTROY event here. */ + mutex_lock(¬ification_mutex); + notify_resource_state(NULL, 0, resource, NULL, NOTIFY_DESTROY); + mutex_unlock(¬ification_mutex); + + mutex_lock(&resources_mutex); list_del_rcu(&resource->resources); + mutex_unlock(&resources_mutex); /* Make sure all threads have actually stopped: state handling only * does drbd_thread_stop_nowait(). */ list_for_each_entry(connection, &resource->connections, connections) @@ -3637,7 +4369,6 @@ finish: void drbd_bcast_event(struct drbd_device *device, const struct sib_info *sib) { - static atomic_t drbd_genl_seq = ATOMIC_INIT(2); /* two. */ struct sk_buff *msg; struct drbd_genlmsghdr *d_out; unsigned seq; @@ -3658,7 +4389,7 @@ void drbd_bcast_event(struct drbd_device *device, const struct sib_info *sib) if (nla_put_status_info(msg, device, sib)) goto nla_put_failure; genlmsg_end(msg, d_out); - err = drbd_genl_multicast_events(msg, 0); + err = drbd_genl_multicast_events(msg, GFP_NOWAIT); /* msg has been consumed or freed in netlink_broadcast() */ if (err && err != -ESRCH) goto failed; @@ -3672,3 +4403,405 @@ failed: "Event seq:%u sib_reason:%u\n", err, seq, sib->sib_reason); } + +static int nla_put_notification_header(struct sk_buff *msg, + enum drbd_notification_type type) +{ + struct drbd_notification_header nh = { + .nh_type = type, + }; + + return drbd_notification_header_to_skb(msg, &nh, true); +} + +void notify_resource_state(struct sk_buff *skb, + unsigned int seq, + struct drbd_resource *resource, + struct resource_info *resource_info, + enum drbd_notification_type type) +{ + struct resource_statistics resource_statistics; + struct drbd_genlmsghdr *dh; + bool multicast = false; + int err; + + if (!skb) { + seq = atomic_inc_return(¬ify_genl_seq); + skb = genlmsg_new(NLMSG_GOODSIZE, GFP_NOIO); + err = -ENOMEM; + if (!skb) + goto failed; + multicast = true; + } + + err = -EMSGSIZE; + dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_RESOURCE_STATE); + if (!dh) + goto nla_put_failure; + dh->minor = -1U; + dh->ret_code = NO_ERROR; + if (nla_put_drbd_cfg_context(skb, resource, NULL, NULL) || + nla_put_notification_header(skb, type) || + ((type & ~NOTIFY_FLAGS) != NOTIFY_DESTROY && + resource_info_to_skb(skb, resource_info, true))) + goto nla_put_failure; + resource_statistics.res_stat_write_ordering = resource->write_ordering; + err = resource_statistics_to_skb(skb, &resource_statistics, !capable(CAP_SYS_ADMIN)); + if (err) + goto nla_put_failure; + genlmsg_end(skb, dh); + if (multicast) { + err = drbd_genl_multicast_events(skb, GFP_NOWAIT); + /* skb has been consumed or freed in netlink_broadcast() */ + if (err && err != -ESRCH) + goto failed; + } + return; + +nla_put_failure: + nlmsg_free(skb); +failed: + drbd_err(resource, "Error %d while broadcasting event. Event seq:%u\n", + err, seq); +} + +void notify_device_state(struct sk_buff *skb, + unsigned int seq, + struct drbd_device *device, + struct device_info *device_info, + enum drbd_notification_type type) +{ + struct device_statistics device_statistics; + struct drbd_genlmsghdr *dh; + bool multicast = false; + int err; + + if (!skb) { + seq = atomic_inc_return(¬ify_genl_seq); + skb = genlmsg_new(NLMSG_GOODSIZE, GFP_NOIO); + err = -ENOMEM; + if (!skb) + goto failed; + multicast = true; + } + + err = -EMSGSIZE; + dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_DEVICE_STATE); + if (!dh) + goto nla_put_failure; + dh->minor = device->minor; + dh->ret_code = NO_ERROR; + if (nla_put_drbd_cfg_context(skb, device->resource, NULL, device) || + nla_put_notification_header(skb, type) || + ((type & ~NOTIFY_FLAGS) != NOTIFY_DESTROY && + device_info_to_skb(skb, device_info, true))) + goto nla_put_failure; + device_to_statistics(&device_statistics, device); + device_statistics_to_skb(skb, &device_statistics, !capable(CAP_SYS_ADMIN)); + genlmsg_end(skb, dh); + if (multicast) { + err = drbd_genl_multicast_events(skb, GFP_NOWAIT); + /* skb has been consumed or freed in netlink_broadcast() */ + if (err && err != -ESRCH) + goto failed; + } + return; + +nla_put_failure: + nlmsg_free(skb); +failed: + drbd_err(device, "Error %d while broadcasting event. Event seq:%u\n", + err, seq); +} + +void notify_connection_state(struct sk_buff *skb, + unsigned int seq, + struct drbd_connection *connection, + struct connection_info *connection_info, + enum drbd_notification_type type) +{ + struct connection_statistics connection_statistics; + struct drbd_genlmsghdr *dh; + bool multicast = false; + int err; + + if (!skb) { + seq = atomic_inc_return(¬ify_genl_seq); + skb = genlmsg_new(NLMSG_GOODSIZE, GFP_NOIO); + err = -ENOMEM; + if (!skb) + goto failed; + multicast = true; + } + + err = -EMSGSIZE; + dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_CONNECTION_STATE); + if (!dh) + goto nla_put_failure; + dh->minor = -1U; + dh->ret_code = NO_ERROR; + if (nla_put_drbd_cfg_context(skb, connection->resource, connection, NULL) || + nla_put_notification_header(skb, type) || + ((type & ~NOTIFY_FLAGS) != NOTIFY_DESTROY && + connection_info_to_skb(skb, connection_info, true))) + goto nla_put_failure; + connection_statistics.conn_congested = test_bit(NET_CONGESTED, &connection->flags); + connection_statistics_to_skb(skb, &connection_statistics, !capable(CAP_SYS_ADMIN)); + genlmsg_end(skb, dh); + if (multicast) { + err = drbd_genl_multicast_events(skb, GFP_NOWAIT); + /* skb has been consumed or freed in netlink_broadcast() */ + if (err && err != -ESRCH) + goto failed; + } + return; + +nla_put_failure: + nlmsg_free(skb); +failed: + drbd_err(connection, "Error %d while broadcasting event. Event seq:%u\n", + err, seq); +} + +void notify_peer_device_state(struct sk_buff *skb, + unsigned int seq, + struct drbd_peer_device *peer_device, + struct peer_device_info *peer_device_info, + enum drbd_notification_type type) +{ + struct peer_device_statistics peer_device_statistics; + struct drbd_resource *resource = peer_device->device->resource; + struct drbd_genlmsghdr *dh; + bool multicast = false; + int err; + + if (!skb) { + seq = atomic_inc_return(¬ify_genl_seq); + skb = genlmsg_new(NLMSG_GOODSIZE, GFP_NOIO); + err = -ENOMEM; + if (!skb) + goto failed; + multicast = true; + } + + err = -EMSGSIZE; + dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_PEER_DEVICE_STATE); + if (!dh) + goto nla_put_failure; + dh->minor = -1U; + dh->ret_code = NO_ERROR; + if (nla_put_drbd_cfg_context(skb, resource, peer_device->connection, peer_device->device) || + nla_put_notification_header(skb, type) || + ((type & ~NOTIFY_FLAGS) != NOTIFY_DESTROY && + peer_device_info_to_skb(skb, peer_device_info, true))) + goto nla_put_failure; + peer_device_to_statistics(&peer_device_statistics, peer_device); + peer_device_statistics_to_skb(skb, &peer_device_statistics, !capable(CAP_SYS_ADMIN)); + genlmsg_end(skb, dh); + if (multicast) { + err = drbd_genl_multicast_events(skb, GFP_NOWAIT); + /* skb has been consumed or freed in netlink_broadcast() */ + if (err && err != -ESRCH) + goto failed; + } + return; + +nla_put_failure: + nlmsg_free(skb); +failed: + drbd_err(peer_device, "Error %d while broadcasting event. Event seq:%u\n", + err, seq); +} + +void notify_helper(enum drbd_notification_type type, + struct drbd_device *device, struct drbd_connection *connection, + const char *name, int status) +{ + struct drbd_resource *resource = device ? device->resource : connection->resource; + struct drbd_helper_info helper_info; + unsigned int seq = atomic_inc_return(¬ify_genl_seq); + struct sk_buff *skb = NULL; + struct drbd_genlmsghdr *dh; + int err; + + strlcpy(helper_info.helper_name, name, sizeof(helper_info.helper_name)); + helper_info.helper_name_len = min(strlen(name), sizeof(helper_info.helper_name)); + helper_info.helper_status = status; + + skb = genlmsg_new(NLMSG_GOODSIZE, GFP_NOIO); + err = -ENOMEM; + if (!skb) + goto fail; + + err = -EMSGSIZE; + dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_HELPER); + if (!dh) + goto fail; + dh->minor = device ? device->minor : -1; + dh->ret_code = NO_ERROR; + mutex_lock(¬ification_mutex); + if (nla_put_drbd_cfg_context(skb, resource, connection, device) || + nla_put_notification_header(skb, type) || + drbd_helper_info_to_skb(skb, &helper_info, true)) + goto unlock_fail; + genlmsg_end(skb, dh); + err = drbd_genl_multicast_events(skb, GFP_NOWAIT); + skb = NULL; + /* skb has been consumed or freed in netlink_broadcast() */ + if (err && err != -ESRCH) + goto unlock_fail; + mutex_unlock(¬ification_mutex); + return; + +unlock_fail: + mutex_unlock(¬ification_mutex); +fail: + nlmsg_free(skb); + drbd_err(resource, "Error %d while broadcasting event. Event seq:%u\n", + err, seq); +} + +static void notify_initial_state_done(struct sk_buff *skb, unsigned int seq) +{ + struct drbd_genlmsghdr *dh; + int err; + + err = -EMSGSIZE; + dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_INITIAL_STATE_DONE); + if (!dh) + goto nla_put_failure; + dh->minor = -1U; + dh->ret_code = NO_ERROR; + if (nla_put_notification_header(skb, NOTIFY_EXISTS)) + goto nla_put_failure; + genlmsg_end(skb, dh); + return; + +nla_put_failure: + nlmsg_free(skb); + pr_err("Error %d sending event. Event seq:%u\n", err, seq); +} + +static void free_state_changes(struct list_head *list) +{ + while (!list_empty(list)) { + struct drbd_state_change *state_change = + list_first_entry(list, struct drbd_state_change, list); + list_del(&state_change->list); + forget_state_change(state_change); + } +} + +static unsigned int notifications_for_state_change(struct drbd_state_change *state_change) +{ + return 1 + + state_change->n_connections + + state_change->n_devices + + state_change->n_devices * state_change->n_connections; +} + +static int get_initial_state(struct sk_buff *skb, struct netlink_callback *cb) +{ + struct drbd_state_change *state_change = (struct drbd_state_change *)cb->args[0]; + unsigned int seq = cb->args[2]; + unsigned int n; + enum drbd_notification_type flags = 0; + + /* There is no need for taking notification_mutex here: it doesn't + matter if the initial state events mix with later state chage + events; we can always tell the events apart by the NOTIFY_EXISTS + flag. */ + + cb->args[5]--; + if (cb->args[5] == 1) { + notify_initial_state_done(skb, seq); + goto out; + } + n = cb->args[4]++; + if (cb->args[4] < cb->args[3]) + flags |= NOTIFY_CONTINUES; + if (n < 1) { + notify_resource_state_change(skb, seq, state_change->resource, + NOTIFY_EXISTS | flags); + goto next; + } + n--; + if (n < state_change->n_connections) { + notify_connection_state_change(skb, seq, &state_change->connections[n], + NOTIFY_EXISTS | flags); + goto next; + } + n -= state_change->n_connections; + if (n < state_change->n_devices) { + notify_device_state_change(skb, seq, &state_change->devices[n], + NOTIFY_EXISTS | flags); + goto next; + } + n -= state_change->n_devices; + if (n < state_change->n_devices * state_change->n_connections) { + notify_peer_device_state_change(skb, seq, &state_change->peer_devices[n], + NOTIFY_EXISTS | flags); + goto next; + } + +next: + if (cb->args[4] == cb->args[3]) { + struct drbd_state_change *next_state_change = + list_entry(state_change->list.next, + struct drbd_state_change, list); + cb->args[0] = (long)next_state_change; + cb->args[3] = notifications_for_state_change(next_state_change); + cb->args[4] = 0; + } +out: + return skb->len; +} + +int drbd_adm_get_initial_state(struct sk_buff *skb, struct netlink_callback *cb) +{ + struct drbd_resource *resource; + LIST_HEAD(head); + + if (cb->args[5] >= 1) { + if (cb->args[5] > 1) + return get_initial_state(skb, cb); + if (cb->args[0]) { + struct drbd_state_change *state_change = + (struct drbd_state_change *)cb->args[0]; + + /* connect list to head */ + list_add(&head, &state_change->list); + free_state_changes(&head); + } + return 0; + } + + cb->args[5] = 2; /* number of iterations */ + mutex_lock(&resources_mutex); + for_each_resource(resource, &drbd_resources) { + struct drbd_state_change *state_change; + + state_change = remember_old_state(resource, GFP_KERNEL); + if (!state_change) { + if (!list_empty(&head)) + free_state_changes(&head); + mutex_unlock(&resources_mutex); + return -ENOMEM; + } + copy_old_to_new_state_change(state_change); + list_add_tail(&state_change->list, &head); + cb->args[5] += notifications_for_state_change(state_change); + } + mutex_unlock(&resources_mutex); + + if (!list_empty(&head)) { + struct drbd_state_change *state_change = + list_entry(head.next, struct drbd_state_change, list); + cb->args[0] = (long)state_change; + cb->args[3] = notifications_for_state_change(state_change); + list_del(&head); /* detach list from head */ + } + + cb->args[2] = cb->nlh->nlmsg_seq; + return get_initial_state(skb, cb); +} |