From 8d438288145f19f253a82ca71290b44fce79e23f Mon Sep 17 00:00:00 2001 From: Akinobu Mita Date: Mon, 30 Jul 2012 14:43:02 -0700 Subject: fault-injection: notifier error injection This patchset provides kernel modules that can be used to test the error handling of notifier call chain failures by injecting artifical errors to the following notifier chain callbacks. * CPU notifier * PM notifier * memory hotplug notifier * powerpc pSeries reconfig notifier Example: Inject CPU offline error (-1 == -EPERM) # cd /sys/kernel/debug/notifier-error-inject/cpu # echo -1 > actions/CPU_DOWN_PREPARE/error # echo 0 > /sys/devices/system/cpu/cpu1/online bash: echo: write error: Operation not permitted The patchset also adds cpu and memory hotplug tests to tools/testing/selftests These tests first do simple online and offline test and then do fault injection tests if notifier error injection module is available. This patch: The notifier error injection provides the ability to inject artifical errors to specified notifier chain callbacks. It is useful to test the error handling of notifier call chain failures. This adds common basic functions to define which type of events can be fail and to initialize the debugfs interface to control what error code should be returned and which event should be failed. Signed-off-by: Akinobu Mita Cc: Pavel Machek Cc: "Rafael J. Wysocki" Cc: Greg KH Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: Michael Ellerman Cc: Dave Jones Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/Kconfig.debug | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'lib/Kconfig.debug') diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 4a186508bf8b..bdee91d50a5b 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1084,6 +1084,17 @@ config LKDTM Documentation on how to use the module can be found in Documentation/fault-injection/provoke-crashes.txt +config NOTIFIER_ERROR_INJECTION + tristate "Notifier error injection" + depends on DEBUG_KERNEL + select DEBUG_FS + help + This option provides the ability to inject artifical errors to + specified notifier chain callbacks. It is useful to test the error + handling of notifier call chain failures. + + Say N if unsure. + config CPU_NOTIFIER_ERROR_INJECT tristate "CPU notifier error injection module" depends on HOTPLUG_CPU && DEBUG_KERNEL -- cgit v1.3-14-g43fede From f5a9f52e2c5654c3d212dbf7e7a169c60876691a Mon Sep 17 00:00:00 2001 From: Akinobu Mita Date: Mon, 30 Jul 2012 14:43:03 -0700 Subject: cpu: rewrite cpu-notifier-error-inject module Rewrite existing cpu-notifier-error-inject module to use debugfs based new framework. This change removes cpu_up_prepare_error and cpu_down_prepare_error module parameters which were used to specify error code to be injected. We could keep these module parameters for backward compatibility by module_param_cb but it seems overkill for this module. This provides the ability to inject artifical errors to CPU notifier chain callbacks. It is controlled through debugfs interface under /sys/kernel/debug/notifier-error-inject/cpu If the notifier call chain should be failed with some events notified, write the error code to "actions//error". Example1: inject CPU offline error (-1 == -EPERM) # cd /sys/kernel/debug/notifier-error-inject/cpu # echo -1 > actions/CPU_DOWN_PREPARE/error # echo 0 > /sys/devices/system/cpu/cpu1/online bash: echo: write error: Operation not permitted Example2: inject CPU online error (-2 == -ENOENT) # cd /sys/kernel/debug/notifier-error-inject/cpu # echo -2 > actions/CPU_UP_PREPARE/error # echo 1 > /sys/devices/system/cpu/cpu1/online bash: echo: write error: No such file or directory Signed-off-by: Akinobu Mita Cc: Pavel Machek Cc: "Rafael J. Wysocki" Cc: Greg KH Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: Michael Ellerman Cc: Dave Jones Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/Kconfig.debug | 16 +++++++++-- lib/cpu-notifier-error-inject.c | 63 ++++++++++++++++------------------------- 2 files changed, 39 insertions(+), 40 deletions(-) (limited to 'lib/Kconfig.debug') diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index bdee91d50a5b..20341480bb54 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1097,10 +1097,22 @@ config NOTIFIER_ERROR_INJECTION config CPU_NOTIFIER_ERROR_INJECT tristate "CPU notifier error injection module" - depends on HOTPLUG_CPU && DEBUG_KERNEL + depends on HOTPLUG_CPU && NOTIFIER_ERROR_INJECTION help This option provides a kernel module that can be used to test - the error handling of the cpu notifiers + the error handling of the cpu notifiers by injecting artifical + errors to CPU notifier chain callbacks. It is controlled through + debugfs interface under /sys/kernel/debug/notifier-error-inject/cpu + + If the notifier call chain should be failed with some events + notified, write the error code to "actions//error". + + Example: Inject CPU offline error (-1 == -EPERM) + + # cd /sys/kernel/debug/notifier-error-inject/cpu + # echo -1 > actions/CPU_DOWN_PREPARE/error + # echo 0 > /sys/devices/system/cpu/cpu1/online + bash: echo: write error: Operation not permitted To compile this code as a module, choose M here: the module will be called cpu-notifier-error-inject. diff --git a/lib/cpu-notifier-error-inject.c b/lib/cpu-notifier-error-inject.c index 4dc20321b0d5..707ca24f7b18 100644 --- a/lib/cpu-notifier-error-inject.c +++ b/lib/cpu-notifier-error-inject.c @@ -1,58 +1,45 @@ #include -#include #include -#include +#include -static int priority; -static int cpu_up_prepare_error; -static int cpu_down_prepare_error; +#include "notifier-error-inject.h" +static int priority; module_param(priority, int, 0); MODULE_PARM_DESC(priority, "specify cpu notifier priority"); -module_param(cpu_up_prepare_error, int, 0644); -MODULE_PARM_DESC(cpu_up_prepare_error, - "specify error code to inject CPU_UP_PREPARE action"); - -module_param(cpu_down_prepare_error, int, 0644); -MODULE_PARM_DESC(cpu_down_prepare_error, - "specify error code to inject CPU_DOWN_PREPARE action"); - -static int err_inject_cpu_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu) -{ - int err = 0; - - switch (action) { - case CPU_UP_PREPARE: - case CPU_UP_PREPARE_FROZEN: - err = cpu_up_prepare_error; - break; - case CPU_DOWN_PREPARE: - case CPU_DOWN_PREPARE_FROZEN: - err = cpu_down_prepare_error; - break; +static struct notifier_err_inject cpu_notifier_err_inject = { + .actions = { + { NOTIFIER_ERR_INJECT_ACTION(CPU_UP_PREPARE) }, + { NOTIFIER_ERR_INJECT_ACTION(CPU_UP_PREPARE_FROZEN) }, + { NOTIFIER_ERR_INJECT_ACTION(CPU_DOWN_PREPARE) }, + { NOTIFIER_ERR_INJECT_ACTION(CPU_DOWN_PREPARE_FROZEN) }, + {} } - if (err) - printk(KERN_INFO "Injecting error (%d) at cpu notifier\n", err); - - return notifier_from_errno(err); -} - -static struct notifier_block err_inject_cpu_notifier = { - .notifier_call = err_inject_cpu_callback, }; +static struct dentry *dir; + static int err_inject_init(void) { - err_inject_cpu_notifier.priority = priority; + int err; + + dir = notifier_err_inject_init("cpu", notifier_err_inject_dir, + &cpu_notifier_err_inject, priority); + if (IS_ERR(dir)) + return PTR_ERR(dir); + + err = register_hotcpu_notifier(&cpu_notifier_err_inject.nb); + if (err) + debugfs_remove_recursive(dir); - return register_hotcpu_notifier(&err_inject_cpu_notifier); + return err; } static void err_inject_exit(void) { - unregister_hotcpu_notifier(&err_inject_cpu_notifier); + unregister_hotcpu_notifier(&cpu_notifier_err_inject.nb); + debugfs_remove_recursive(dir); } module_init(err_inject_init); -- cgit v1.3-14-g43fede From 048b9c3549790af21eabd06a5ebdad305e75b1c5 Mon Sep 17 00:00:00 2001 From: Akinobu Mita Date: Mon, 30 Jul 2012 14:43:07 -0700 Subject: PM: PM notifier error injection module This provides the ability to inject artifical errors to PM notifier chain callbacks. It is controlled through debugfs interface under /sys/kernel/debug/notifier-error-inject/pm Each of the files in "error" directory represents an event which can be failed and contains the error code. If the notifier call chain should be failed with some events notified, write the error code to the files. If the notifier call chain should be failed with some events notified, write the error code to "actions//error". Example: Inject PM suspend error (-12 = -ENOMEM) # cd /sys/kernel/debug/notifier-error-inject/pm # echo -12 > actions/PM_SUSPEND_PREPARE/error # echo mem > /sys/power/state bash: echo: write error: Cannot allocate memory Signed-off-by: Akinobu Mita Acked-by: "Rafael J. Wysocki" Cc: Pavel Machek Cc: Greg KH Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: Michael Ellerman Cc: Dave Jones Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/Kconfig.debug | 24 +++++++++++++++++++++ lib/Makefile | 1 + lib/pm-notifier-error-inject.c | 49 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+) create mode 100644 lib/pm-notifier-error-inject.c (limited to 'lib/Kconfig.debug') diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 20341480bb54..c2024d86fb12 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1119,6 +1119,30 @@ config CPU_NOTIFIER_ERROR_INJECT If unsure, say N. +config PM_NOTIFIER_ERROR_INJECT + tristate "PM notifier error injection module" + depends on PM && NOTIFIER_ERROR_INJECTION + default m if PM_DEBUG + help + This option provides the ability to inject artifical errors to + PM notifier chain callbacks. It is controlled through debugfs + interface /sys/kernel/debug/notifier-error-inject/pm + + If the notifier call chain should be failed with some events + notified, write the error code to "actions//error". + + Example: Inject PM suspend error (-12 = -ENOMEM) + + # cd /sys/kernel/debug/notifier-error-inject/pm/ + # echo -12 > actions/PM_SUSPEND_PREPARE/error + # echo mem > /sys/power/state + bash: echo: write error: Cannot allocate memory + + To compile this code as a module, choose M here: the module will + be called pm-notifier-error-inject. + + If unsure, say N. + config FAULT_INJECTION bool "Fault-injection framework" depends on DEBUG_KERNEL diff --git a/lib/Makefile b/lib/Makefile index 71acfa4489f3..c32c66473ba1 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -92,6 +92,7 @@ obj-$(CONFIG_IOMMU_HELPER) += iommu-helper.o obj-$(CONFIG_FAULT_INJECTION) += fault-inject.o obj-$(CONFIG_NOTIFIER_ERROR_INJECTION) += notifier-error-inject.o obj-$(CONFIG_CPU_NOTIFIER_ERROR_INJECT) += cpu-notifier-error-inject.o +obj-$(CONFIG_PM_NOTIFIER_ERROR_INJECT) += pm-notifier-error-inject.o lib-$(CONFIG_GENERIC_BUG) += bug.o diff --git a/lib/pm-notifier-error-inject.c b/lib/pm-notifier-error-inject.c new file mode 100644 index 000000000000..c094b2dedc23 --- /dev/null +++ b/lib/pm-notifier-error-inject.c @@ -0,0 +1,49 @@ +#include +#include +#include + +#include "notifier-error-inject.h" + +static int priority; +module_param(priority, int, 0); +MODULE_PARM_DESC(priority, "specify PM notifier priority"); + +static struct notifier_err_inject pm_notifier_err_inject = { + .actions = { + { NOTIFIER_ERR_INJECT_ACTION(PM_HIBERNATION_PREPARE) }, + { NOTIFIER_ERR_INJECT_ACTION(PM_SUSPEND_PREPARE) }, + { NOTIFIER_ERR_INJECT_ACTION(PM_RESTORE_PREPARE) }, + {} + } +}; + +static struct dentry *dir; + +static int err_inject_init(void) +{ + int err; + + dir = notifier_err_inject_init("pm", notifier_err_inject_dir, + &pm_notifier_err_inject, priority); + if (IS_ERR(dir)) + return PTR_ERR(dir); + + err = register_pm_notifier(&pm_notifier_err_inject.nb); + if (err) + debugfs_remove_recursive(dir); + + return err; +} + +static void err_inject_exit(void) +{ + unregister_pm_notifier(&pm_notifier_err_inject.nb); + debugfs_remove_recursive(dir); +} + +module_init(err_inject_init); +module_exit(err_inject_exit); + +MODULE_DESCRIPTION("PM notifier error injection module"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Akinobu Mita "); -- cgit v1.3-14-g43fede From 9579f5bd31a04e80a87a7b58bd52dff6dc68bc99 Mon Sep 17 00:00:00 2001 From: Akinobu Mita Date: Mon, 30 Jul 2012 14:43:10 -0700 Subject: memory: memory notifier error injection module This provides the ability to inject artifical errors to memory hotplug notifier chain callbacks. It is controlled through debugfs interface under /sys/kernel/debug/notifier-error-inject/memory If the notifier call chain should be failed with some events notified, write the error code to "actions//error". Example: Inject memory hotplug offline error (-12 == -ENOMEM) # cd /sys/kernel/debug/notifier-error-inject/memory # echo -12 > actions/MEM_GOING_OFFLINE/error # echo offline > /sys/devices/system/memory/memoryXXX/state bash: echo: write error: Cannot allocate memory Signed-off-by: Akinobu Mita Cc: Pavel Machek Cc: "Rafael J. Wysocki" Cc: Greg KH Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: Michael Ellerman Cc: Dave Jones Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/Kconfig.debug | 23 ++++++++++++++++++ lib/Makefile | 1 + lib/memory-notifier-error-inject.c | 48 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100644 lib/memory-notifier-error-inject.c (limited to 'lib/Kconfig.debug') diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index c2024d86fb12..35cfd9eef0e0 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1143,6 +1143,29 @@ config PM_NOTIFIER_ERROR_INJECT If unsure, say N. +config MEMORY_NOTIFIER_ERROR_INJECT + tristate "Memory hotplug notifier error injection module" + depends on MEMORY_HOTPLUG_SPARSE && NOTIFIER_ERROR_INJECTION + help + This option provides the ability to inject artifical errors to + memory hotplug notifier chain callbacks. It is controlled through + debugfs interface under /sys/kernel/debug/notifier-error-inject/memory + + If the notifier call chain should be failed with some events + notified, write the error code to "actions//error". + + Example: Inject memory hotplug offline error (-12 == -ENOMEM) + + # cd /sys/kernel/debug/notifier-error-inject/memory + # echo -12 > actions/MEM_GOING_OFFLINE/error + # echo offline > /sys/devices/system/memory/memoryXXX/state + bash: echo: write error: Cannot allocate memory + + To compile this code as a module, choose M here: the module will + be called pSeries-reconfig-notifier-error-inject. + + If unsure, say N. + config FAULT_INJECTION bool "Fault-injection framework" depends on DEBUG_KERNEL diff --git a/lib/Makefile b/lib/Makefile index c32c66473ba1..c34719337d26 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -93,6 +93,7 @@ obj-$(CONFIG_FAULT_INJECTION) += fault-inject.o obj-$(CONFIG_NOTIFIER_ERROR_INJECTION) += notifier-error-inject.o obj-$(CONFIG_CPU_NOTIFIER_ERROR_INJECT) += cpu-notifier-error-inject.o obj-$(CONFIG_PM_NOTIFIER_ERROR_INJECT) += pm-notifier-error-inject.o +obj-$(CONFIG_MEMORY_NOTIFIER_ERROR_INJECT) += memory-notifier-error-inject.o lib-$(CONFIG_GENERIC_BUG) += bug.o diff --git a/lib/memory-notifier-error-inject.c b/lib/memory-notifier-error-inject.c new file mode 100644 index 000000000000..e6239bf0b0df --- /dev/null +++ b/lib/memory-notifier-error-inject.c @@ -0,0 +1,48 @@ +#include +#include +#include + +#include "notifier-error-inject.h" + +static int priority; +module_param(priority, int, 0); +MODULE_PARM_DESC(priority, "specify memory notifier priority"); + +static struct notifier_err_inject memory_notifier_err_inject = { + .actions = { + { NOTIFIER_ERR_INJECT_ACTION(MEM_GOING_ONLINE) }, + { NOTIFIER_ERR_INJECT_ACTION(MEM_GOING_OFFLINE) }, + {} + } +}; + +static struct dentry *dir; + +static int err_inject_init(void) +{ + int err; + + dir = notifier_err_inject_init("memory", notifier_err_inject_dir, + &memory_notifier_err_inject, priority); + if (IS_ERR(dir)) + return PTR_ERR(dir); + + err = register_memory_notifier(&memory_notifier_err_inject.nb); + if (err) + debugfs_remove_recursive(dir); + + return err; +} + +static void err_inject_exit(void) +{ + unregister_memory_notifier(&memory_notifier_err_inject.nb); + debugfs_remove_recursive(dir); +} + +module_init(err_inject_init); +module_exit(err_inject_exit); + +MODULE_DESCRIPTION("memory notifier error injection module"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Akinobu Mita "); -- cgit v1.3-14-g43fede From 08dfb4ddeeeebdee4f3d5a08a87dc9aa68d26f81 Mon Sep 17 00:00:00 2001 From: Akinobu Mita Date: Mon, 30 Jul 2012 14:43:13 -0700 Subject: powerpc: pSeries reconfig notifier error injection module This provides the ability to inject artifical errors to pSeries reconfig notifier chain callbacks. It is controlled through debugfs interface under /sys/kernel/debug/notifier-error-inject/pSeries-reconfig If the notifier call chain should be failed with some events notified, write the error code to "actions//error". Signed-off-by: Akinobu Mita Cc: Pavel Machek Cc: "Rafael J. Wysocki" Cc: Greg KH Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: Michael Ellerman Cc: Dave Jones Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/Kconfig.debug | 17 ++++++++++ lib/Makefile | 2 ++ lib/pSeries-reconfig-notifier-error-inject.c | 51 ++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 lib/pSeries-reconfig-notifier-error-inject.c (limited to 'lib/Kconfig.debug') diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 35cfd9eef0e0..2403a63b5da5 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1166,6 +1166,23 @@ config MEMORY_NOTIFIER_ERROR_INJECT If unsure, say N. +config PSERIES_RECONFIG_NOTIFIER_ERROR_INJECT + tristate "pSeries reconfig notifier error injection module" + depends on PPC_PSERIES && NOTIFIER_ERROR_INJECTION + help + This option provides the ability to inject artifical errors to + pSeries reconfig notifier chain callbacks. It is controlled + through debugfs interface under + /sys/kernel/debug/notifier-error-inject/pSeries-reconfig/ + + If the notifier call chain should be failed with some events + notified, write the error code to "actions//error". + + To compile this code as a module, choose M here: the module will + be called memory-notifier-error-inject. + + If unsure, say N. + config FAULT_INJECTION bool "Fault-injection framework" depends on DEBUG_KERNEL diff --git a/lib/Makefile b/lib/Makefile index c34719337d26..1054de86ae82 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -94,6 +94,8 @@ obj-$(CONFIG_NOTIFIER_ERROR_INJECTION) += notifier-error-inject.o obj-$(CONFIG_CPU_NOTIFIER_ERROR_INJECT) += cpu-notifier-error-inject.o obj-$(CONFIG_PM_NOTIFIER_ERROR_INJECT) += pm-notifier-error-inject.o obj-$(CONFIG_MEMORY_NOTIFIER_ERROR_INJECT) += memory-notifier-error-inject.o +obj-$(CONFIG_PSERIES_RECONFIG_NOTIFIER_ERROR_INJECT) += \ + pSeries-reconfig-notifier-error-inject.o lib-$(CONFIG_GENERIC_BUG) += bug.o diff --git a/lib/pSeries-reconfig-notifier-error-inject.c b/lib/pSeries-reconfig-notifier-error-inject.c new file mode 100644 index 000000000000..7f7c98dcd5c4 --- /dev/null +++ b/lib/pSeries-reconfig-notifier-error-inject.c @@ -0,0 +1,51 @@ +#include +#include + +#include + +#include "notifier-error-inject.h" + +static int priority; +module_param(priority, int, 0); +MODULE_PARM_DESC(priority, "specify pSeries reconfig notifier priority"); + +static struct notifier_err_inject reconfig_err_inject = { + .actions = { + { NOTIFIER_ERR_INJECT_ACTION(PSERIES_RECONFIG_ADD) }, + { NOTIFIER_ERR_INJECT_ACTION(PSERIES_RECONFIG_REMOVE) }, + { NOTIFIER_ERR_INJECT_ACTION(PSERIES_DRCONF_MEM_ADD) }, + { NOTIFIER_ERR_INJECT_ACTION(PSERIES_DRCONF_MEM_REMOVE) }, + {} + } +}; + +static struct dentry *dir; + +static int err_inject_init(void) +{ + int err; + + dir = notifier_err_inject_init("pSeries-reconfig", + notifier_err_inject_dir, &reconfig_err_inject, priority); + if (IS_ERR(dir)) + return PTR_ERR(dir); + + err = pSeries_reconfig_notifier_register(&reconfig_err_inject.nb); + if (err) + debugfs_remove_recursive(dir); + + return err; +} + +static void err_inject_exit(void) +{ + pSeries_reconfig_notifier_unregister(&reconfig_err_inject.nb); + debugfs_remove_recursive(dir); +} + +module_init(err_inject_init); +module_exit(err_inject_exit); + +MODULE_DESCRIPTION("pSeries reconfig notifier error injection module"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Akinobu Mita "); -- cgit v1.3-14-g43fede