diff options
Diffstat (limited to 'arch')
132 files changed, 2755 insertions, 2248 deletions
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 43594d5116ef..cd5c1c97b043 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -2064,8 +2064,7 @@ config KEXEC It is an ongoing process to be certain the hardware in a machine is properly shutdown, so do not be surprised if this code does not - initially work for you. It may help to enable device hotplugging - support. + initially work for you. config ATAGS_PROC bool "Export atags in procfs" diff --git a/arch/arm/boot/dts/am335x-bone.dts b/arch/arm/boot/dts/am335x-bone.dts index 444b4ede0d60..d318987d44a1 100644 --- a/arch/arm/boot/dts/am335x-bone.dts +++ b/arch/arm/boot/dts/am335x-bone.dts @@ -120,6 +120,35 @@ status = "okay"; }; + musb: usb@47400000 { + status = "okay"; + + control@44e10000 { + status = "okay"; + }; + + usb-phy@47401300 { + status = "okay"; + }; + + usb-phy@47401b00 { + status = "okay"; + }; + + usb@47401000 { + status = "okay"; + }; + + usb@47401800 { + status = "okay"; + dr_mode = "host"; + }; + + dma-controller@07402000 { + status = "okay"; + }; + }; + i2c0: i2c@44e0b000 { pinctrl-names = "default"; pinctrl-0 = <&i2c0_pins>; diff --git a/arch/arm/boot/dts/am335x-evm.dts b/arch/arm/boot/dts/am335x-evm.dts index 3aee1a43782d..e8ec8756e498 100644 --- a/arch/arm/boot/dts/am335x-evm.dts +++ b/arch/arm/boot/dts/am335x-evm.dts @@ -171,6 +171,35 @@ }; }; + musb: usb@47400000 { + status = "okay"; + + control@44e10000 { + status = "okay"; + }; + + usb-phy@47401300 { + status = "okay"; + }; + + usb-phy@47401b00 { + status = "okay"; + }; + + usb@47401000 { + status = "okay"; + }; + + usb@47401800 { + status = "okay"; + dr_mode = "host"; + }; + + dma-controller@07402000 { + status = "okay"; + }; + }; + i2c1: i2c@4802a000 { pinctrl-names = "default"; pinctrl-0 = <&i2c1_pins>; diff --git a/arch/arm/boot/dts/am335x-evmsk.dts b/arch/arm/boot/dts/am335x-evmsk.dts index 0c8ad173d2b0..4f339fa91c57 100644 --- a/arch/arm/boot/dts/am335x-evmsk.dts +++ b/arch/arm/boot/dts/am335x-evmsk.dts @@ -14,6 +14,7 @@ /dts-v1/; #include "am33xx.dtsi" +#include <dt-bindings/pwm/pwm.h> / { model = "TI AM335x EVM-SK"; @@ -207,6 +208,22 @@ }; }; + musb: usb@47400000 { + status = "okay"; + + control@44e10000 { + status = "okay"; + }; + + usb-phy@47401300 { + status = "okay"; + }; + + usb@47401000 { + status = "okay"; + }; + }; + epwmss2: epwmss@48304000 { status = "okay"; @@ -298,7 +315,7 @@ backlight { compatible = "pwm-backlight"; - pwms = <&ecap2 0 50000 1>; + pwms = <&ecap2 0 50000 PWM_POLARITY_INVERTED>; brightness-levels = <0 58 61 66 75 90 125 170 255>; default-brightness-level = <8>; }; diff --git a/arch/arm/boot/dts/am33xx.dtsi b/arch/arm/boot/dts/am33xx.dtsi index 38b446ba1ce1..f9c5da9c7fe1 100644 --- a/arch/arm/boot/dts/am33xx.dtsi +++ b/arch/arm/boot/dts/am33xx.dtsi @@ -26,6 +26,10 @@ serial5 = &uart5; d_can0 = &dcan0; d_can1 = &dcan1; + usb0 = &usb0; + usb1 = &usb1; + phy0 = &usb0_phy; + phy1 = &usb1_phy; }; cpus { @@ -333,21 +337,132 @@ status = "disabled"; }; - usb@47400000 { - compatible = "ti,musb-am33xx"; - reg = <0x47400000 0x1000 /* usbss */ - 0x47401000 0x800 /* musb instance 0 */ - 0x47401800 0x800>; /* musb instance 1 */ - interrupts = <17 /* usbss */ - 18 /* musb instance 0 */ - 19>; /* musb instance 1 */ - multipoint = <1>; - num-eps = <16>; - ram-bits = <12>; - port0-mode = <3>; - port1-mode = <3>; - power = <250>; + usb: usb@47400000 { + compatible = "ti,am33xx-usb"; + reg = <0x47400000 0x1000>; + ranges; + #address-cells = <1>; + #size-cells = <1>; ti,hwmods = "usb_otg_hs"; + status = "disabled"; + + ctrl_mod: control@44e10000 { + compatible = "ti,am335x-usb-ctrl-module"; + reg = <0x44e10620 0x10 + 0x44e10648 0x4>; + reg-names = "phy_ctrl", "wakeup"; + status = "disabled"; + }; + + usb0_phy: usb-phy@47401300 { + compatible = "ti,am335x-usb-phy"; + reg = <0x47401300 0x100>; + reg-names = "phy"; + status = "disabled"; + ti,ctrl_mod = <&ctrl_mod>; + }; + + usb0: usb@47401000 { + compatible = "ti,musb-am33xx"; + status = "disabled"; + reg = <0x47401400 0x400 + 0x47401000 0x200>; + reg-names = "mc", "control"; + + interrupts = <18>; + interrupt-names = "mc"; + dr_mode = "otg"; + mentor,multipoint = <1>; + mentor,num-eps = <16>; + mentor,ram-bits = <12>; + mentor,power = <500>; + phys = <&usb0_phy>; + + dmas = <&cppi41dma 0 0 &cppi41dma 1 0 + &cppi41dma 2 0 &cppi41dma 3 0 + &cppi41dma 4 0 &cppi41dma 5 0 + &cppi41dma 6 0 &cppi41dma 7 0 + &cppi41dma 8 0 &cppi41dma 9 0 + &cppi41dma 10 0 &cppi41dma 11 0 + &cppi41dma 12 0 &cppi41dma 13 0 + &cppi41dma 14 0 &cppi41dma 0 1 + &cppi41dma 1 1 &cppi41dma 2 1 + &cppi41dma 3 1 &cppi41dma 4 1 + &cppi41dma 5 1 &cppi41dma 6 1 + &cppi41dma 7 1 &cppi41dma 8 1 + &cppi41dma 9 1 &cppi41dma 10 1 + &cppi41dma 11 1 &cppi41dma 12 1 + &cppi41dma 13 1 &cppi41dma 14 1>; + dma-names = + "rx1", "rx2", "rx3", "rx4", "rx5", "rx6", "rx7", + "rx8", "rx9", "rx10", "rx11", "rx12", "rx13", + "rx14", "rx15", + "tx1", "tx2", "tx3", "tx4", "tx5", "tx6", "tx7", + "tx8", "tx9", "tx10", "tx11", "tx12", "tx13", + "tx14", "tx15"; + }; + + usb1_phy: usb-phy@47401b00 { + compatible = "ti,am335x-usb-phy"; + reg = <0x47401b00 0x100>; + reg-names = "phy"; + status = "disabled"; + ti,ctrl_mod = <&ctrl_mod>; + }; + + usb1: usb@47401800 { + compatible = "ti,musb-am33xx"; + status = "disabled"; + reg = <0x47401c00 0x400 + 0x47401800 0x200>; + reg-names = "mc", "control"; + interrupts = <19>; + interrupt-names = "mc"; + dr_mode = "otg"; + mentor,multipoint = <1>; + mentor,num-eps = <16>; + mentor,ram-bits = <12>; + mentor,power = <500>; + phys = <&usb1_phy>; + + dmas = <&cppi41dma 15 0 &cppi41dma 16 0 + &cppi41dma 17 0 &cppi41dma 18 0 + &cppi41dma 19 0 &cppi41dma 20 0 + &cppi41dma 21 0 &cppi41dma 22 0 + &cppi41dma 23 0 &cppi41dma 24 0 + &cppi41dma 25 0 &cppi41dma 26 0 + &cppi41dma 27 0 &cppi41dma 28 0 + &cppi41dma 29 0 &cppi41dma 15 1 + &cppi41dma 16 1 &cppi41dma 17 1 + &cppi41dma 18 1 &cppi41dma 19 1 + &cppi41dma 20 1 &cppi41dma 21 1 + &cppi41dma 22 1 &cppi41dma 23 1 + &cppi41dma 24 1 &cppi41dma 25 1 + &cppi41dma 26 1 &cppi41dma 27 1 + &cppi41dma 28 1 &cppi41dma 29 1>; + dma-names = + "rx1", "rx2", "rx3", "rx4", "rx5", "rx6", "rx7", + "rx8", "rx9", "rx10", "rx11", "rx12", "rx13", + "rx14", "rx15", + "tx1", "tx2", "tx3", "tx4", "tx5", "tx6", "tx7", + "tx8", "tx9", "tx10", "tx11", "tx12", "tx13", + "tx14", "tx15"; + }; + + cppi41dma: dma-controller@07402000 { + compatible = "ti,am3359-cppi41"; + reg = <0x47400000 0x1000 + 0x47402000 0x1000 + 0x47403000 0x1000 + 0x47404000 0x4000>; + reg-names = "glue", "controller", "scheduler", "queuemgr"; + interrupts = <17>; + interrupt-names = "glue"; + #dma-cells = <2>; + #dma-channels = <30>; + #dma-requests = <256>; + status = "disabled"; + }; }; epwmss0: epwmss@48300000 { diff --git a/arch/arm/boot/dts/exynos5440.dtsi b/arch/arm/boot/dts/exynos5440.dtsi index ff7f5d855845..586134e2a382 100644 --- a/arch/arm/boot/dts/exynos5440.dtsi +++ b/arch/arm/boot/dts/exynos5440.dtsi @@ -248,6 +248,7 @@ #interrupt-cells = <1>; interrupt-map-mask = <0 0 0 0>; interrupt-map = <0x0 0 &gic 53>; + num-lanes = <4>; }; pcie@2a0000 { @@ -267,5 +268,6 @@ #interrupt-cells = <1>; interrupt-map-mask = <0 0 0 0>; interrupt-map = <0x0 0 &gic 56>; + num-lanes = <4>; }; }; diff --git a/arch/arm/boot/dts/imx28-evk.dts b/arch/arm/boot/dts/imx28-evk.dts index e035f4664b97..15715d921d14 100644 --- a/arch/arm/boot/dts/imx28-evk.dts +++ b/arch/arm/boot/dts/imx28-evk.dts @@ -220,6 +220,7 @@ auart0: serial@8006a000 { pinctrl-names = "default"; pinctrl-0 = <&auart0_pins_a>; + fsl,uart-has-rtscts; status = "okay"; }; diff --git a/arch/arm/boot/dts/msm8660-surf.dts b/arch/arm/boot/dts/msm8660-surf.dts index cdc010e0f93e..386d42870215 100644 --- a/arch/arm/boot/dts/msm8660-surf.dts +++ b/arch/arm/boot/dts/msm8660-surf.dts @@ -38,7 +38,7 @@ }; serial@19c40000 { - compatible = "qcom,msm-hsuart", "qcom,msm-uart"; + compatible = "qcom,msm-uartdm-v1.3", "qcom,msm-uartdm"; reg = <0x19c40000 0x1000>, <0x19c00000 0x1000>; interrupts = <0 195 0x0>; diff --git a/arch/arm/boot/dts/msm8960-cdp.dts b/arch/arm/boot/dts/msm8960-cdp.dts index 9c1167b0459b..93e9f7e0b7ad 100644 --- a/arch/arm/boot/dts/msm8960-cdp.dts +++ b/arch/arm/boot/dts/msm8960-cdp.dts @@ -38,7 +38,7 @@ }; serial@16440000 { - compatible = "qcom,msm-hsuart", "qcom,msm-uart"; + compatible = "qcom,msm-uartdm-v1.3", "qcom,msm-uartdm"; reg = <0x16440000 0x1000>, <0x16400000 0x1000>; interrupts = <0 154 0x0>; diff --git a/arch/arm/boot/dts/omap5.dtsi b/arch/arm/boot/dts/omap5.dtsi index e643620417a9..07be2cd7b318 100644 --- a/arch/arm/boot/dts/omap5.dtsi +++ b/arch/arm/boot/dts/omap5.dtsi @@ -644,7 +644,7 @@ utmi-mode = <2>; ranges; dwc3@4a030000 { - compatible = "synopsys,dwc3"; + compatible = "snps,dwc3"; reg = <0x4a030000 0x1000>; interrupts = <GIC_SPI 92 IRQ_TYPE_LEVEL_HIGH>; usb-phy = <&usb2_phy>, <&usb3_phy>; diff --git a/arch/arm/boot/dts/tegra20-seaboard.dts b/arch/arm/boot/dts/tegra20-seaboard.dts index 40e6fb280333..c8242533268f 100644 --- a/arch/arm/boot/dts/tegra20-seaboard.dts +++ b/arch/arm/boot/dts/tegra20-seaboard.dts @@ -566,7 +566,6 @@ usb@c5000000 { status = "okay"; - nvidia,vbus-gpio = <&gpio TEGRA_GPIO(D, 0) GPIO_ACTIVE_HIGH>; dr_mode = "otg"; }; diff --git a/arch/arm/boot/dts/tegra20-trimslice.dts b/arch/arm/boot/dts/tegra20-trimslice.dts index 37c93d3c4812..1e9d33adb925 100644 --- a/arch/arm/boot/dts/tegra20-trimslice.dts +++ b/arch/arm/boot/dts/tegra20-trimslice.dts @@ -312,7 +312,6 @@ usb@c5000000 { status = "okay"; - nvidia,vbus-gpio = <&gpio TEGRA_GPIO(V, 2) GPIO_ACTIVE_HIGH>; }; usb-phy@c5000000 { diff --git a/arch/arm/boot/dts/tegra20-whistler.dts b/arch/arm/boot/dts/tegra20-whistler.dts index a3d0ebad78a1..c703197dca6e 100644 --- a/arch/arm/boot/dts/tegra20-whistler.dts +++ b/arch/arm/boot/dts/tegra20-whistler.dts @@ -509,7 +509,6 @@ usb@c5000000 { status = "okay"; - nvidia,vbus-gpio = <&tca6416 0 GPIO_ACTIVE_HIGH>; }; usb-phy@c5000000 { @@ -519,7 +518,6 @@ usb@c5008000 { status = "okay"; - nvidia,vbus-gpio = <&tca6416 1 GPIO_ACTIVE_HIGH>; }; usb-phy@c5008000 { diff --git a/arch/arm/boot/dts/tegra20.dtsi b/arch/arm/boot/dts/tegra20.dtsi index 9653fd8288d2..e4570834512e 100644 --- a/arch/arm/boot/dts/tegra20.dtsi +++ b/arch/arm/boot/dts/tegra20.dtsi @@ -477,13 +477,13 @@ <&tegra_car TEGRA20_CLK_USBD>; clock-names = "reg", "pll_u", "timer", "utmi-pads"; nvidia,has-legacy-mode; - hssync_start_delay = <9>; - idle_wait_delay = <17>; - elastic_limit = <16>; - term_range_adj = <6>; - xcvr_setup = <9>; - xcvr_lsfslew = <1>; - xcvr_lsrslew = <1>; + nvidia,hssync-start-delay = <9>; + nvidia,idle-wait-delay = <17>; + nvidia,elastic-limit = <16>; + nvidia,term-range-adj = <6>; + nvidia,xcvr-setup = <9>; + nvidia,xcvr-lsfslew = <1>; + nvidia,xcvr-lsrslew = <1>; status = "disabled"; }; @@ -527,13 +527,13 @@ <&tegra_car TEGRA20_CLK_CLK_M>, <&tegra_car TEGRA20_CLK_USBD>; clock-names = "reg", "pll_u", "timer", "utmi-pads"; - hssync_start_delay = <9>; - idle_wait_delay = <17>; - elastic_limit = <16>; - term_range_adj = <6>; - xcvr_setup = <9>; - xcvr_lsfslew = <2>; - xcvr_lsrslew = <2>; + nvidia,hssync-start-delay = <9>; + nvidia,idle-wait-delay = <17>; + nvidia,elastic-limit = <16>; + nvidia,term-range-adj = <6>; + nvidia,xcvr-setup = <9>; + nvidia,xcvr-lsfslew = <2>; + nvidia,xcvr-lsrslew = <2>; status = "disabled"; }; diff --git a/arch/arm/boot/dts/wm8850-w70v2.dts b/arch/arm/boot/dts/wm8850-w70v2.dts index 90e913fb64be..7a563d2523b0 100644 --- a/arch/arm/boot/dts/wm8850-w70v2.dts +++ b/arch/arm/boot/dts/wm8850-w70v2.dts @@ -11,13 +11,14 @@ /dts-v1/; /include/ "wm8850.dtsi" +#include <dt-bindings/pwm/pwm.h> / { model = "Wondermedia WM8850-W70v2 Tablet"; backlight { compatible = "pwm-backlight"; - pwms = <&pwm 0 50000 1>; /* duty inverted */ + pwms = <&pwm 0 50000 PWM_POLARITY_INVERTED>; brightness-levels = <0 40 60 80 100 130 190 255>; default-brightness-level = <5>; diff --git a/arch/arm/kernel/bios32.c b/arch/arm/kernel/bios32.c index 261fcc826169..88e14d74b6de 100644 --- a/arch/arm/kernel/bios32.c +++ b/arch/arm/kernel/bios32.c @@ -525,11 +525,6 @@ void pci_common_init_dev(struct device *parent, struct hw_pci *hw) * Assign resources. */ pci_bus_assign_resources(bus); - - /* - * Enable bridges - */ - pci_enable_bridges(bus); } /* diff --git a/arch/arm/kernel/devtree.c b/arch/arm/kernel/devtree.c index 5859c8bc727c..2ee8a17d2b01 100644 --- a/arch/arm/kernel/devtree.c +++ b/arch/arm/kernel/devtree.c @@ -169,6 +169,11 @@ void __init arm_dt_init_cpu_maps(void) } } +bool arch_match_cpu_phys_id(int cpu, u64 phys_id) +{ + return (phys_id & MPIDR_HWID_BITMASK) == cpu_logical_map(cpu); +} + /** * setup_machine_fdt - Machine setup when an dtb was passed to the kernel * @dt_phys: physical address of dt blob diff --git a/arch/arm/kernel/topology.c b/arch/arm/kernel/topology.c index c5a59546a256..85a87370f144 100644 --- a/arch/arm/kernel/topology.c +++ b/arch/arm/kernel/topology.c @@ -74,12 +74,8 @@ struct cpu_efficiency table_efficiency[] = { {NULL, }, }; -struct cpu_capacity { - unsigned long hwid; - unsigned long capacity; -}; - -struct cpu_capacity *cpu_capacity; +unsigned long *__cpu_capacity; +#define cpu_capacity(cpu) __cpu_capacity[cpu] unsigned long middle_capacity = 1; @@ -100,15 +96,19 @@ static void __init parse_dt_topology(void) unsigned long capacity = 0; int alloc_size, cpu = 0; - alloc_size = nr_cpu_ids * sizeof(struct cpu_capacity); - cpu_capacity = kzalloc(alloc_size, GFP_NOWAIT); + alloc_size = nr_cpu_ids * sizeof(*__cpu_capacity); + __cpu_capacity = kzalloc(alloc_size, GFP_NOWAIT); - while ((cn = of_find_node_by_type(cn, "cpu"))) { - const u32 *rate, *reg; + for_each_possible_cpu(cpu) { + const u32 *rate; int len; - if (cpu >= num_possible_cpus()) - break; + /* too early to use cpu->of_node */ + cn = of_get_cpu_node(cpu, NULL); + if (!cn) { + pr_err("missing device node for CPU %d\n", cpu); + continue; + } for (cpu_eff = table_efficiency; cpu_eff->compatible; cpu_eff++) if (of_device_is_compatible(cn, cpu_eff->compatible)) @@ -124,12 +124,6 @@ static void __init parse_dt_topology(void) continue; } - reg = of_get_property(cn, "reg", &len); - if (!reg || len != 4) { - pr_err("%s missing reg property\n", cn->full_name); - continue; - } - capacity = ((be32_to_cpup(rate)) >> 20) * cpu_eff->efficiency; /* Save min capacity of the system */ @@ -140,13 +134,9 @@ static void __init parse_dt_topology(void) if (capacity > max_capacity) max_capacity = capacity; - cpu_capacity[cpu].capacity = capacity; - cpu_capacity[cpu++].hwid = be32_to_cpup(reg); + cpu_capacity(cpu) = capacity; } - if (cpu < num_possible_cpus()) - cpu_capacity[cpu].hwid = (unsigned long)(-1); - /* If min and max capacities are equals, we bypass the update of the * cpu_scale because all CPUs have the same capacity. Otherwise, we * compute a middle_capacity factor that will ensure that the capacity @@ -154,9 +144,7 @@ static void __init parse_dt_topology(void) * SCHED_POWER_SCALE, which is the default value, but with the * constraint explained near table_efficiency[]. */ - if (min_capacity == max_capacity) - cpu_capacity[0].hwid = (unsigned long)(-1); - else if (4*max_capacity < (3*(max_capacity + min_capacity))) + if (4*max_capacity < (3*(max_capacity + min_capacity))) middle_capacity = (min_capacity + max_capacity) >> (SCHED_POWER_SHIFT+1); else @@ -170,23 +158,12 @@ static void __init parse_dt_topology(void) * boot. The update of all CPUs is in O(n^2) for heteregeneous system but the * function returns directly for SMP system. */ -void update_cpu_power(unsigned int cpu, unsigned long hwid) +void update_cpu_power(unsigned int cpu) { - unsigned int idx = 0; - - /* look for the cpu's hwid in the cpu capacity table */ - for (idx = 0; idx < num_possible_cpus(); idx++) { - if (cpu_capacity[idx].hwid == hwid) - break; - - if (cpu_capacity[idx].hwid == -1) - return; - } - - if (idx == num_possible_cpus()) + if (!cpu_capacity(cpu)) return; - set_power_scale(cpu, cpu_capacity[idx].capacity / middle_capacity); + set_power_scale(cpu, cpu_capacity(cpu) / middle_capacity); printk(KERN_INFO "CPU%u: update cpu_power %lu\n", cpu, arch_scale_freq_power(NULL, cpu)); @@ -194,7 +171,7 @@ void update_cpu_power(unsigned int cpu, unsigned long hwid) #else static inline void parse_dt_topology(void) {} -static inline void update_cpu_power(unsigned int cpuid, unsigned int mpidr) {} +static inline void update_cpu_power(unsigned int cpuid) {} #endif /* @@ -281,7 +258,7 @@ void store_cpu_topology(unsigned int cpuid) update_siblings_masks(cpuid); - update_cpu_power(cpuid, mpidr & MPIDR_HWID_BITMASK); + update_cpu_power(cpuid); printk(KERN_INFO "CPU%u: thread %d, cpu %d, socket %d, mpidr %x\n", cpuid, cpu_topology[cpuid].thread_id, diff --git a/arch/arm/mach-at91/include/mach/at91_adc.h b/arch/arm/mach-at91/include/mach/at91_adc.h index 8e7ed5c90817..048a57f76bd3 100644 --- a/arch/arm/mach-at91/include/mach/at91_adc.h +++ b/arch/arm/mach-at91/include/mach/at91_adc.h @@ -28,9 +28,12 @@ #define AT91_ADC_TRGSEL_EXTERNAL (6 << 1) #define AT91_ADC_LOWRES (1 << 4) /* Low Resolution */ #define AT91_ADC_SLEEP (1 << 5) /* Sleep Mode */ -#define AT91_ADC_PRESCAL (0x3f << 8) /* Prescalar Rate Selection */ +#define AT91_ADC_PRESCAL_9260 (0x3f << 8) /* Prescalar Rate Selection */ +#define AT91_ADC_PRESCAL_9G45 (0xff << 8) #define AT91_ADC_PRESCAL_(x) ((x) << 8) -#define AT91_ADC_STARTUP (0x1f << 16) /* Startup Up Time */ +#define AT91_ADC_STARTUP_9260 (0x1f << 16) /* Startup Up Time */ +#define AT91_ADC_STARTUP_9G45 (0x7f << 16) +#define AT91_ADC_STARTUP_9X5 (0xf << 16) #define AT91_ADC_STARTUP_(x) ((x) << 16) #define AT91_ADC_SHTIM (0xf << 24) /* Sample & Hold Time */ #define AT91_ADC_SHTIM_(x) ((x) << 24) @@ -48,6 +51,9 @@ #define AT91_ADC_ENDRX (1 << 18) /* End of RX Buffer */ #define AT91_ADC_RXFUFF (1 << 19) /* RX Buffer Full */ +#define AT91_ADC_SR_9X5 0x30 /* Status Register for 9x5 */ +#define AT91_ADC_SR_DRDY_9X5 (1 << 24) /* Data Ready */ + #define AT91_ADC_LCDR 0x20 /* Last Converted Data Register */ #define AT91_ADC_LDATA (0x3ff) @@ -58,4 +64,10 @@ #define AT91_ADC_CHR(n) (0x30 + ((n) * 4)) /* Channel Data Register N */ #define AT91_ADC_DATA (0x3ff) +#define AT91_ADC_CDR0_9X5 (0x50) /* Channel Data Register 0 for 9X5 */ + +#define AT91_ADC_TRGR_9260 AT91_ADC_MR +#define AT91_ADC_TRGR_9G45 0x08 +#define AT91_ADC_TRGR_9X5 0xC0 + #endif diff --git a/arch/arm/mach-davinci/cpuidle.c b/arch/arm/mach-davinci/cpuidle.c index 36aef3a7dedb..f1ac1c94ac0f 100644 --- a/arch/arm/mach-davinci/cpuidle.c +++ b/arch/arm/mach-davinci/cpuidle.c @@ -65,7 +65,7 @@ static struct cpuidle_driver davinci_idle_driver = { .states[1] = { .enter = davinci_enter_idle, .exit_latency = 10, - .target_residency = 100000, + .target_residency = 10000, .flags = CPUIDLE_FLAG_TIME_VALID, .name = "DDR SR", .desc = "WFI and DDR Self Refresh", diff --git a/arch/arm/mach-imx/mach-imx6q.c b/arch/arm/mach-imx/mach-imx6q.c index 7be13f8e69a0..a02f275a198d 100644 --- a/arch/arm/mach-imx/mach-imx6q.c +++ b/arch/arm/mach-imx/mach-imx6q.c @@ -254,13 +254,12 @@ static void __init imx6q_opp_init(struct device *cpu_dev) { struct device_node *np; - np = of_find_node_by_path("/cpus/cpu@0"); + np = of_node_get(cpu_dev->of_node); if (!np) { pr_warn("failed to find cpu0 node\n"); return; } - cpu_dev->of_node = np; if (of_init_opp_table(cpu_dev)) { pr_warn("failed to init OPP table\n"); goto put_node; diff --git a/arch/arm/mach-msm/devices-msm7x00.c b/arch/arm/mach-msm/devices-msm7x00.c index 6d50fb964863..d83404d4b328 100644 --- a/arch/arm/mach-msm/devices-msm7x00.c +++ b/arch/arm/mach-msm/devices-msm7x00.c @@ -456,9 +456,9 @@ static struct clk_pcom_desc msm_clocks_7x01a[] = { CLK_PCOM("tsif_ref_clk", TSIF_REF_CLK, NULL, 0), CLK_PCOM("tv_dac_clk", TV_DAC_CLK, NULL, 0), CLK_PCOM("tv_enc_clk", TV_ENC_CLK, NULL, 0), - CLK_PCOM("uart_clk", UART1_CLK, "msm_serial.0", OFF), - CLK_PCOM("uart_clk", UART2_CLK, "msm_serial.1", 0), - CLK_PCOM("uart_clk", UART3_CLK, "msm_serial.2", OFF), + CLK_PCOM("core", UART1_CLK, "msm_serial.0", OFF), + CLK_PCOM("core", UART2_CLK, "msm_serial.1", 0), + CLK_PCOM("core", UART3_CLK, "msm_serial.2", OFF), CLK_PCOM("uart1dm_clk", UART1DM_CLK, NULL, OFF), CLK_PCOM("uart2dm_clk", UART2DM_CLK, NULL, 0), CLK_PCOM("usb_hs_clk", USB_HS_CLK, "msm_hsusb", OFF), diff --git a/arch/arm/mach-msm/devices-msm7x30.c b/arch/arm/mach-msm/devices-msm7x30.c index d4db75acff56..14e286948f69 100644 --- a/arch/arm/mach-msm/devices-msm7x30.c +++ b/arch/arm/mach-msm/devices-msm7x30.c @@ -211,7 +211,7 @@ static struct clk_pcom_desc msm_clocks_7x30[] = { CLK_PCOM("spi_pclk", SPI_P_CLK, NULL, 0), CLK_PCOM("tv_dac_clk", TV_DAC_CLK, NULL, 0), CLK_PCOM("tv_enc_clk", TV_ENC_CLK, NULL, 0), - CLK_PCOM("uart_clk", UART2_CLK, "msm_serial.1", 0), + CLK_PCOM("core", UART2_CLK, "msm_serial.1", 0), CLK_PCOM("usb_phy_clk", USB_PHY_CLK, NULL, 0), CLK_PCOM("usb_hs_clk", USB_HS_CLK, NULL, OFF), CLK_PCOM("usb_hs_pclk", USB_HS_P_CLK, NULL, OFF), diff --git a/arch/arm/mach-msm/devices-qsd8x50.c b/arch/arm/mach-msm/devices-qsd8x50.c index f5518112284b..2ed89b25d304 100644 --- a/arch/arm/mach-msm/devices-qsd8x50.c +++ b/arch/arm/mach-msm/devices-qsd8x50.c @@ -358,9 +358,9 @@ static struct clk_pcom_desc msm_clocks_8x50[] = { CLK_PCOM("tsif_ref_clk", TSIF_REF_CLK, NULL, 0), CLK_PCOM("tv_dac_clk", TV_DAC_CLK, NULL, 0), CLK_PCOM("tv_enc_clk", TV_ENC_CLK, NULL, 0), - CLK_PCOM("uart_clk", UART1_CLK, NULL, OFF), - CLK_PCOM("uart_clk", UART2_CLK, NULL, 0), - CLK_PCOM("uart_clk", UART3_CLK, "msm_serial.2", OFF), + CLK_PCOM("core", UART1_CLK, NULL, OFF), + CLK_PCOM("core", UART2_CLK, NULL, 0), + CLK_PCOM("core", UART3_CLK, "msm_serial.2", OFF), CLK_PCOM("uartdm_clk", UART1DM_CLK, NULL, OFF), CLK_PCOM("uartdm_clk", UART2DM_CLK, NULL, 0), CLK_PCOM("usb_hs_clk", USB_HS_CLK, NULL, OFF), diff --git a/arch/arm/mach-mvebu/platsmp.c b/arch/arm/mach-mvebu/platsmp.c index ce81d3031405..594b63db4215 100644 --- a/arch/arm/mach-mvebu/platsmp.c +++ b/arch/arm/mach-mvebu/platsmp.c @@ -29,45 +29,40 @@ #include "pmsu.h" #include "coherency.h" +static struct clk *__init get_cpu_clk(int cpu) +{ + struct clk *cpu_clk; + struct device_node *np = of_get_cpu_node(cpu, NULL); + + if (WARN(!np, "missing cpu node\n")) + return NULL; + cpu_clk = of_clk_get(np, 0); + if (WARN_ON(IS_ERR(cpu_clk))) + return NULL; + return cpu_clk; +} + void __init set_secondary_cpus_clock(void) { - int thiscpu; + int thiscpu, cpu; unsigned long rate; - struct clk *cpu_clk = NULL; - struct device_node *np = NULL; + struct clk *cpu_clk; thiscpu = smp_processor_id(); - for_each_node_by_type(np, "cpu") { - int err; - int cpu; - - err = of_property_read_u32(np, "reg", &cpu); - if (WARN_ON(err)) - return; - - if (cpu == thiscpu) { - cpu_clk = of_clk_get(np, 0); - break; - } - } - if (WARN_ON(IS_ERR(cpu_clk))) + cpu_clk = get_cpu_clk(thiscpu); + if (!cpu_clk) return; clk_prepare_enable(cpu_clk); rate = clk_get_rate(cpu_clk); /* set all the other CPU clk to the same rate than the boot CPU */ - for_each_node_by_type(np, "cpu") { - int err; - int cpu; - - err = of_property_read_u32(np, "reg", &cpu); - if (WARN_ON(err)) + for_each_possible_cpu(cpu) { + if (cpu == thiscpu) + continue; + cpu_clk = get_cpu_clk(cpu); + if (!cpu_clk) return; - - if (cpu != thiscpu) { - cpu_clk = of_clk_get(np, 0); - clk_set_rate(cpu_clk, rate); - } + clk_set_rate(cpu_clk, rate); } } diff --git a/arch/arm/mach-omap2/board-omap3beagle.c b/arch/arm/mach-omap2/board-omap3beagle.c index 04c116555412..1c6ae5f5bae7 100644 --- a/arch/arm/mach-omap2/board-omap3beagle.c +++ b/arch/arm/mach-omap2/board-omap3beagle.c @@ -33,7 +33,7 @@ #include <linux/mtd/nand.h> #include <linux/mmc/host.h> #include <linux/usb/phy.h> -#include <linux/usb/nop-usb-xceiv.h> +#include <linux/usb/usb_phy_gen_xceiv.h> #include <linux/regulator/machine.h> #include <linux/i2c/twl.h> @@ -279,7 +279,7 @@ static struct regulator_consumer_supply beagle_vsim_supply[] = { static struct gpio_led gpio_leds[]; /* PHY's VCC regulator might be added later, so flag that we need it */ -static struct nop_usb_xceiv_platform_data hsusb2_phy_data = { +static struct usb_phy_gen_xceiv_platform_data hsusb2_phy_data = { .needs_vcc = true, }; diff --git a/arch/arm/mach-omap2/board-omap3evm.c b/arch/arm/mach-omap2/board-omap3evm.c index 8c026269baca..52bdddd41e0e 100644 --- a/arch/arm/mach-omap2/board-omap3evm.c +++ b/arch/arm/mach-omap2/board-omap3evm.c @@ -33,7 +33,7 @@ #include <linux/i2c/twl.h> #include <linux/usb/otg.h> #include <linux/usb/musb.h> -#include <linux/usb/nop-usb-xceiv.h> +#include <linux/usb/usb_phy_gen_xceiv.h> #include <linux/smsc911x.h> #include <linux/wl12xx.h> @@ -468,7 +468,7 @@ struct wl12xx_platform_data omap3evm_wlan_data __initdata = { static struct regulator_consumer_supply omap3evm_vaux2_supplies[] = { REGULATOR_SUPPLY("VDD_CSIPHY1", "omap3isp"), /* OMAP ISP */ REGULATOR_SUPPLY("VDD_CSIPHY2", "omap3isp"), /* OMAP ISP */ - REGULATOR_SUPPLY("vcc", "nop_usb_xceiv.2"), /* hsusb port 2 */ + REGULATOR_SUPPLY("vcc", "usb_phy_gen_xceiv.2"), /* hsusb port 2 */ REGULATOR_SUPPLY("vaux2", NULL), }; diff --git a/arch/arm/mach-omap2/board-omap3pandora.c b/arch/arm/mach-omap2/board-omap3pandora.c index b1547a0edfcd..d2b455e70486 100644 --- a/arch/arm/mach-omap2/board-omap3pandora.c +++ b/arch/arm/mach-omap2/board-omap3pandora.c @@ -352,7 +352,7 @@ static struct regulator_consumer_supply pandora_vcc_lcd_supply[] = { }; static struct regulator_consumer_supply pandora_usb_phy_supply[] = { - REGULATOR_SUPPLY("vcc", "nop_usb_xceiv.2"), /* hsusb port 2 */ + REGULATOR_SUPPLY("vcc", "usb_phy_gen_xceiv.2"), /* hsusb port 2 */ }; /* ads7846 on SPI and 2 nub controllers on I2C */ diff --git a/arch/arm/mach-omap2/usb-host.c b/arch/arm/mach-omap2/usb-host.c index 2eb19d4d0aa1..e83a6a4b184a 100644 --- a/arch/arm/mach-omap2/usb-host.c +++ b/arch/arm/mach-omap2/usb-host.c @@ -28,7 +28,7 @@ #include <linux/io.h> #include <linux/gpio.h> #include <linux/usb/phy.h> -#include <linux/usb/nop-usb-xceiv.h> +#include <linux/usb/usb_phy_gen_xceiv.h> #include "soc.h" #include "omap_device.h" @@ -349,7 +349,7 @@ static struct fixed_voltage_config hsusb_reg_config = { /* .init_data filled later */ }; -static const char *nop_name = "nop_usb_xceiv"; /* NOP PHY driver */ +static const char *nop_name = "usb_phy_gen_xceiv"; /* NOP PHY driver */ static const char *reg_name = "reg-fixed-voltage"; /* Regulator driver */ /** @@ -460,9 +460,9 @@ int usbhs_init_phys(struct usbhs_phy_data *phy, int num_phys) pdevinfo.name = nop_name; pdevinfo.id = phy->port; pdevinfo.data = phy->platform_data; - pdevinfo.size_data = sizeof(struct nop_usb_xceiv_platform_data); - - scnprintf(phy_id, MAX_STR, "nop_usb_xceiv.%d", + pdevinfo.size_data = + sizeof(struct usb_phy_gen_xceiv_platform_data); + scnprintf(phy_id, MAX_STR, "usb_phy_gen_xceiv.%d", phy->port); pdev = platform_device_register_full(&pdevinfo); if (IS_ERR(pdev)) { diff --git a/arch/arm/mach-prima2/common.c b/arch/arm/mach-prima2/common.c index 2c70f74fed5d..e110b6d4ae8c 100644 --- a/arch/arm/mach-prima2/common.c +++ b/arch/arm/mach-prima2/common.c @@ -42,7 +42,6 @@ static const char *atlas6_dt_match[] __initdata = { DT_MACHINE_START(ATLAS6_DT, "Generic ATLAS6 (Flattened Device Tree)") /* Maintainer: Barry Song <baohua.song@csr.com> */ - .nr_irqs = 128, .map_io = sirfsoc_map_io, .init_time = sirfsoc_init_time, .init_late = sirfsoc_init_late, @@ -59,7 +58,6 @@ static const char *prima2_dt_match[] __initdata = { DT_MACHINE_START(PRIMA2_DT, "Generic PRIMA2 (Flattened Device Tree)") /* Maintainer: Barry Song <baohua.song@csr.com> */ - .nr_irqs = 128, .map_io = sirfsoc_map_io, .init_time = sirfsoc_init_time, .dma_zone_size = SZ_256M, diff --git a/arch/arm/mach-tegra/tegra.c b/arch/arm/mach-tegra/tegra.c index 0d1e4128d460..fc97cfd52769 100644 --- a/arch/arm/mach-tegra/tegra.c +++ b/arch/arm/mach-tegra/tegra.c @@ -29,7 +29,6 @@ #include <linux/of_fdt.h> #include <linux/of_platform.h> #include <linux/pda_power.h> -#include <linux/platform_data/tegra_usb.h> #include <linux/io.h> #include <linux/slab.h> #include <linux/sys_soc.h> @@ -46,40 +45,6 @@ #include "fuse.h" #include "iomap.h" -static struct tegra_ehci_platform_data tegra_ehci1_pdata = { - .operating_mode = TEGRA_USB_OTG, - .power_down_on_bus_suspend = 1, - .vbus_gpio = -1, -}; - -static struct tegra_ulpi_config tegra_ehci2_ulpi_phy_config = { - .reset_gpio = -1, - .clk = "cdev2", -}; - -static struct tegra_ehci_platform_data tegra_ehci2_pdata = { - .phy_config = &tegra_ehci2_ulpi_phy_config, - .operating_mode = TEGRA_USB_HOST, - .power_down_on_bus_suspend = 1, - .vbus_gpio = -1, -}; - -static struct tegra_ehci_platform_data tegra_ehci3_pdata = { - .operating_mode = TEGRA_USB_HOST, - .power_down_on_bus_suspend = 1, - .vbus_gpio = -1, -}; - -static struct of_dev_auxdata tegra20_auxdata_lookup[] __initdata = { - OF_DEV_AUXDATA("nvidia,tegra20-ehci", 0xC5000000, "tegra-ehci.0", - &tegra_ehci1_pdata), - OF_DEV_AUXDATA("nvidia,tegra20-ehci", 0xC5004000, "tegra-ehci.1", - &tegra_ehci2_pdata), - OF_DEV_AUXDATA("nvidia,tegra20-ehci", 0xC5008000, "tegra-ehci.2", - &tegra_ehci3_pdata), - {} -}; - static void __init tegra_dt_init(void) { struct soc_device_attribute *soc_dev_attr; @@ -112,8 +77,7 @@ static void __init tegra_dt_init(void) * devices */ out: - of_platform_populate(NULL, of_default_bus_match_table, - tegra20_auxdata_lookup, parent); + of_platform_populate(NULL, of_default_bus_match_table, NULL, parent); } static void __init trimslice_init(void) diff --git a/arch/arm/mach-ux500/Makefile b/arch/arm/mach-ux500/Makefile index bf9b6be5b180..fe1f3e26b88b 100644 --- a/arch/arm/mach-ux500/Makefile +++ b/arch/arm/mach-ux500/Makefile @@ -4,7 +4,6 @@ obj-y := cpu.o devices.o devices-common.o \ id.o usb.o timer.o pm.o -obj-$(CONFIG_CPU_IDLE) += cpuidle.o obj-$(CONFIG_CACHE_L2X0) += cache-l2x0.o obj-$(CONFIG_UX500_SOC_DB8500) += cpu-db8500.o devices-db8500.o obj-$(CONFIG_MACH_MOP500) += board-mop500.o board-mop500-sdi.o \ diff --git a/arch/arm/mach-ux500/cpuidle.c b/arch/arm/mach-ux500/cpuidle.c deleted file mode 100644 index a45dd09daed9..000000000000 --- a/arch/arm/mach-ux500/cpuidle.c +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright (c) 2012 Linaro : Daniel Lezcano <daniel.lezcano@linaro.org> (IBM) - * - * Based on the work of Rickard Andersson <rickard.andersson@stericsson.com> - * and Jonas Aaberg <jonas.aberg@stericsson.com>. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ - -#include <linux/module.h> -#include <linux/cpuidle.h> -#include <linux/spinlock.h> -#include <linux/atomic.h> -#include <linux/smp.h> -#include <linux/mfd/dbx500-prcmu.h> -#include <linux/platform_data/arm-ux500-pm.h> - -#include <asm/cpuidle.h> -#include <asm/proc-fns.h> - -#include "db8500-regs.h" -#include "id.h" - -static atomic_t master = ATOMIC_INIT(0); -static DEFINE_SPINLOCK(master_lock); - -static inline int ux500_enter_idle(struct cpuidle_device *dev, - struct cpuidle_driver *drv, int index) -{ - int this_cpu = smp_processor_id(); - bool recouple = false; - - if (atomic_inc_return(&master) == num_online_cpus()) { - - /* With this lock, we prevent the other cpu to exit and enter - * this function again and become the master */ - if (!spin_trylock(&master_lock)) - goto wfi; - - /* decouple the gic from the A9 cores */ - if (prcmu_gic_decouple()) { - spin_unlock(&master_lock); - goto out; - } - - /* If an error occur, we will have to recouple the gic - * manually */ - recouple = true; - - /* At this state, as the gic is decoupled, if the other - * cpu is in WFI, we have the guarantee it won't be wake - * up, so we can safely go to retention */ - if (!prcmu_is_cpu_in_wfi(this_cpu ? 0 : 1)) - goto out; - - /* The prcmu will be in charge of watching the interrupts - * and wake up the cpus */ - if (prcmu_copy_gic_settings()) - goto out; - - /* Check in the meantime an interrupt did - * not occur on the gic ... */ - if (prcmu_gic_pending_irq()) - goto out; - - /* ... and the prcmu */ - if (prcmu_pending_irq()) - goto out; - - /* Go to the retention state, the prcmu will wait for the - * cpu to go WFI and this is what happens after exiting this - * 'master' critical section */ - if (prcmu_set_power_state(PRCMU_AP_IDLE, true, true)) - goto out; - - /* When we switch to retention, the prcmu is in charge - * of recoupling the gic automatically */ - recouple = false; - - spin_unlock(&master_lock); - } -wfi: - cpu_do_idle(); -out: - atomic_dec(&master); - - if (recouple) { - prcmu_gic_recouple(); - spin_unlock(&master_lock); - } - - return index; -} - -static struct cpuidle_driver ux500_idle_driver = { - .name = "ux500_idle", - .owner = THIS_MODULE, - .states = { - ARM_CPUIDLE_WFI_STATE, - { - .enter = ux500_enter_idle, - .exit_latency = 70, - .target_residency = 260, - .flags = CPUIDLE_FLAG_TIME_VALID | - CPUIDLE_FLAG_TIMER_STOP, - .name = "ApIdle", - .desc = "ARM Retention", - }, - }, - .safe_state_index = 0, - .state_count = 2, -}; - -int __init ux500_idle_init(void) -{ - if (!(cpu_is_u8500_family() || cpu_is_ux540_family())) - return -ENODEV; - - /* Configure wake up reasons */ - prcmu_enable_wakeups(PRCMU_WAKEUP(ARM) | PRCMU_WAKEUP(RTC) | - PRCMU_WAKEUP(ABB)); - - return cpuidle_register(&ux500_idle_driver, NULL); -} - -device_initcall(ux500_idle_init); diff --git a/arch/frv/mb93090-mb00/pci-vdk.c b/arch/frv/mb93090-mb00/pci-vdk.c index 0aa35f0eb0db..deb67843693c 100644 --- a/arch/frv/mb93090-mb00/pci-vdk.c +++ b/arch/frv/mb93090-mb00/pci-vdk.c @@ -320,7 +320,7 @@ DECLARE_PCI_FIXUP_HEADER(PCI_ANY_ID, PCI_ANY_ID, pci_fixup_ide_bases); * are examined. */ -void __init pcibios_fixup_bus(struct pci_bus *bus) +void pcibios_fixup_bus(struct pci_bus *bus) { #if 0 printk("### PCIBIOS_FIXUP_BUS(%d)\n",bus->number); diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig index 5a768ad8e893..b36370d3eab9 100644 --- a/arch/ia64/Kconfig +++ b/arch/ia64/Kconfig @@ -565,9 +565,9 @@ config KEXEC It is an ongoing process to be certain the hardware in a machine is properly shutdown, so do not be surprised if this code does not - initially work for you. It may help to enable device hotplugging - support. As of this writing the exact hardware interface is - strongly in flux, so no good recommendation can be made. + initially work for you. As of this writing the exact hardware + interface is strongly in flux, so no good recommendation can be + made. config CRASH_DUMP bool "kernel crash dumps" diff --git a/arch/m68k/emu/natfeat.c b/arch/m68k/emu/natfeat.c index fa277aecfb78..121a6660ad4e 100644 --- a/arch/m68k/emu/natfeat.c +++ b/arch/m68k/emu/natfeat.c @@ -18,11 +18,11 @@ #include <asm/machdep.h> #include <asm/natfeat.h> -extern long nf_get_id2(const char *feature_name); +extern long nf_get_id_phys(unsigned long feature_name); asm("\n" -" .global nf_get_id2,nf_call\n" -"nf_get_id2:\n" +" .global nf_get_id_phys,nf_call\n" +"nf_get_id_phys:\n" " .short 0x7300\n" " rts\n" "nf_call:\n" @@ -31,7 +31,7 @@ asm("\n" "1: moveq.l #0,%d0\n" " rts\n" " .section __ex_table,\"a\"\n" -" .long nf_get_id2,1b\n" +" .long nf_get_id_phys,1b\n" " .long nf_call,1b\n" " .previous"); EXPORT_SYMBOL_GPL(nf_call); @@ -46,7 +46,7 @@ long nf_get_id(const char *feature_name) if (n >= sizeof(name_copy)) return 0; - return nf_get_id2(name_copy); + return nf_get_id_phys(virt_to_phys(name_copy)); } EXPORT_SYMBOL_GPL(nf_get_id); @@ -58,7 +58,7 @@ void nfprint(const char *fmt, ...) va_start(ap, fmt); n = vsnprintf(buf, 256, fmt, ap); - nf_call(nf_get_id("NF_STDERR"), buf); + nf_call(nf_get_id("NF_STDERR"), virt_to_phys(buf)); va_end(ap); } @@ -83,7 +83,7 @@ void nf_init(void) id = nf_get_id("NF_NAME"); if (!id) return; - nf_call(id, buf, 256); + nf_call(id, virt_to_phys(buf), 256); buf[255] = 0; pr_info("NatFeats found (%s, %lu.%lu)\n", buf, version >> 16, diff --git a/arch/m68k/emu/nfblock.c b/arch/m68k/emu/nfblock.c index e3011338ab40..0721858fbd1e 100644 --- a/arch/m68k/emu/nfblock.c +++ b/arch/m68k/emu/nfblock.c @@ -41,8 +41,8 @@ static inline s32 nfhd_read_write(u32 major, u32 minor, u32 rwflag, u32 recno, static inline s32 nfhd_get_capacity(u32 major, u32 minor, u32 *blocks, u32 *blocksize) { - return nf_call(nfhd_id + NFHD_GET_CAPACITY, major, minor, blocks, - blocksize); + return nf_call(nfhd_id + NFHD_GET_CAPACITY, major, minor, + virt_to_phys(blocks), virt_to_phys(blocksize)); } static LIST_HEAD(nfhd_list); diff --git a/arch/m68k/emu/nfcon.c b/arch/m68k/emu/nfcon.c index 6685bf45c2c3..57e8c8fb5eba 100644 --- a/arch/m68k/emu/nfcon.c +++ b/arch/m68k/emu/nfcon.c @@ -15,6 +15,7 @@ #include <linux/slab.h> #include <linux/err.h> #include <linux/uaccess.h> +#include <linux/io.h> #include <asm/natfeat.h> @@ -25,17 +26,18 @@ static struct tty_driver *nfcon_tty_driver; static void nfputs(const char *str, unsigned int count) { char buf[68]; + unsigned long phys = virt_to_phys(buf); buf[64] = 0; while (count > 64) { memcpy(buf, str, 64); - nf_call(stderr_id, buf); + nf_call(stderr_id, phys); str += 64; count -= 64; } memcpy(buf, str, count); buf[count] = 0; - nf_call(stderr_id, buf); + nf_call(stderr_id, phys); } static void nfcon_write(struct console *con, const char *str, @@ -79,7 +81,7 @@ static int nfcon_tty_put_char(struct tty_struct *tty, unsigned char ch) { char temp[2] = { ch, 0 }; - nf_call(stderr_id, temp); + nf_call(stderr_id, virt_to_phys(temp)); return 1; } diff --git a/arch/m68k/emu/nfeth.c b/arch/m68k/emu/nfeth.c index 695cd737a42e..a0985fd088d1 100644 --- a/arch/m68k/emu/nfeth.c +++ b/arch/m68k/emu/nfeth.c @@ -195,7 +195,8 @@ static struct net_device * __init nfeth_probe(int unit) char mac[ETH_ALEN], host_ip[32], local_ip[32]; int err; - if (!nf_call(nfEtherID + XIF_GET_MAC, unit, mac, ETH_ALEN)) + if (!nf_call(nfEtherID + XIF_GET_MAC, unit, virt_to_phys(mac), + ETH_ALEN)) return NULL; dev = alloc_etherdev(sizeof(struct nfeth_private)); @@ -217,9 +218,9 @@ static struct net_device * __init nfeth_probe(int unit) } nf_call(nfEtherID + XIF_GET_IPHOST, unit, - host_ip, sizeof(host_ip)); + virt_to_phys(host_ip), sizeof(host_ip)); nf_call(nfEtherID + XIF_GET_IPATARI, unit, - local_ip, sizeof(local_ip)); + virt_to_phys(local_ip), sizeof(local_ip)); netdev_info(dev, KBUILD_MODNAME " addr:%s (%s) HWaddr:%pM\n", host_ip, local_ip, mac); diff --git a/arch/m68k/include/asm/irqflags.h b/arch/m68k/include/asm/irqflags.h index 7ef4115b8c4a..5053092b369f 100644 --- a/arch/m68k/include/asm/irqflags.h +++ b/arch/m68k/include/asm/irqflags.h @@ -67,6 +67,10 @@ static inline void arch_local_irq_restore(unsigned long flags) static inline bool arch_irqs_disabled_flags(unsigned long flags) { + if (MACH_IS_ATARI) { + /* Ignore HSYNC = ipl 2 on Atari */ + return (flags & ~(ALLOWINT | 0x200)) != 0; + } return (flags & ~ALLOWINT) != 0; } diff --git a/arch/m68k/platform/coldfire/pci.c b/arch/m68k/platform/coldfire/pci.c index b33f97a13e6d..df9679238b6d 100644 --- a/arch/m68k/platform/coldfire/pci.c +++ b/arch/m68k/platform/coldfire/pci.c @@ -319,7 +319,6 @@ static int __init mcf_pci_init(void) pci_fixup_irqs(pci_common_swizzle, mcf_pci_map_irq); pci_bus_size_bridges(rootbus); pci_bus_assign_resources(rootbus); - pci_enable_bridges(rootbus); return 0; } diff --git a/arch/microblaze/include/asm/prom.h b/arch/microblaze/include/asm/prom.h index 20c5e8e5121b..9977816c5ad3 100644 --- a/arch/microblaze/include/asm/prom.h +++ b/arch/microblaze/include/asm/prom.h @@ -50,9 +50,6 @@ void of_parse_dma_window(struct device_node *dn, const void *dma_window_prop, extern void kdump_move_device_tree(void); -/* CPU OF node matching */ -struct device_node *of_get_cpu_node(int cpu, unsigned int *thread); - #endif /* __ASSEMBLY__ */ #endif /* __KERNEL__ */ diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index e12764c2a9d0..dccd7cec442d 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -2305,9 +2305,9 @@ config KEXEC It is an ongoing process to be certain the hardware in a machine is properly shutdown, so do not be surprised if this code does not - initially work for you. It may help to enable device hotplugging - support. As of this writing the exact hardware interface is - strongly in flux, so no good recommendation can be made. + initially work for you. As of this writing the exact hardware + interface is strongly in flux, so no good recommendation can be + made. config CRASH_DUMP bool "Kernel crash dumps" diff --git a/arch/mips/kernel/vpe.c b/arch/mips/kernel/vpe.c index 1765bab000a0..faf84c5f2629 100644 --- a/arch/mips/kernel/vpe.c +++ b/arch/mips/kernel/vpe.c @@ -1335,8 +1335,9 @@ static ssize_t store_kill(struct device *dev, struct device_attribute *attr, return len; } +static DEVICE_ATTR(kill, S_IWUSR, NULL, store_kill); -static ssize_t show_ntcs(struct device *cd, struct device_attribute *attr, +static ssize_t ntcs_show(struct device *cd, struct device_attribute *attr, char *buf) { struct vpe *vpe = get_vpe(tclimit); @@ -1344,7 +1345,7 @@ static ssize_t show_ntcs(struct device *cd, struct device_attribute *attr, return sprintf(buf, "%d\n", vpe->ntcs); } -static ssize_t store_ntcs(struct device *dev, struct device_attribute *attr, +static ssize_t ntcs_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len) { struct vpe *vpe = get_vpe(tclimit); @@ -1365,12 +1366,14 @@ static ssize_t store_ntcs(struct device *dev, struct device_attribute *attr, out_einval: return -EINVAL; } +static DEVICE_ATTR_RW(ntcs); -static struct device_attribute vpe_class_attributes[] = { - __ATTR(kill, S_IWUSR, NULL, store_kill), - __ATTR(ntcs, S_IRUGO | S_IWUSR, show_ntcs, store_ntcs), - {} +static struct attribute vpe_attrs[] = { + &dev_attr_kill.attr, + &dev_attr_ntcs.attr, + NULL, }; +ATTRIBUTE_GROUPS(vpe); static void vpe_device_release(struct device *cd) { @@ -1381,7 +1384,7 @@ struct class vpe_class = { .name = "vpe", .owner = THIS_MODULE, .dev_release = vpe_device_release, - .dev_attrs = vpe_class_attributes, + .dev_groups = vpe_groups, }; struct device vpe_device; diff --git a/arch/mips/pci/pci.c b/arch/mips/pci/pci.c index 594e60d6a43b..33e7aa52d9c4 100644 --- a/arch/mips/pci/pci.c +++ b/arch/mips/pci/pci.c @@ -113,7 +113,6 @@ static void pcibios_scanbus(struct pci_controller *hose) if (!pci_has_flag(PCI_PROBE_ONLY)) { pci_bus_size_bridges(bus); pci_bus_assign_resources(bus); - pci_enable_bridges(bus); } } } diff --git a/arch/mips/sni/a20r.c b/arch/mips/sni/a20r.c index dd0ab982d77e..f9407e170476 100644 --- a/arch/mips/sni/a20r.c +++ b/arch/mips/sni/a20r.c @@ -122,7 +122,6 @@ static struct resource sc26xx_rsrc[] = { static struct sccnxp_pdata sccnxp_data = { .reg_shift = 2, - .frequency = 3686400, .mctrl_cfg[0] = MCTRL_SIG(DTR_OP, LINE_OP7) | MCTRL_SIG(RTS_OP, LINE_OP3) | MCTRL_SIG(DSR_IP, LINE_IP5) | diff --git a/arch/openrisc/include/asm/prom.h b/arch/openrisc/include/asm/prom.h index bbb34e5343a2..eb59bfe23e85 100644 --- a/arch/openrisc/include/asm/prom.h +++ b/arch/openrisc/include/asm/prom.h @@ -44,9 +44,6 @@ void of_parse_dma_window(struct device_node *dn, const void *dma_window_prop, extern void kdump_move_device_tree(void); -/* CPU OF node matching */ -struct device_node *of_get_cpu_node(int cpu, unsigned int *thread); - /* Get the MAC address */ extern const void *of_get_mac_address(struct device_node *np); diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 9cf59816d3e9..5aecda05e0da 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -369,9 +369,9 @@ config KEXEC It is an ongoing process to be certain the hardware in a machine is properly shutdown, so do not be surprised if this code does not - initially work for you. It may help to enable device hotplugging - support. As of this writing the exact hardware interface is - strongly in flux, so no good recommendation can be made. + initially work for you. As of this writing the exact hardware + interface is strongly in flux, so no good recommendation can be + made. config CRASH_DUMP bool "Build a kdump crash kernel" diff --git a/arch/powerpc/include/asm/perf_event_server.h b/arch/powerpc/include/asm/perf_event_server.h index 8b2492644754..3fd2f1b6f906 100644 --- a/arch/powerpc/include/asm/perf_event_server.h +++ b/arch/powerpc/include/asm/perf_event_server.h @@ -138,11 +138,11 @@ extern ssize_t power_events_sysfs_show(struct device *dev, #define EVENT_PTR(_id, _suffix) &EVENT_VAR(_id, _suffix).attr.attr #define EVENT_ATTR(_name, _id, _suffix) \ - PMU_EVENT_ATTR(_name, EVENT_VAR(_id, _suffix), PME_PM_##_id, \ + PMU_EVENT_ATTR(_name, EVENT_VAR(_id, _suffix), PME_##_id, \ power_events_sysfs_show) #define GENERIC_EVENT_ATTR(_name, _id) EVENT_ATTR(_name, _id, _g) #define GENERIC_EVENT_PTR(_id) EVENT_PTR(_id, _g) -#define POWER_EVENT_ATTR(_name, _id) EVENT_ATTR(PM_##_name, _id, _p) +#define POWER_EVENT_ATTR(_name, _id) EVENT_ATTR(_name, _id, _p) #define POWER_EVENT_PTR(_id) EVENT_PTR(_id, _p) diff --git a/arch/powerpc/include/asm/prom.h b/arch/powerpc/include/asm/prom.h index bc2da154f68b..ac204e022922 100644 --- a/arch/powerpc/include/asm/prom.h +++ b/arch/powerpc/include/asm/prom.h @@ -43,9 +43,6 @@ void of_parse_dma_window(struct device_node *dn, const void *dma_window_prop, extern void kdump_move_device_tree(void); -/* CPU OF node matching */ -struct device_node *of_get_cpu_node(int cpu, unsigned int *thread); - /* cache lookup */ struct device_node *of_find_next_cache_node(struct device_node *np); diff --git a/arch/powerpc/kernel/pci-common.c b/arch/powerpc/kernel/pci-common.c index 7d22a675fe1a..2b4a9a4db7d9 100644 --- a/arch/powerpc/kernel/pci-common.c +++ b/arch/powerpc/kernel/pci-common.c @@ -1674,12 +1674,8 @@ void pcibios_scan_phb(struct pci_controller *hose) /* Configure PCI Express settings */ if (bus && !pci_has_flag(PCI_PROBE_ONLY)) { struct pci_bus *child; - list_for_each_entry(child, &bus->children, node) { - struct pci_dev *self = child->self; - if (!self) - continue; - pcie_bus_configure_settings(child, self->pcie_mpss); - } + list_for_each_entry(child, &bus->children, node) + pcie_bus_configure_settings(child); } } diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c index eb23ac92abb9..1c14cd4a5e05 100644 --- a/arch/powerpc/kernel/prom.c +++ b/arch/powerpc/kernel/prom.c @@ -865,49 +865,10 @@ static int __init prom_reconfig_setup(void) __initcall(prom_reconfig_setup); #endif -/* Find the device node for a given logical cpu number, also returns the cpu - * local thread number (index in ibm,interrupt-server#s) if relevant and - * asked for (non NULL) - */ -struct device_node *of_get_cpu_node(int cpu, unsigned int *thread) +bool arch_match_cpu_phys_id(int cpu, u64 phys_id) { - int hardid; - struct device_node *np; - - hardid = get_hard_smp_processor_id(cpu); - - for_each_node_by_type(np, "cpu") { - const u32 *intserv; - unsigned int plen, t; - - /* Check for ibm,ppc-interrupt-server#s. If it doesn't exist - * fallback to "reg" property and assume no threads - */ - intserv = of_get_property(np, "ibm,ppc-interrupt-server#s", - &plen); - if (intserv == NULL) { - const u32 *reg = of_get_property(np, "reg", NULL); - if (reg == NULL) - continue; - if (*reg == hardid) { - if (thread) - *thread = 0; - return np; - } - } else { - plen /= sizeof(u32); - for (t = 0; t < plen; t++) { - if (hardid == intserv[t]) { - if (thread) - *thread = t; - return np; - } - } - } - } - return NULL; + return (int)phys_id == get_hard_smp_processor_id(cpu); } -EXPORT_SYMBOL(of_get_cpu_node); #if defined(CONFIG_DEBUG_FS) && defined(DEBUG) static struct debugfs_blob_wrapper flat_dt_blob; diff --git a/arch/powerpc/perf/power7-events-list.h b/arch/powerpc/perf/power7-events-list.h new file mode 100644 index 000000000000..687790a2c0b8 --- /dev/null +++ b/arch/powerpc/perf/power7-events-list.h @@ -0,0 +1,548 @@ +/* + * Performance counter support for POWER7 processors. + * + * Copyright 2013 Runzhen Wang, IBM Corporation. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + */ + +EVENT(PM_IC_DEMAND_L2_BR_ALL, 0x04898) +EVENT(PM_GCT_UTIL_7_TO_10_SLOTS, 0x020a0) +EVENT(PM_PMC2_SAVED, 0x10022) +EVENT(PM_CMPLU_STALL_DFU, 0x2003c) +EVENT(PM_VSU0_16FLOP, 0x0a0a4) +EVENT(PM_MRK_LSU_DERAT_MISS, 0x3d05a) +EVENT(PM_MRK_ST_CMPL, 0x10034) +EVENT(PM_NEST_PAIR3_ADD, 0x40881) +EVENT(PM_L2_ST_DISP, 0x46180) +EVENT(PM_L2_CASTOUT_MOD, 0x16180) +EVENT(PM_ISEG, 0x020a4) +EVENT(PM_MRK_INST_TIMEO, 0x40034) +EVENT(PM_L2_RCST_DISP_FAIL_ADDR, 0x36282) +EVENT(PM_LSU1_DC_PREF_STREAM_CONFIRM, 0x0d0b6) +EVENT(PM_IERAT_WR_64K, 0x040be) +EVENT(PM_MRK_DTLB_MISS_16M, 0x4d05e) +EVENT(PM_IERAT_MISS, 0x100f6) +EVENT(PM_MRK_PTEG_FROM_LMEM, 0x4d052) +EVENT(PM_FLOP, 0x100f4) +EVENT(PM_THRD_PRIO_4_5_CYC, 0x040b4) +EVENT(PM_BR_PRED_TA, 0x040aa) +EVENT(PM_CMPLU_STALL_FXU, 0x20014) +EVENT(PM_EXT_INT, 0x200f8) +EVENT(PM_VSU_FSQRT_FDIV, 0x0a888) +EVENT(PM_MRK_LD_MISS_EXPOSED_CYC, 0x1003e) +EVENT(PM_LSU1_LDF, 0x0c086) +EVENT(PM_IC_WRITE_ALL, 0x0488c) +EVENT(PM_LSU0_SRQ_STFWD, 0x0c0a0) +EVENT(PM_PTEG_FROM_RL2L3_MOD, 0x1c052) +EVENT(PM_MRK_DATA_FROM_L31_SHR, 0x1d04e) +EVENT(PM_DATA_FROM_L21_MOD, 0x3c046) +EVENT(PM_VSU1_SCAL_DOUBLE_ISSUED, 0x0b08a) +EVENT(PM_VSU0_8FLOP, 0x0a0a0) +EVENT(PM_POWER_EVENT1, 0x1006e) +EVENT(PM_DISP_CLB_HELD_BAL, 0x02092) +EVENT(PM_VSU1_2FLOP, 0x0a09a) +EVENT(PM_LWSYNC_HELD, 0x0209a) +EVENT(PM_PTEG_FROM_DL2L3_SHR, 0x3c054) +EVENT(PM_INST_FROM_L21_MOD, 0x34046) +EVENT(PM_IERAT_XLATE_WR_16MPLUS, 0x040bc) +EVENT(PM_IC_REQ_ALL, 0x04888) +EVENT(PM_DSLB_MISS, 0x0d090) +EVENT(PM_L3_MISS, 0x1f082) +EVENT(PM_LSU0_L1_PREF, 0x0d0b8) +EVENT(PM_VSU_SCALAR_SINGLE_ISSUED, 0x0b884) +EVENT(PM_LSU1_DC_PREF_STREAM_CONFIRM_STRIDE, 0x0d0be) +EVENT(PM_L2_INST, 0x36080) +EVENT(PM_VSU0_FRSP, 0x0a0b4) +EVENT(PM_FLUSH_DISP, 0x02082) +EVENT(PM_PTEG_FROM_L2MISS, 0x4c058) +EVENT(PM_VSU1_DQ_ISSUED, 0x0b09a) +EVENT(PM_CMPLU_STALL_LSU, 0x20012) +EVENT(PM_MRK_DATA_FROM_DMEM, 0x1d04a) +EVENT(PM_LSU_FLUSH_ULD, 0x0c8b0) +EVENT(PM_PTEG_FROM_LMEM, 0x4c052) +EVENT(PM_MRK_DERAT_MISS_16M, 0x3d05c) +EVENT(PM_THRD_ALL_RUN_CYC, 0x2000c) +EVENT(PM_MEM0_PREFETCH_DISP, 0x20083) +EVENT(PM_MRK_STALL_CMPLU_CYC_COUNT, 0x3003f) +EVENT(PM_DATA_FROM_DL2L3_MOD, 0x3c04c) +EVENT(PM_VSU_FRSP, 0x0a8b4) +EVENT(PM_MRK_DATA_FROM_L21_MOD, 0x3d046) +EVENT(PM_PMC1_OVERFLOW, 0x20010) +EVENT(PM_VSU0_SINGLE, 0x0a0a8) +EVENT(PM_MRK_PTEG_FROM_L3MISS, 0x2d058) +EVENT(PM_MRK_PTEG_FROM_L31_SHR, 0x2d056) +EVENT(PM_VSU0_VECTOR_SP_ISSUED, 0x0b090) +EVENT(PM_VSU1_FEST, 0x0a0ba) +EVENT(PM_MRK_INST_DISP, 0x20030) +EVENT(PM_VSU0_COMPLEX_ISSUED, 0x0b096) +EVENT(PM_LSU1_FLUSH_UST, 0x0c0b6) +EVENT(PM_INST_CMPL, 0x00002) +EVENT(PM_FXU_IDLE, 0x1000e) +EVENT(PM_LSU0_FLUSH_ULD, 0x0c0b0) +EVENT(PM_MRK_DATA_FROM_DL2L3_MOD, 0x3d04c) +EVENT(PM_LSU_LMQ_SRQ_EMPTY_ALL_CYC, 0x3001c) +EVENT(PM_LSU1_REJECT_LMQ_FULL, 0x0c0a6) +EVENT(PM_INST_PTEG_FROM_L21_MOD, 0x3e056) +EVENT(PM_INST_FROM_RL2L3_MOD, 0x14042) +EVENT(PM_SHL_CREATED, 0x05082) +EVENT(PM_L2_ST_HIT, 0x46182) +EVENT(PM_DATA_FROM_DMEM, 0x1c04a) +EVENT(PM_L3_LD_MISS, 0x2f082) +EVENT(PM_FXU1_BUSY_FXU0_IDLE, 0x4000e) +EVENT(PM_DISP_CLB_HELD_RES, 0x02094) +EVENT(PM_L2_SN_SX_I_DONE, 0x36382) +EVENT(PM_GRP_CMPL, 0x30004) +EVENT(PM_STCX_CMPL, 0x0c098) +EVENT(PM_VSU0_2FLOP, 0x0a098) +EVENT(PM_L3_PREF_MISS, 0x3f082) +EVENT(PM_LSU_SRQ_SYNC_CYC, 0x0d096) +EVENT(PM_LSU_REJECT_ERAT_MISS, 0x20064) +EVENT(PM_L1_ICACHE_MISS, 0x200fc) +EVENT(PM_LSU1_FLUSH_SRQ, 0x0c0be) +EVENT(PM_LD_REF_L1_LSU0, 0x0c080) +EVENT(PM_VSU0_FEST, 0x0a0b8) +EVENT(PM_VSU_VECTOR_SINGLE_ISSUED, 0x0b890) +EVENT(PM_FREQ_UP, 0x4000c) +EVENT(PM_DATA_FROM_LMEM, 0x3c04a) +EVENT(PM_LSU1_LDX, 0x0c08a) +EVENT(PM_PMC3_OVERFLOW, 0x40010) +EVENT(PM_MRK_BR_MPRED, 0x30036) +EVENT(PM_SHL_MATCH, 0x05086) +EVENT(PM_MRK_BR_TAKEN, 0x10036) +EVENT(PM_CMPLU_STALL_BRU, 0x4004e) +EVENT(PM_ISLB_MISS, 0x0d092) +EVENT(PM_CYC, 0x0001e) +EVENT(PM_DISP_HELD_THERMAL, 0x30006) +EVENT(PM_INST_PTEG_FROM_RL2L3_SHR, 0x2e054) +EVENT(PM_LSU1_SRQ_STFWD, 0x0c0a2) +EVENT(PM_GCT_NOSLOT_BR_MPRED, 0x4001a) +EVENT(PM_1PLUS_PPC_CMPL, 0x100f2) +EVENT(PM_PTEG_FROM_DMEM, 0x2c052) +EVENT(PM_VSU_2FLOP, 0x0a898) +EVENT(PM_GCT_FULL_CYC, 0x04086) +EVENT(PM_MRK_DATA_FROM_L3_CYC, 0x40020) +EVENT(PM_LSU_SRQ_S0_ALLOC, 0x0d09d) +EVENT(PM_MRK_DERAT_MISS_4K, 0x1d05c) +EVENT(PM_BR_MPRED_TA, 0x040ae) +EVENT(PM_INST_PTEG_FROM_L2MISS, 0x4e058) +EVENT(PM_DPU_HELD_POWER, 0x20006) +EVENT(PM_RUN_INST_CMPL, 0x400fa) +EVENT(PM_MRK_VSU_FIN, 0x30032) +EVENT(PM_LSU_SRQ_S0_VALID, 0x0d09c) +EVENT(PM_GCT_EMPTY_CYC, 0x20008) +EVENT(PM_IOPS_DISP, 0x30014) +EVENT(PM_RUN_SPURR, 0x10008) +EVENT(PM_PTEG_FROM_L21_MOD, 0x3c056) +EVENT(PM_VSU0_1FLOP, 0x0a080) +EVENT(PM_SNOOP_TLBIE, 0x0d0b2) +EVENT(PM_DATA_FROM_L3MISS, 0x2c048) +EVENT(PM_VSU_SINGLE, 0x0a8a8) +EVENT(PM_DTLB_MISS_16G, 0x1c05e) +EVENT(PM_CMPLU_STALL_VECTOR, 0x2001c) +EVENT(PM_FLUSH, 0x400f8) +EVENT(PM_L2_LD_HIT, 0x36182) +EVENT(PM_NEST_PAIR2_AND, 0x30883) +EVENT(PM_VSU1_1FLOP, 0x0a082) +EVENT(PM_IC_PREF_REQ, 0x0408a) +EVENT(PM_L3_LD_HIT, 0x2f080) +EVENT(PM_GCT_NOSLOT_IC_MISS, 0x2001a) +EVENT(PM_DISP_HELD, 0x10006) +EVENT(PM_L2_LD, 0x16080) +EVENT(PM_LSU_FLUSH_SRQ, 0x0c8bc) +EVENT(PM_BC_PLUS_8_CONV, 0x040b8) +EVENT(PM_MRK_DATA_FROM_L31_MOD_CYC, 0x40026) +EVENT(PM_CMPLU_STALL_VECTOR_LONG, 0x4004a) +EVENT(PM_L2_RCST_BUSY_RC_FULL, 0x26282) +EVENT(PM_TB_BIT_TRANS, 0x300f8) +EVENT(PM_THERMAL_MAX, 0x40006) +EVENT(PM_LSU1_FLUSH_ULD, 0x0c0b2) +EVENT(PM_LSU1_REJECT_LHS, 0x0c0ae) +EVENT(PM_LSU_LRQ_S0_ALLOC, 0x0d09f) +EVENT(PM_L3_CO_L31, 0x4f080) +EVENT(PM_POWER_EVENT4, 0x4006e) +EVENT(PM_DATA_FROM_L31_SHR, 0x1c04e) +EVENT(PM_BR_UNCOND, 0x0409e) +EVENT(PM_LSU1_DC_PREF_STREAM_ALLOC, 0x0d0aa) +EVENT(PM_PMC4_REWIND, 0x10020) +EVENT(PM_L2_RCLD_DISP, 0x16280) +EVENT(PM_THRD_PRIO_2_3_CYC, 0x040b2) +EVENT(PM_MRK_PTEG_FROM_L2MISS, 0x4d058) +EVENT(PM_IC_DEMAND_L2_BHT_REDIRECT, 0x04098) +EVENT(PM_LSU_DERAT_MISS, 0x200f6) +EVENT(PM_IC_PREF_CANCEL_L2, 0x04094) +EVENT(PM_MRK_FIN_STALL_CYC_COUNT, 0x1003d) +EVENT(PM_BR_PRED_CCACHE, 0x040a0) +EVENT(PM_GCT_UTIL_1_TO_2_SLOTS, 0x0209c) +EVENT(PM_MRK_ST_CMPL_INT, 0x30034) +EVENT(PM_LSU_TWO_TABLEWALK_CYC, 0x0d0a6) +EVENT(PM_MRK_DATA_FROM_L3MISS, 0x2d048) +EVENT(PM_GCT_NOSLOT_CYC, 0x100f8) +EVENT(PM_LSU_SET_MPRED, 0x0c0a8) +EVENT(PM_FLUSH_DISP_TLBIE, 0x0208a) +EVENT(PM_VSU1_FCONV, 0x0a0b2) +EVENT(PM_DERAT_MISS_16G, 0x4c05c) +EVENT(PM_INST_FROM_LMEM, 0x3404a) +EVENT(PM_IC_DEMAND_L2_BR_REDIRECT, 0x0409a) +EVENT(PM_CMPLU_STALL_SCALAR_LONG, 0x20018) +EVENT(PM_INST_PTEG_FROM_L2, 0x1e050) +EVENT(PM_PTEG_FROM_L2, 0x1c050) +EVENT(PM_MRK_DATA_FROM_L21_SHR_CYC, 0x20024) +EVENT(PM_MRK_DTLB_MISS_4K, 0x2d05a) +EVENT(PM_VSU0_FPSCR, 0x0b09c) +EVENT(PM_VSU1_VECT_DOUBLE_ISSUED, 0x0b082) +EVENT(PM_MRK_PTEG_FROM_RL2L3_MOD, 0x1d052) +EVENT(PM_MEM0_RQ_DISP, 0x10083) +EVENT(PM_L2_LD_MISS, 0x26080) +EVENT(PM_VMX_RESULT_SAT_1, 0x0b0a0) +EVENT(PM_L1_PREF, 0x0d8b8) +EVENT(PM_MRK_DATA_FROM_LMEM_CYC, 0x2002c) +EVENT(PM_GRP_IC_MISS_NONSPEC, 0x1000c) +EVENT(PM_PB_NODE_PUMP, 0x10081) +EVENT(PM_SHL_MERGED, 0x05084) +EVENT(PM_NEST_PAIR1_ADD, 0x20881) +EVENT(PM_DATA_FROM_L3, 0x1c048) +EVENT(PM_LSU_FLUSH, 0x0208e) +EVENT(PM_LSU_SRQ_SYNC_COUNT, 0x0d097) +EVENT(PM_PMC2_OVERFLOW, 0x30010) +EVENT(PM_LSU_LDF, 0x0c884) +EVENT(PM_POWER_EVENT3, 0x3006e) +EVENT(PM_DISP_WT, 0x30008) +EVENT(PM_CMPLU_STALL_REJECT, 0x40016) +EVENT(PM_IC_BANK_CONFLICT, 0x04082) +EVENT(PM_BR_MPRED_CR_TA, 0x048ae) +EVENT(PM_L2_INST_MISS, 0x36082) +EVENT(PM_CMPLU_STALL_ERAT_MISS, 0x40018) +EVENT(PM_NEST_PAIR2_ADD, 0x30881) +EVENT(PM_MRK_LSU_FLUSH, 0x0d08c) +EVENT(PM_L2_LDST, 0x16880) +EVENT(PM_INST_FROM_L31_SHR, 0x1404e) +EVENT(PM_VSU0_FIN, 0x0a0bc) +EVENT(PM_LARX_LSU, 0x0c894) +EVENT(PM_INST_FROM_RMEM, 0x34042) +EVENT(PM_DISP_CLB_HELD_TLBIE, 0x02096) +EVENT(PM_MRK_DATA_FROM_DMEM_CYC, 0x2002e) +EVENT(PM_BR_PRED_CR, 0x040a8) +EVENT(PM_LSU_REJECT, 0x10064) +EVENT(PM_GCT_UTIL_3_TO_6_SLOTS, 0x0209e) +EVENT(PM_CMPLU_STALL_END_GCT_NOSLOT, 0x10028) +EVENT(PM_LSU0_REJECT_LMQ_FULL, 0x0c0a4) +EVENT(PM_VSU_FEST, 0x0a8b8) +EVENT(PM_NEST_PAIR0_AND, 0x10883) +EVENT(PM_PTEG_FROM_L3, 0x2c050) +EVENT(PM_POWER_EVENT2, 0x2006e) +EVENT(PM_IC_PREF_CANCEL_PAGE, 0x04090) +EVENT(PM_VSU0_FSQRT_FDIV, 0x0a088) +EVENT(PM_MRK_GRP_CMPL, 0x40030) +EVENT(PM_VSU0_SCAL_DOUBLE_ISSUED, 0x0b088) +EVENT(PM_GRP_DISP, 0x3000a) +EVENT(PM_LSU0_LDX, 0x0c088) +EVENT(PM_DATA_FROM_L2, 0x1c040) +EVENT(PM_MRK_DATA_FROM_RL2L3_MOD, 0x1d042) +EVENT(PM_LD_REF_L1, 0x0c880) +EVENT(PM_VSU0_VECT_DOUBLE_ISSUED, 0x0b080) +EVENT(PM_VSU1_2FLOP_DOUBLE, 0x0a08e) +EVENT(PM_THRD_PRIO_6_7_CYC, 0x040b6) +EVENT(PM_BC_PLUS_8_RSLV_TAKEN, 0x040ba) +EVENT(PM_BR_MPRED_CR, 0x040ac) +EVENT(PM_L3_CO_MEM, 0x4f082) +EVENT(PM_LD_MISS_L1, 0x400f0) +EVENT(PM_DATA_FROM_RL2L3_MOD, 0x1c042) +EVENT(PM_LSU_SRQ_FULL_CYC, 0x1001a) +EVENT(PM_TABLEWALK_CYC, 0x10026) +EVENT(PM_MRK_PTEG_FROM_RMEM, 0x3d052) +EVENT(PM_LSU_SRQ_STFWD, 0x0c8a0) +EVENT(PM_INST_PTEG_FROM_RMEM, 0x3e052) +EVENT(PM_FXU0_FIN, 0x10004) +EVENT(PM_LSU1_L1_SW_PREF, 0x0c09e) +EVENT(PM_PTEG_FROM_L31_MOD, 0x1c054) +EVENT(PM_PMC5_OVERFLOW, 0x10024) +EVENT(PM_LD_REF_L1_LSU1, 0x0c082) +EVENT(PM_INST_PTEG_FROM_L21_SHR, 0x4e056) +EVENT(PM_CMPLU_STALL_THRD, 0x1001c) +EVENT(PM_DATA_FROM_RMEM, 0x3c042) +EVENT(PM_VSU0_SCAL_SINGLE_ISSUED, 0x0b084) +EVENT(PM_BR_MPRED_LSTACK, 0x040a6) +EVENT(PM_MRK_DATA_FROM_RL2L3_MOD_CYC, 0x40028) +EVENT(PM_LSU0_FLUSH_UST, 0x0c0b4) +EVENT(PM_LSU_NCST, 0x0c090) +EVENT(PM_BR_TAKEN, 0x20004) +EVENT(PM_INST_PTEG_FROM_LMEM, 0x4e052) +EVENT(PM_GCT_NOSLOT_BR_MPRED_IC_MISS, 0x4001c) +EVENT(PM_DTLB_MISS_4K, 0x2c05a) +EVENT(PM_PMC4_SAVED, 0x30022) +EVENT(PM_VSU1_PERMUTE_ISSUED, 0x0b092) +EVENT(PM_SLB_MISS, 0x0d890) +EVENT(PM_LSU1_FLUSH_LRQ, 0x0c0ba) +EVENT(PM_DTLB_MISS, 0x300fc) +EVENT(PM_VSU1_FRSP, 0x0a0b6) +EVENT(PM_VSU_VECTOR_DOUBLE_ISSUED, 0x0b880) +EVENT(PM_L2_CASTOUT_SHR, 0x16182) +EVENT(PM_DATA_FROM_DL2L3_SHR, 0x3c044) +EVENT(PM_VSU1_STF, 0x0b08e) +EVENT(PM_ST_FIN, 0x200f0) +EVENT(PM_PTEG_FROM_L21_SHR, 0x4c056) +EVENT(PM_L2_LOC_GUESS_WRONG, 0x26480) +EVENT(PM_MRK_STCX_FAIL, 0x0d08e) +EVENT(PM_LSU0_REJECT_LHS, 0x0c0ac) +EVENT(PM_IC_PREF_CANCEL_HIT, 0x04092) +EVENT(PM_L3_PREF_BUSY, 0x4f080) +EVENT(PM_MRK_BRU_FIN, 0x2003a) +EVENT(PM_LSU1_NCLD, 0x0c08e) +EVENT(PM_INST_PTEG_FROM_L31_MOD, 0x1e054) +EVENT(PM_LSU_NCLD, 0x0c88c) +EVENT(PM_LSU_LDX, 0x0c888) +EVENT(PM_L2_LOC_GUESS_CORRECT, 0x16480) +EVENT(PM_THRESH_TIMEO, 0x10038) +EVENT(PM_L3_PREF_ST, 0x0d0ae) +EVENT(PM_DISP_CLB_HELD_SYNC, 0x02098) +EVENT(PM_VSU_SIMPLE_ISSUED, 0x0b894) +EVENT(PM_VSU1_SINGLE, 0x0a0aa) +EVENT(PM_DATA_TABLEWALK_CYC, 0x3001a) +EVENT(PM_L2_RC_ST_DONE, 0x36380) +EVENT(PM_MRK_PTEG_FROM_L21_MOD, 0x3d056) +EVENT(PM_LARX_LSU1, 0x0c096) +EVENT(PM_MRK_DATA_FROM_RMEM, 0x3d042) +EVENT(PM_DISP_CLB_HELD, 0x02090) +EVENT(PM_DERAT_MISS_4K, 0x1c05c) +EVENT(PM_L2_RCLD_DISP_FAIL_ADDR, 0x16282) +EVENT(PM_SEG_EXCEPTION, 0x028a4) +EVENT(PM_FLUSH_DISP_SB, 0x0208c) +EVENT(PM_L2_DC_INV, 0x26182) +EVENT(PM_PTEG_FROM_DL2L3_MOD, 0x4c054) +EVENT(PM_DSEG, 0x020a6) +EVENT(PM_BR_PRED_LSTACK, 0x040a2) +EVENT(PM_VSU0_STF, 0x0b08c) +EVENT(PM_LSU_FX_FIN, 0x10066) +EVENT(PM_DERAT_MISS_16M, 0x3c05c) +EVENT(PM_MRK_PTEG_FROM_DL2L3_MOD, 0x4d054) +EVENT(PM_GCT_UTIL_11_PLUS_SLOTS, 0x020a2) +EVENT(PM_INST_FROM_L3, 0x14048) +EVENT(PM_MRK_IFU_FIN, 0x3003a) +EVENT(PM_ITLB_MISS, 0x400fc) +EVENT(PM_VSU_STF, 0x0b88c) +EVENT(PM_LSU_FLUSH_UST, 0x0c8b4) +EVENT(PM_L2_LDST_MISS, 0x26880) +EVENT(PM_FXU1_FIN, 0x40004) +EVENT(PM_SHL_DEALLOCATED, 0x05080) +EVENT(PM_L2_SN_M_WR_DONE, 0x46382) +EVENT(PM_LSU_REJECT_SET_MPRED, 0x0c8a8) +EVENT(PM_L3_PREF_LD, 0x0d0ac) +EVENT(PM_L2_SN_M_RD_DONE, 0x46380) +EVENT(PM_MRK_DERAT_MISS_16G, 0x4d05c) +EVENT(PM_VSU_FCONV, 0x0a8b0) +EVENT(PM_ANY_THRD_RUN_CYC, 0x100fa) +EVENT(PM_LSU_LMQ_FULL_CYC, 0x0d0a4) +EVENT(PM_MRK_LSU_REJECT_LHS, 0x0d082) +EVENT(PM_MRK_LD_MISS_L1_CYC, 0x4003e) +EVENT(PM_MRK_DATA_FROM_L2_CYC, 0x20020) +EVENT(PM_INST_IMC_MATCH_DISP, 0x30016) +EVENT(PM_MRK_DATA_FROM_RMEM_CYC, 0x4002c) +EVENT(PM_VSU0_SIMPLE_ISSUED, 0x0b094) +EVENT(PM_CMPLU_STALL_DIV, 0x40014) +EVENT(PM_MRK_PTEG_FROM_RL2L3_SHR, 0x2d054) +EVENT(PM_VSU_FMA_DOUBLE, 0x0a890) +EVENT(PM_VSU_4FLOP, 0x0a89c) +EVENT(PM_VSU1_FIN, 0x0a0be) +EVENT(PM_NEST_PAIR1_AND, 0x20883) +EVENT(PM_INST_PTEG_FROM_RL2L3_MOD, 0x1e052) +EVENT(PM_RUN_CYC, 0x200f4) +EVENT(PM_PTEG_FROM_RMEM, 0x3c052) +EVENT(PM_LSU_LRQ_S0_VALID, 0x0d09e) +EVENT(PM_LSU0_LDF, 0x0c084) +EVENT(PM_FLUSH_COMPLETION, 0x30012) +EVENT(PM_ST_MISS_L1, 0x300f0) +EVENT(PM_L2_NODE_PUMP, 0x36480) +EVENT(PM_INST_FROM_DL2L3_SHR, 0x34044) +EVENT(PM_MRK_STALL_CMPLU_CYC, 0x3003e) +EVENT(PM_VSU1_DENORM, 0x0a0ae) +EVENT(PM_MRK_DATA_FROM_L31_SHR_CYC, 0x20026) +EVENT(PM_NEST_PAIR0_ADD, 0x10881) +EVENT(PM_INST_FROM_L3MISS, 0x24048) +EVENT(PM_EE_OFF_EXT_INT, 0x02080) +EVENT(PM_INST_PTEG_FROM_DMEM, 0x2e052) +EVENT(PM_INST_FROM_DL2L3_MOD, 0x3404c) +EVENT(PM_PMC6_OVERFLOW, 0x30024) +EVENT(PM_VSU_2FLOP_DOUBLE, 0x0a88c) +EVENT(PM_TLB_MISS, 0x20066) +EVENT(PM_FXU_BUSY, 0x2000e) +EVENT(PM_L2_RCLD_DISP_FAIL_OTHER, 0x26280) +EVENT(PM_LSU_REJECT_LMQ_FULL, 0x0c8a4) +EVENT(PM_IC_RELOAD_SHR, 0x04096) +EVENT(PM_GRP_MRK, 0x10031) +EVENT(PM_MRK_ST_NEST, 0x20034) +EVENT(PM_VSU1_FSQRT_FDIV, 0x0a08a) +EVENT(PM_LSU0_FLUSH_LRQ, 0x0c0b8) +EVENT(PM_LARX_LSU0, 0x0c094) +EVENT(PM_IBUF_FULL_CYC, 0x04084) +EVENT(PM_MRK_DATA_FROM_DL2L3_SHR_CYC, 0x2002a) +EVENT(PM_LSU_DC_PREF_STREAM_ALLOC, 0x0d8a8) +EVENT(PM_GRP_MRK_CYC, 0x10030) +EVENT(PM_MRK_DATA_FROM_RL2L3_SHR_CYC, 0x20028) +EVENT(PM_L2_GLOB_GUESS_CORRECT, 0x16482) +EVENT(PM_LSU_REJECT_LHS, 0x0c8ac) +EVENT(PM_MRK_DATA_FROM_LMEM, 0x3d04a) +EVENT(PM_INST_PTEG_FROM_L3, 0x2e050) +EVENT(PM_FREQ_DOWN, 0x3000c) +EVENT(PM_PB_RETRY_NODE_PUMP, 0x30081) +EVENT(PM_INST_FROM_RL2L3_SHR, 0x1404c) +EVENT(PM_MRK_INST_ISSUED, 0x10032) +EVENT(PM_PTEG_FROM_L3MISS, 0x2c058) +EVENT(PM_RUN_PURR, 0x400f4) +EVENT(PM_MRK_GRP_IC_MISS, 0x40038) +EVENT(PM_MRK_DATA_FROM_L3, 0x1d048) +EVENT(PM_CMPLU_STALL_DCACHE_MISS, 0x20016) +EVENT(PM_PTEG_FROM_RL2L3_SHR, 0x2c054) +EVENT(PM_LSU_FLUSH_LRQ, 0x0c8b8) +EVENT(PM_MRK_DERAT_MISS_64K, 0x2d05c) +EVENT(PM_INST_PTEG_FROM_DL2L3_MOD, 0x4e054) +EVENT(PM_L2_ST_MISS, 0x26082) +EVENT(PM_MRK_PTEG_FROM_L21_SHR, 0x4d056) +EVENT(PM_LWSYNC, 0x0d094) +EVENT(PM_LSU0_DC_PREF_STREAM_CONFIRM_STRIDE, 0x0d0bc) +EVENT(PM_MRK_LSU_FLUSH_LRQ, 0x0d088) +EVENT(PM_INST_IMC_MATCH_CMPL, 0x100f0) +EVENT(PM_NEST_PAIR3_AND, 0x40883) +EVENT(PM_PB_RETRY_SYS_PUMP, 0x40081) +EVENT(PM_MRK_INST_FIN, 0x30030) +EVENT(PM_MRK_PTEG_FROM_DL2L3_SHR, 0x3d054) +EVENT(PM_INST_FROM_L31_MOD, 0x14044) +EVENT(PM_MRK_DTLB_MISS_64K, 0x3d05e) +EVENT(PM_LSU_FIN, 0x30066) +EVENT(PM_MRK_LSU_REJECT, 0x40064) +EVENT(PM_L2_CO_FAIL_BUSY, 0x16382) +EVENT(PM_MEM0_WQ_DISP, 0x40083) +EVENT(PM_DATA_FROM_L31_MOD, 0x1c044) +EVENT(PM_THERMAL_WARN, 0x10016) +EVENT(PM_VSU0_4FLOP, 0x0a09c) +EVENT(PM_BR_MPRED_CCACHE, 0x040a4) +EVENT(PM_CMPLU_STALL_IFU, 0x4004c) +EVENT(PM_L1_DEMAND_WRITE, 0x0408c) +EVENT(PM_FLUSH_BR_MPRED, 0x02084) +EVENT(PM_MRK_DTLB_MISS_16G, 0x1d05e) +EVENT(PM_MRK_PTEG_FROM_DMEM, 0x2d052) +EVENT(PM_L2_RCST_DISP, 0x36280) +EVENT(PM_CMPLU_STALL, 0x4000a) +EVENT(PM_LSU_PARTIAL_CDF, 0x0c0aa) +EVENT(PM_DISP_CLB_HELD_SB, 0x020a8) +EVENT(PM_VSU0_FMA_DOUBLE, 0x0a090) +EVENT(PM_FXU0_BUSY_FXU1_IDLE, 0x3000e) +EVENT(PM_IC_DEMAND_CYC, 0x10018) +EVENT(PM_MRK_DATA_FROM_L21_SHR, 0x3d04e) +EVENT(PM_MRK_LSU_FLUSH_UST, 0x0d086) +EVENT(PM_INST_PTEG_FROM_L3MISS, 0x2e058) +EVENT(PM_VSU_DENORM, 0x0a8ac) +EVENT(PM_MRK_LSU_PARTIAL_CDF, 0x0d080) +EVENT(PM_INST_FROM_L21_SHR, 0x3404e) +EVENT(PM_IC_PREF_WRITE, 0x0408e) +EVENT(PM_BR_PRED, 0x0409c) +EVENT(PM_INST_FROM_DMEM, 0x1404a) +EVENT(PM_IC_PREF_CANCEL_ALL, 0x04890) +EVENT(PM_LSU_DC_PREF_STREAM_CONFIRM, 0x0d8b4) +EVENT(PM_MRK_LSU_FLUSH_SRQ, 0x0d08a) +EVENT(PM_MRK_FIN_STALL_CYC, 0x1003c) +EVENT(PM_L2_RCST_DISP_FAIL_OTHER, 0x46280) +EVENT(PM_VSU1_DD_ISSUED, 0x0b098) +EVENT(PM_PTEG_FROM_L31_SHR, 0x2c056) +EVENT(PM_DATA_FROM_L21_SHR, 0x3c04e) +EVENT(PM_LSU0_NCLD, 0x0c08c) +EVENT(PM_VSU1_4FLOP, 0x0a09e) +EVENT(PM_VSU1_8FLOP, 0x0a0a2) +EVENT(PM_VSU_8FLOP, 0x0a8a0) +EVENT(PM_LSU_LMQ_SRQ_EMPTY_CYC, 0x2003e) +EVENT(PM_DTLB_MISS_64K, 0x3c05e) +EVENT(PM_THRD_CONC_RUN_INST, 0x300f4) +EVENT(PM_MRK_PTEG_FROM_L2, 0x1d050) +EVENT(PM_PB_SYS_PUMP, 0x20081) +EVENT(PM_VSU_FIN, 0x0a8bc) +EVENT(PM_MRK_DATA_FROM_L31_MOD, 0x1d044) +EVENT(PM_THRD_PRIO_0_1_CYC, 0x040b0) +EVENT(PM_DERAT_MISS_64K, 0x2c05c) +EVENT(PM_PMC2_REWIND, 0x30020) +EVENT(PM_INST_FROM_L2, 0x14040) +EVENT(PM_GRP_BR_MPRED_NONSPEC, 0x1000a) +EVENT(PM_INST_DISP, 0x200f2) +EVENT(PM_MEM0_RD_CANCEL_TOTAL, 0x30083) +EVENT(PM_LSU0_DC_PREF_STREAM_CONFIRM, 0x0d0b4) +EVENT(PM_L1_DCACHE_RELOAD_VALID, 0x300f6) +EVENT(PM_VSU_SCALAR_DOUBLE_ISSUED, 0x0b888) +EVENT(PM_L3_PREF_HIT, 0x3f080) +EVENT(PM_MRK_PTEG_FROM_L31_MOD, 0x1d054) +EVENT(PM_CMPLU_STALL_STORE, 0x2004a) +EVENT(PM_MRK_FXU_FIN, 0x20038) +EVENT(PM_PMC4_OVERFLOW, 0x10010) +EVENT(PM_MRK_PTEG_FROM_L3, 0x2d050) +EVENT(PM_LSU0_LMQ_LHR_MERGE, 0x0d098) +EVENT(PM_BTAC_HIT, 0x0508a) +EVENT(PM_L3_RD_BUSY, 0x4f082) +EVENT(PM_LSU0_L1_SW_PREF, 0x0c09c) +EVENT(PM_INST_FROM_L2MISS, 0x44048) +EVENT(PM_LSU0_DC_PREF_STREAM_ALLOC, 0x0d0a8) +EVENT(PM_L2_ST, 0x16082) +EVENT(PM_VSU0_DENORM, 0x0a0ac) +EVENT(PM_MRK_DATA_FROM_DL2L3_SHR, 0x3d044) +EVENT(PM_BR_PRED_CR_TA, 0x048aa) +EVENT(PM_VSU0_FCONV, 0x0a0b0) +EVENT(PM_MRK_LSU_FLUSH_ULD, 0x0d084) +EVENT(PM_BTAC_MISS, 0x05088) +EVENT(PM_MRK_LD_MISS_EXPOSED_CYC_COUNT, 0x1003f) +EVENT(PM_MRK_DATA_FROM_L2, 0x1d040) +EVENT(PM_LSU_DCACHE_RELOAD_VALID, 0x0d0a2) +EVENT(PM_VSU_FMA, 0x0a884) +EVENT(PM_LSU0_FLUSH_SRQ, 0x0c0bc) +EVENT(PM_LSU1_L1_PREF, 0x0d0ba) +EVENT(PM_IOPS_CMPL, 0x10014) +EVENT(PM_L2_SYS_PUMP, 0x36482) +EVENT(PM_L2_RCLD_BUSY_RC_FULL, 0x46282) +EVENT(PM_LSU_LMQ_S0_ALLOC, 0x0d0a1) +EVENT(PM_FLUSH_DISP_SYNC, 0x02088) +EVENT(PM_MRK_DATA_FROM_DL2L3_MOD_CYC, 0x4002a) +EVENT(PM_L2_IC_INV, 0x26180) +EVENT(PM_MRK_DATA_FROM_L21_MOD_CYC, 0x40024) +EVENT(PM_L3_PREF_LDST, 0x0d8ac) +EVENT(PM_LSU_SRQ_EMPTY_CYC, 0x40008) +EVENT(PM_LSU_LMQ_S0_VALID, 0x0d0a0) +EVENT(PM_FLUSH_PARTIAL, 0x02086) +EVENT(PM_VSU1_FMA_DOUBLE, 0x0a092) +EVENT(PM_1PLUS_PPC_DISP, 0x400f2) +EVENT(PM_DATA_FROM_L2MISS, 0x200fe) +EVENT(PM_SUSPENDED, 0x00000) +EVENT(PM_VSU0_FMA, 0x0a084) +EVENT(PM_CMPLU_STALL_SCALAR, 0x40012) +EVENT(PM_STCX_FAIL, 0x0c09a) +EVENT(PM_VSU0_FSQRT_FDIV_DOUBLE, 0x0a094) +EVENT(PM_DC_PREF_DST, 0x0d0b0) +EVENT(PM_VSU1_SCAL_SINGLE_ISSUED, 0x0b086) +EVENT(PM_L3_HIT, 0x1f080) +EVENT(PM_L2_GLOB_GUESS_WRONG, 0x26482) +EVENT(PM_MRK_DFU_FIN, 0x20032) +EVENT(PM_INST_FROM_L1, 0x04080) +EVENT(PM_BRU_FIN, 0x10068) +EVENT(PM_IC_DEMAND_REQ, 0x04088) +EVENT(PM_VSU1_FSQRT_FDIV_DOUBLE, 0x0a096) +EVENT(PM_VSU1_FMA, 0x0a086) +EVENT(PM_MRK_LD_MISS_L1, 0x20036) +EVENT(PM_VSU0_2FLOP_DOUBLE, 0x0a08c) +EVENT(PM_LSU_DC_PREF_STRIDED_STREAM_CONFIRM, 0x0d8bc) +EVENT(PM_INST_PTEG_FROM_L31_SHR, 0x2e056) +EVENT(PM_MRK_LSU_REJECT_ERAT_MISS, 0x30064) +EVENT(PM_MRK_DATA_FROM_L2MISS, 0x4d048) +EVENT(PM_DATA_FROM_RL2L3_SHR, 0x1c04c) +EVENT(PM_INST_FROM_PREF, 0x14046) +EVENT(PM_VSU1_SQ, 0x0b09e) +EVENT(PM_L2_LD_DISP, 0x36180) +EVENT(PM_L2_DISP_ALL, 0x46080) +EVENT(PM_THRD_GRP_CMPL_BOTH_CYC, 0x10012) +EVENT(PM_VSU_FSQRT_FDIV_DOUBLE, 0x0a894) +EVENT(PM_BR_MPRED, 0x400f6) +EVENT(PM_INST_PTEG_FROM_DL2L3_SHR, 0x3e054) +EVENT(PM_VSU_1FLOP, 0x0a880) +EVENT(PM_HV_CYC, 0x2000a) +EVENT(PM_MRK_LSU_FIN, 0x40032) +EVENT(PM_MRK_DATA_FROM_RL2L3_SHR, 0x1d04c) +EVENT(PM_DTLB_MISS_16M, 0x4c05e) +EVENT(PM_LSU1_LMQ_LHR_MERGE, 0x0d09a) +EVENT(PM_IFU_FIN, 0x40066) diff --git a/arch/powerpc/perf/power7-pmu.c b/arch/powerpc/perf/power7-pmu.c index d1821b8bbc4c..56c67bca2f75 100644 --- a/arch/powerpc/perf/power7-pmu.c +++ b/arch/powerpc/perf/power7-pmu.c @@ -53,37 +53,13 @@ /* * Power7 event codes. */ -#define PME_PM_CYC 0x1e -#define PME_PM_GCT_NOSLOT_CYC 0x100f8 -#define PME_PM_CMPLU_STALL 0x4000a -#define PME_PM_INST_CMPL 0x2 -#define PME_PM_LD_REF_L1 0xc880 -#define PME_PM_LD_MISS_L1 0x400f0 -#define PME_PM_BRU_FIN 0x10068 -#define PME_PM_BR_MPRED 0x400f6 - -#define PME_PM_CMPLU_STALL_FXU 0x20014 -#define PME_PM_CMPLU_STALL_DIV 0x40014 -#define PME_PM_CMPLU_STALL_SCALAR 0x40012 -#define PME_PM_CMPLU_STALL_SCALAR_LONG 0x20018 -#define PME_PM_CMPLU_STALL_VECTOR 0x2001c -#define PME_PM_CMPLU_STALL_VECTOR_LONG 0x4004a -#define PME_PM_CMPLU_STALL_LSU 0x20012 -#define PME_PM_CMPLU_STALL_REJECT 0x40016 -#define PME_PM_CMPLU_STALL_ERAT_MISS 0x40018 -#define PME_PM_CMPLU_STALL_DCACHE_MISS 0x20016 -#define PME_PM_CMPLU_STALL_STORE 0x2004a -#define PME_PM_CMPLU_STALL_THRD 0x1001c -#define PME_PM_CMPLU_STALL_IFU 0x4004c -#define PME_PM_CMPLU_STALL_BRU 0x4004e -#define PME_PM_GCT_NOSLOT_IC_MISS 0x2001a -#define PME_PM_GCT_NOSLOT_BR_MPRED 0x4001a -#define PME_PM_GCT_NOSLOT_BR_MPRED_IC_MISS 0x4001c -#define PME_PM_GRP_CMPL 0x30004 -#define PME_PM_1PLUS_PPC_CMPL 0x100f2 -#define PME_PM_CMPLU_STALL_DFU 0x2003c -#define PME_PM_RUN_CYC 0x200f4 -#define PME_PM_RUN_INST_CMPL 0x400fa +#define EVENT(_name, _code) \ + PME_##_name = _code, + +enum { +#include "power7-events-list.h" +}; +#undef EVENT /* * Layout of constraint bits: @@ -398,96 +374,36 @@ static int power7_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = { }; -GENERIC_EVENT_ATTR(cpu-cycles, CYC); -GENERIC_EVENT_ATTR(stalled-cycles-frontend, GCT_NOSLOT_CYC); -GENERIC_EVENT_ATTR(stalled-cycles-backend, CMPLU_STALL); -GENERIC_EVENT_ATTR(instructions, INST_CMPL); -GENERIC_EVENT_ATTR(cache-references, LD_REF_L1); -GENERIC_EVENT_ATTR(cache-misses, LD_MISS_L1); -GENERIC_EVENT_ATTR(branch-instructions, BRU_FIN); -GENERIC_EVENT_ATTR(branch-misses, BR_MPRED); - -POWER_EVENT_ATTR(CYC, CYC); -POWER_EVENT_ATTR(GCT_NOSLOT_CYC, GCT_NOSLOT_CYC); -POWER_EVENT_ATTR(CMPLU_STALL, CMPLU_STALL); -POWER_EVENT_ATTR(INST_CMPL, INST_CMPL); -POWER_EVENT_ATTR(LD_REF_L1, LD_REF_L1); -POWER_EVENT_ATTR(LD_MISS_L1, LD_MISS_L1); -POWER_EVENT_ATTR(BRU_FIN, BRU_FIN) -POWER_EVENT_ATTR(BR_MPRED, BR_MPRED); - -POWER_EVENT_ATTR(CMPLU_STALL_FXU, CMPLU_STALL_FXU); -POWER_EVENT_ATTR(CMPLU_STALL_DIV, CMPLU_STALL_DIV); -POWER_EVENT_ATTR(CMPLU_STALL_SCALAR, CMPLU_STALL_SCALAR); -POWER_EVENT_ATTR(CMPLU_STALL_SCALAR_LONG, CMPLU_STALL_SCALAR_LONG); -POWER_EVENT_ATTR(CMPLU_STALL_VECTOR, CMPLU_STALL_VECTOR); -POWER_EVENT_ATTR(CMPLU_STALL_VECTOR_LONG, CMPLU_STALL_VECTOR_LONG); -POWER_EVENT_ATTR(CMPLU_STALL_LSU, CMPLU_STALL_LSU); -POWER_EVENT_ATTR(CMPLU_STALL_REJECT, CMPLU_STALL_REJECT); - -POWER_EVENT_ATTR(CMPLU_STALL_ERAT_MISS, CMPLU_STALL_ERAT_MISS); -POWER_EVENT_ATTR(CMPLU_STALL_DCACHE_MISS, CMPLU_STALL_DCACHE_MISS); -POWER_EVENT_ATTR(CMPLU_STALL_STORE, CMPLU_STALL_STORE); -POWER_EVENT_ATTR(CMPLU_STALL_THRD, CMPLU_STALL_THRD); -POWER_EVENT_ATTR(CMPLU_STALL_IFU, CMPLU_STALL_IFU); -POWER_EVENT_ATTR(CMPLU_STALL_BRU, CMPLU_STALL_BRU); -POWER_EVENT_ATTR(GCT_NOSLOT_IC_MISS, GCT_NOSLOT_IC_MISS); - -POWER_EVENT_ATTR(GCT_NOSLOT_BR_MPRED, GCT_NOSLOT_BR_MPRED); -POWER_EVENT_ATTR(GCT_NOSLOT_BR_MPRED_IC_MISS, GCT_NOSLOT_BR_MPRED_IC_MISS); -POWER_EVENT_ATTR(GRP_CMPL, GRP_CMPL); -POWER_EVENT_ATTR(1PLUS_PPC_CMPL, 1PLUS_PPC_CMPL); -POWER_EVENT_ATTR(CMPLU_STALL_DFU, CMPLU_STALL_DFU); -POWER_EVENT_ATTR(RUN_CYC, RUN_CYC); -POWER_EVENT_ATTR(RUN_INST_CMPL, RUN_INST_CMPL); +GENERIC_EVENT_ATTR(cpu-cycles, PM_CYC); +GENERIC_EVENT_ATTR(stalled-cycles-frontend, PM_GCT_NOSLOT_CYC); +GENERIC_EVENT_ATTR(stalled-cycles-backend, PM_CMPLU_STALL); +GENERIC_EVENT_ATTR(instructions, PM_INST_CMPL); +GENERIC_EVENT_ATTR(cache-references, PM_LD_REF_L1); +GENERIC_EVENT_ATTR(cache-misses, PM_LD_MISS_L1); +GENERIC_EVENT_ATTR(branch-instructions, PM_BRU_FIN); +GENERIC_EVENT_ATTR(branch-misses, PM_BR_MPRED); + +#define EVENT(_name, _code) POWER_EVENT_ATTR(_name, _name); +#include "power7-events-list.h" +#undef EVENT + +#define EVENT(_name, _code) POWER_EVENT_PTR(_name), static struct attribute *power7_events_attr[] = { - GENERIC_EVENT_PTR(CYC), - GENERIC_EVENT_PTR(GCT_NOSLOT_CYC), - GENERIC_EVENT_PTR(CMPLU_STALL), - GENERIC_EVENT_PTR(INST_CMPL), - GENERIC_EVENT_PTR(LD_REF_L1), - GENERIC_EVENT_PTR(LD_MISS_L1), - GENERIC_EVENT_PTR(BRU_FIN), - GENERIC_EVENT_PTR(BR_MPRED), - - POWER_EVENT_PTR(CYC), - POWER_EVENT_PTR(GCT_NOSLOT_CYC), - POWER_EVENT_PTR(CMPLU_STALL), - POWER_EVENT_PTR(INST_CMPL), - POWER_EVENT_PTR(LD_REF_L1), - POWER_EVENT_PTR(LD_MISS_L1), - POWER_EVENT_PTR(BRU_FIN), - POWER_EVENT_PTR(BR_MPRED), - - POWER_EVENT_PTR(CMPLU_STALL_FXU), - POWER_EVENT_PTR(CMPLU_STALL_DIV), - POWER_EVENT_PTR(CMPLU_STALL_SCALAR), - POWER_EVENT_PTR(CMPLU_STALL_SCALAR_LONG), - POWER_EVENT_PTR(CMPLU_STALL_VECTOR), - POWER_EVENT_PTR(CMPLU_STALL_VECTOR_LONG), - POWER_EVENT_PTR(CMPLU_STALL_LSU), - POWER_EVENT_PTR(CMPLU_STALL_REJECT), - - POWER_EVENT_PTR(CMPLU_STALL_ERAT_MISS), - POWER_EVENT_PTR(CMPLU_STALL_DCACHE_MISS), - POWER_EVENT_PTR(CMPLU_STALL_STORE), - POWER_EVENT_PTR(CMPLU_STALL_THRD), - POWER_EVENT_PTR(CMPLU_STALL_IFU), - POWER_EVENT_PTR(CMPLU_STALL_BRU), - POWER_EVENT_PTR(GCT_NOSLOT_IC_MISS), - POWER_EVENT_PTR(GCT_NOSLOT_BR_MPRED), - - POWER_EVENT_PTR(GCT_NOSLOT_BR_MPRED_IC_MISS), - POWER_EVENT_PTR(GRP_CMPL), - POWER_EVENT_PTR(1PLUS_PPC_CMPL), - POWER_EVENT_PTR(CMPLU_STALL_DFU), - POWER_EVENT_PTR(RUN_CYC), - POWER_EVENT_PTR(RUN_INST_CMPL), + GENERIC_EVENT_PTR(PM_CYC), + GENERIC_EVENT_PTR(PM_GCT_NOSLOT_CYC), + GENERIC_EVENT_PTR(PM_CMPLU_STALL), + GENERIC_EVENT_PTR(PM_INST_CMPL), + GENERIC_EVENT_PTR(PM_LD_REF_L1), + GENERIC_EVENT_PTR(PM_LD_MISS_L1), + GENERIC_EVENT_PTR(PM_BRU_FIN), + GENERIC_EVENT_PTR(PM_BR_MPRED), + + #include "power7-events-list.h" + #undef EVENT NULL }; - static struct attribute_group power7_pmu_events_group = { .name = "events", .attrs = power7_events_attr, diff --git a/arch/powerpc/platforms/pseries/nvram.c b/arch/powerpc/platforms/pseries/nvram.c index 6a5f2b1f32ca..d276cd3edd8f 100644 --- a/arch/powerpc/platforms/pseries/nvram.c +++ b/arch/powerpc/platforms/pseries/nvram.c @@ -539,36 +539,6 @@ static int zip_oops(size_t text_len) } #ifdef CONFIG_PSTORE -/* Derived from logfs_uncompress */ -int nvram_decompress(void *in, void *out, size_t inlen, size_t outlen) -{ - int err, ret; - - ret = -EIO; - err = zlib_inflateInit(&stream); - if (err != Z_OK) - goto error; - - stream.next_in = in; - stream.avail_in = inlen; - stream.total_in = 0; - stream.next_out = out; - stream.avail_out = outlen; - stream.total_out = 0; - - err = zlib_inflate(&stream, Z_FINISH); - if (err != Z_STREAM_END) - goto error; - - err = zlib_inflateEnd(&stream); - if (err != Z_OK) - goto error; - - ret = stream.total_out; -error: - return ret; -} - static int nvram_pstore_open(struct pstore_info *psi) { /* Reset the iterator to start reading partitions again */ @@ -584,7 +554,7 @@ static int nvram_pstore_open(struct pstore_info *psi) * @part: pstore writes data to registered buffer in parts, * part number will indicate the same. * @count: Indicates oops count - * @hsize: Size of header added by pstore + * @compressed: Flag to indicate the log is compressed * @size: number of bytes written to the registered buffer * @psi: registered pstore_info structure * @@ -595,7 +565,7 @@ static int nvram_pstore_open(struct pstore_info *psi) static int nvram_pstore_write(enum pstore_type_id type, enum kmsg_dump_reason reason, u64 *id, unsigned int part, int count, - size_t hsize, size_t size, + bool compressed, size_t size, struct pstore_info *psi) { int rc; @@ -611,30 +581,11 @@ static int nvram_pstore_write(enum pstore_type_id type, oops_hdr->report_length = (u16) size; oops_hdr->timestamp = get_seconds(); - if (big_oops_buf) { - rc = zip_oops(size); - /* - * If compression fails copy recent log messages from - * big_oops_buf to oops_data. - */ - if (rc != 0) { - size_t diff = size - oops_data_sz + hsize; - - if (size > oops_data_sz) { - memcpy(oops_data, big_oops_buf, hsize); - memcpy(oops_data + hsize, big_oops_buf + diff, - oops_data_sz - hsize); - - oops_hdr->report_length = (u16) oops_data_sz; - } else - memcpy(oops_data, big_oops_buf, size); - } else - err_type = ERR_TYPE_KERNEL_PANIC_GZ; - } + if (compressed) + err_type = ERR_TYPE_KERNEL_PANIC_GZ; rc = nvram_write_os_partition(&oops_log_partition, oops_buf, - (int) (sizeof(*oops_hdr) + oops_hdr->report_length), err_type, - count); + (int) (sizeof(*oops_hdr) + size), err_type, count); if (rc != 0) return rc; @@ -650,12 +601,12 @@ static int nvram_pstore_write(enum pstore_type_id type, */ static ssize_t nvram_pstore_read(u64 *id, enum pstore_type_id *type, int *count, struct timespec *time, char **buf, - struct pstore_info *psi) + bool *compressed, struct pstore_info *psi) { struct oops_log_info *oops_hdr; unsigned int err_type, id_no, size = 0; struct nvram_os_partition *part = NULL; - char *buff = NULL, *big_buff = NULL; + char *buff = NULL; int sig = 0; loff_t p; @@ -719,8 +670,7 @@ static ssize_t nvram_pstore_read(u64 *id, enum pstore_type_id *type, *id = id_no; if (nvram_type_ids[read_type] == PSTORE_TYPE_DMESG) { - int length, unzipped_len; - size_t hdr_size; + size_t length, hdr_size; oops_hdr = (struct oops_log_info *)buff; if (oops_hdr->version < OOPS_HDR_VERSION) { @@ -741,23 +691,10 @@ static ssize_t nvram_pstore_read(u64 *id, enum pstore_type_id *type, memcpy(*buf, buff + hdr_size, length); kfree(buff); - if (err_type == ERR_TYPE_KERNEL_PANIC_GZ) { - big_buff = kmalloc(big_oops_buf_sz, GFP_KERNEL); - if (!big_buff) - return -ENOMEM; - - unzipped_len = nvram_decompress(*buf, big_buff, - length, big_oops_buf_sz); - - if (unzipped_len < 0) { - pr_err("nvram: decompression failed, returned " - "rc %d\n", unzipped_len); - kfree(big_buff); - } else { - *buf = big_buff; - length = unzipped_len; - } - } + if (err_type == ERR_TYPE_KERNEL_PANIC_GZ) + *compressed = true; + else + *compressed = false; return length; } @@ -777,13 +714,8 @@ static int nvram_pstore_init(void) { int rc = 0; - if (big_oops_buf) { - nvram_pstore_info.buf = big_oops_buf; - nvram_pstore_info.bufsize = big_oops_buf_sz; - } else { - nvram_pstore_info.buf = oops_data; - nvram_pstore_info.bufsize = oops_data_sz; - } + nvram_pstore_info.buf = oops_data; + nvram_pstore_info.bufsize = oops_data_sz; rc = pstore_register(&nvram_pstore_info); if (rc != 0) @@ -802,7 +734,6 @@ static int nvram_pstore_init(void) static void __init nvram_init_oops_partition(int rtas_partition_exists) { int rc; - size_t size; rc = pseries_nvram_init_os_partition(&oops_log_partition); if (rc != 0) { @@ -823,6 +754,11 @@ static void __init nvram_init_oops_partition(int rtas_partition_exists) oops_data = oops_buf + sizeof(struct oops_log_info); oops_data_sz = oops_log_partition.size - sizeof(struct oops_log_info); + rc = nvram_pstore_init(); + + if (!rc) + return; + /* * Figure compression (preceded by elimination of each line's <n> * severity prefix) will reduce the oops/panic report to at most @@ -831,9 +767,8 @@ static void __init nvram_init_oops_partition(int rtas_partition_exists) big_oops_buf_sz = (oops_data_sz * 100) / 45; big_oops_buf = kmalloc(big_oops_buf_sz, GFP_KERNEL); if (big_oops_buf) { - size = max(zlib_deflate_workspacesize(WINDOW_BITS, MEM_LEVEL), - zlib_inflate_workspacesize()); - stream.workspace = kmalloc(size, GFP_KERNEL); + stream.workspace = kmalloc(zlib_deflate_workspacesize( + WINDOW_BITS, MEM_LEVEL), GFP_KERNEL); if (!stream.workspace) { pr_err("nvram: No memory for compression workspace; " "skipping compression of %s partition data\n", @@ -847,11 +782,6 @@ static void __init nvram_init_oops_partition(int rtas_partition_exists) stream.workspace = NULL; } - rc = nvram_pstore_init(); - - if (!rc) - return; - rc = kmsg_dump_register(&nvram_kmsg_dumper); if (rc != 0) { pr_err("nvram: kmsg_dump_register() failed; returned %d\n", rc); diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 8a4cae78f03c..8b7892bf6d8b 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -116,6 +116,7 @@ config S390 select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_TRACER select HAVE_FUNCTION_TRACE_MCOUNT_TEST + select HAVE_GENERIC_HARDIRQS select HAVE_KERNEL_BZIP2 select HAVE_KERNEL_GZIP select HAVE_KERNEL_LZ4 @@ -445,6 +446,16 @@ config PCI_NR_FUNCTIONS This allows you to specify the maximum number of PCI functions which this kernel will support. +config PCI_NR_MSI + int "Maximum number of MSI interrupts (64-32768)" + range 64 32768 + default "256" + help + This defines the number of virtual interrupts the kernel will + provide for MSI interrupts. If you configure your system to have + too few drivers will fail to allocate MSI interrupts for all + PCI devices. + source "drivers/pci/Kconfig" source "drivers/pci/pcie/Kconfig" source "drivers/pci/hotplug/Kconfig" diff --git a/arch/s390/include/asm/airq.h b/arch/s390/include/asm/airq.h index 4066cee0c2d2..4bbb5957ed1b 100644 --- a/arch/s390/include/asm/airq.h +++ b/arch/s390/include/asm/airq.h @@ -9,6 +9,8 @@ #ifndef _ASM_S390_AIRQ_H #define _ASM_S390_AIRQ_H +#include <linux/bit_spinlock.h> + struct airq_struct { struct hlist_node list; /* Handler queueing. */ void (*handler)(struct airq_struct *); /* Thin-interrupt handler */ @@ -23,4 +25,69 @@ struct airq_struct { int register_adapter_interrupt(struct airq_struct *airq); void unregister_adapter_interrupt(struct airq_struct *airq); +/* Adapter interrupt bit vector */ +struct airq_iv { + unsigned long *vector; /* Adapter interrupt bit vector */ + unsigned long *avail; /* Allocation bit mask for the bit vector */ + unsigned long *bitlock; /* Lock bit mask for the bit vector */ + unsigned long *ptr; /* Pointer associated with each bit */ + unsigned int *data; /* 32 bit value associated with each bit */ + unsigned long bits; /* Number of bits in the vector */ + unsigned long end; /* Number of highest allocated bit + 1 */ + spinlock_t lock; /* Lock to protect alloc & free */ +}; + +#define AIRQ_IV_ALLOC 1 /* Use an allocation bit mask */ +#define AIRQ_IV_BITLOCK 2 /* Allocate the lock bit mask */ +#define AIRQ_IV_PTR 4 /* Allocate the ptr array */ +#define AIRQ_IV_DATA 8 /* Allocate the data array */ + +struct airq_iv *airq_iv_create(unsigned long bits, unsigned long flags); +void airq_iv_release(struct airq_iv *iv); +unsigned long airq_iv_alloc_bit(struct airq_iv *iv); +void airq_iv_free_bit(struct airq_iv *iv, unsigned long bit); +unsigned long airq_iv_scan(struct airq_iv *iv, unsigned long start, + unsigned long end); + +static inline unsigned long airq_iv_end(struct airq_iv *iv) +{ + return iv->end; +} + +static inline void airq_iv_lock(struct airq_iv *iv, unsigned long bit) +{ + const unsigned long be_to_le = BITS_PER_LONG - 1; + bit_spin_lock(bit ^ be_to_le, iv->bitlock); +} + +static inline void airq_iv_unlock(struct airq_iv *iv, unsigned long bit) +{ + const unsigned long be_to_le = BITS_PER_LONG - 1; + bit_spin_unlock(bit ^ be_to_le, iv->bitlock); +} + +static inline void airq_iv_set_data(struct airq_iv *iv, unsigned long bit, + unsigned int data) +{ + iv->data[bit] = data; +} + +static inline unsigned int airq_iv_get_data(struct airq_iv *iv, + unsigned long bit) +{ + return iv->data[bit]; +} + +static inline void airq_iv_set_ptr(struct airq_iv *iv, unsigned long bit, + unsigned long ptr) +{ + iv->ptr[bit] = ptr; +} + +static inline unsigned long airq_iv_get_ptr(struct airq_iv *iv, + unsigned long bit) +{ + return iv->ptr[bit]; +} + #endif /* _ASM_S390_AIRQ_H */ diff --git a/arch/s390/include/asm/bitops.h b/arch/s390/include/asm/bitops.h index 7d4676758733..10135a38673c 100644 --- a/arch/s390/include/asm/bitops.h +++ b/arch/s390/include/asm/bitops.h @@ -216,7 +216,7 @@ static inline void __set_bit(unsigned long nr, volatile unsigned long *ptr) addr = (unsigned long) ptr + ((nr ^ (BITS_PER_LONG - 8)) >> 3); asm volatile( " oc %O0(1,%R0),%1" - : "=Q" (*(char *) addr) : "Q" (_oi_bitmap[nr & 7]) : "cc" ); + : "+Q" (*(char *) addr) : "Q" (_oi_bitmap[nr & 7]) : "cc"); } static inline void @@ -244,7 +244,7 @@ __clear_bit(unsigned long nr, volatile unsigned long *ptr) addr = (unsigned long) ptr + ((nr ^ (BITS_PER_LONG - 8)) >> 3); asm volatile( " nc %O0(1,%R0),%1" - : "=Q" (*(char *) addr) : "Q" (_ni_bitmap[nr & 7]) : "cc" ); + : "+Q" (*(char *) addr) : "Q" (_ni_bitmap[nr & 7]) : "cc"); } static inline void @@ -271,7 +271,7 @@ static inline void __change_bit(unsigned long nr, volatile unsigned long *ptr) addr = (unsigned long) ptr + ((nr ^ (BITS_PER_LONG - 8)) >> 3); asm volatile( " xc %O0(1,%R0),%1" - : "=Q" (*(char *) addr) : "Q" (_oi_bitmap[nr & 7]) : "cc" ); + : "+Q" (*(char *) addr) : "Q" (_oi_bitmap[nr & 7]) : "cc"); } static inline void @@ -301,7 +301,7 @@ test_and_set_bit_simple(unsigned long nr, volatile unsigned long *ptr) ch = *(unsigned char *) addr; asm volatile( " oc %O0(1,%R0),%1" - : "=Q" (*(char *) addr) : "Q" (_oi_bitmap[nr & 7]) + : "+Q" (*(char *) addr) : "Q" (_oi_bitmap[nr & 7]) : "cc", "memory"); return (ch >> (nr & 7)) & 1; } @@ -320,7 +320,7 @@ test_and_clear_bit_simple(unsigned long nr, volatile unsigned long *ptr) ch = *(unsigned char *) addr; asm volatile( " nc %O0(1,%R0),%1" - : "=Q" (*(char *) addr) : "Q" (_ni_bitmap[nr & 7]) + : "+Q" (*(char *) addr) : "Q" (_ni_bitmap[nr & 7]) : "cc", "memory"); return (ch >> (nr & 7)) & 1; } @@ -339,7 +339,7 @@ test_and_change_bit_simple(unsigned long nr, volatile unsigned long *ptr) ch = *(unsigned char *) addr; asm volatile( " xc %O0(1,%R0),%1" - : "=Q" (*(char *) addr) : "Q" (_oi_bitmap[nr & 7]) + : "+Q" (*(char *) addr) : "Q" (_oi_bitmap[nr & 7]) : "cc", "memory"); return (ch >> (nr & 7)) & 1; } diff --git a/arch/s390/include/asm/cio.h b/arch/s390/include/asm/cio.h index ffb898961c8d..d42625053c37 100644 --- a/arch/s390/include/asm/cio.h +++ b/arch/s390/include/asm/cio.h @@ -296,6 +296,7 @@ static inline int ccw_dev_id_is_equal(struct ccw_dev_id *dev_id1, return 0; } +void channel_subsystem_reinit(void); extern void css_schedule_reprobe(void); extern void reipl_ccw_dev(struct ccw_dev_id *id); diff --git a/arch/s390/include/asm/hardirq.h b/arch/s390/include/asm/hardirq.h index 0c82ba86e997..a908d2941c5d 100644 --- a/arch/s390/include/asm/hardirq.h +++ b/arch/s390/include/asm/hardirq.h @@ -20,4 +20,9 @@ #define HARDIRQ_BITS 8 +static inline void ack_bad_irq(unsigned int irq) +{ + printk(KERN_CRIT "unexpected IRQ trap at vector %02x\n", irq); +} + #endif /* __ASM_HARDIRQ_H */ diff --git a/arch/s390/include/asm/hugetlb.h b/arch/s390/include/asm/hugetlb.h index bd90359d6d22..11eae5f55b70 100644 --- a/arch/s390/include/asm/hugetlb.h +++ b/arch/s390/include/asm/hugetlb.h @@ -17,6 +17,9 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte); +pte_t huge_ptep_get(pte_t *ptep); +pte_t huge_ptep_get_and_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep); /* * If the arch doesn't supply something else, assume that hugepage @@ -38,147 +41,75 @@ static inline int prepare_hugepage_range(struct file *file, int arch_prepare_hugepage(struct page *page); void arch_release_hugepage(struct page *page); -static inline pte_t huge_pte_wrprotect(pte_t pte) +static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr, + pte_t *ptep) { - pte_val(pte) |= _PAGE_RO; - return pte; + pte_val(*ptep) = _SEGMENT_ENTRY_EMPTY; } -static inline int huge_pte_none(pte_t pte) +static inline void huge_ptep_clear_flush(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep) { - return (pte_val(pte) & _SEGMENT_ENTRY_INV) && - !(pte_val(pte) & _SEGMENT_ENTRY_RO); + huge_ptep_get_and_clear(vma->vm_mm, address, ptep); } -static inline pte_t huge_ptep_get(pte_t *ptep) +static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t pte, int dirty) { - pte_t pte = *ptep; - unsigned long mask; - - if (!MACHINE_HAS_HPAGE) { - ptep = (pte_t *) (pte_val(pte) & _SEGMENT_ENTRY_ORIGIN); - if (ptep) { - mask = pte_val(pte) & - (_SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO); - pte = pte_mkhuge(*ptep); - pte_val(pte) |= mask; - } + int changed = !pte_same(huge_ptep_get(ptep), pte); + if (changed) { + huge_ptep_get_and_clear(vma->vm_mm, addr, ptep); + set_huge_pte_at(vma->vm_mm, addr, ptep, pte); } - return pte; + return changed; } -static inline void __pmd_csp(pmd_t *pmdp) +static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) { - register unsigned long reg2 asm("2") = pmd_val(*pmdp); - register unsigned long reg3 asm("3") = pmd_val(*pmdp) | - _SEGMENT_ENTRY_INV; - register unsigned long reg4 asm("4") = ((unsigned long) pmdp) + 5; - - asm volatile( - " csp %1,%3" - : "=m" (*pmdp) - : "d" (reg2), "d" (reg3), "d" (reg4), "m" (*pmdp) : "cc"); + pte_t pte = huge_ptep_get_and_clear(mm, addr, ptep); + set_huge_pte_at(mm, addr, ptep, pte_wrprotect(pte)); } -static inline void huge_ptep_invalidate(struct mm_struct *mm, - unsigned long address, pte_t *ptep) -{ - pmd_t *pmdp = (pmd_t *) ptep; - - if (MACHINE_HAS_IDTE) - __pmd_idte(address, pmdp); - else - __pmd_csp(pmdp); - pmd_val(*pmdp) = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY; -} - -static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm, - unsigned long addr, pte_t *ptep) -{ - pte_t pte = huge_ptep_get(ptep); - - huge_ptep_invalidate(mm, addr, ptep); - return pte; -} - -#define huge_ptep_set_access_flags(__vma, __addr, __ptep, __entry, __dirty) \ -({ \ - int __changed = !pte_same(huge_ptep_get(__ptep), __entry); \ - if (__changed) { \ - huge_ptep_invalidate((__vma)->vm_mm, __addr, __ptep); \ - set_huge_pte_at((__vma)->vm_mm, __addr, __ptep, __entry); \ - } \ - __changed; \ -}) - -#define huge_ptep_set_wrprotect(__mm, __addr, __ptep) \ -({ \ - pte_t __pte = huge_ptep_get(__ptep); \ - if (huge_pte_write(__pte)) { \ - huge_ptep_invalidate(__mm, __addr, __ptep); \ - set_huge_pte_at(__mm, __addr, __ptep, \ - huge_pte_wrprotect(__pte)); \ - } \ -}) - -static inline void huge_ptep_clear_flush(struct vm_area_struct *vma, - unsigned long address, pte_t *ptep) +static inline pte_t mk_huge_pte(struct page *page, pgprot_t pgprot) { - huge_ptep_invalidate(vma->vm_mm, address, ptep); + return mk_pte(page, pgprot); } -static inline pte_t mk_huge_pte(struct page *page, pgprot_t pgprot) +static inline int huge_pte_none(pte_t pte) { - pte_t pte; - pmd_t pmd; - - pmd = mk_pmd_phys(page_to_phys(page), pgprot); - pte_val(pte) = pmd_val(pmd); - return pte; + return pte_none(pte); } static inline int huge_pte_write(pte_t pte) { - pmd_t pmd; - - pmd_val(pmd) = pte_val(pte); - return pmd_write(pmd); + return pte_write(pte); } static inline int huge_pte_dirty(pte_t pte) { - /* No dirty bit in the segment table entry. */ - return 0; + return pte_dirty(pte); } static inline pte_t huge_pte_mkwrite(pte_t pte) { - pmd_t pmd; - - pmd_val(pmd) = pte_val(pte); - pte_val(pte) = pmd_val(pmd_mkwrite(pmd)); - return pte; + return pte_mkwrite(pte); } static inline pte_t huge_pte_mkdirty(pte_t pte) { - /* No dirty bit in the segment table entry. */ - return pte; + return pte_mkdirty(pte); } -static inline pte_t huge_pte_modify(pte_t pte, pgprot_t newprot) +static inline pte_t huge_pte_wrprotect(pte_t pte) { - pmd_t pmd; - - pmd_val(pmd) = pte_val(pte); - pte_val(pte) = pmd_val(pmd_modify(pmd, newprot)); - return pte; + return pte_wrprotect(pte); } -static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr, - pte_t *ptep) +static inline pte_t huge_pte_modify(pte_t pte, pgprot_t newprot) { - pmd_clear((pmd_t *) ptep); + return pte_modify(pte, newprot); } #endif /* _ASM_S390_HUGETLB_H */ diff --git a/arch/s390/include/asm/hw_irq.h b/arch/s390/include/asm/hw_irq.h index 7e3d2586c1ff..ee96a8b697f9 100644 --- a/arch/s390/include/asm/hw_irq.h +++ b/arch/s390/include/asm/hw_irq.h @@ -4,19 +4,8 @@ #include <linux/msi.h> #include <linux/pci.h> -static inline struct msi_desc *irq_get_msi_desc(unsigned int irq) -{ - return __irq_get_msi_desc(irq); -} - -/* Must be called with msi map lock held */ -static inline int irq_set_msi_desc(unsigned int irq, struct msi_desc *msi) -{ - if (!msi) - return -EINVAL; - - msi->irq = irq; - return 0; -} +void __init init_airq_interrupts(void); +void __init init_cio_interrupts(void); +void __init init_ext_interrupts(void); #endif diff --git a/arch/s390/include/asm/irq.h b/arch/s390/include/asm/irq.h index 87c17bfb2968..1eaa3625803c 100644 --- a/arch/s390/include/asm/irq.h +++ b/arch/s390/include/asm/irq.h @@ -1,17 +1,28 @@ #ifndef _ASM_IRQ_H #define _ASM_IRQ_H +#define EXT_INTERRUPT 1 +#define IO_INTERRUPT 2 +#define THIN_INTERRUPT 3 + +#define NR_IRQS_BASE 4 + +#ifdef CONFIG_PCI_NR_MSI +# define NR_IRQS (NR_IRQS_BASE + CONFIG_PCI_NR_MSI) +#else +# define NR_IRQS NR_IRQS_BASE +#endif + +/* This number is used when no interrupt has been assigned */ +#define NO_IRQ 0 + +#ifndef __ASSEMBLY__ + #include <linux/hardirq.h> #include <linux/percpu.h> #include <linux/cache.h> #include <linux/types.h> -enum interruption_main_class { - EXTERNAL_INTERRUPT, - IO_INTERRUPT, - NR_IRQS -}; - enum interruption_class { IRQEXT_CLK, IRQEXT_EXC, @@ -72,14 +83,8 @@ void service_subclass_irq_unregister(void); void measurement_alert_subclass_register(void); void measurement_alert_subclass_unregister(void); -#ifdef CONFIG_LOCKDEP -# define disable_irq_nosync_lockdep(irq) disable_irq_nosync(irq) -# define disable_irq_nosync_lockdep_irqsave(irq, flags) \ - disable_irq_nosync(irq) -# define disable_irq_lockdep(irq) disable_irq(irq) -# define enable_irq_lockdep(irq) enable_irq(irq) -# define enable_irq_lockdep_irqrestore(irq, flags) \ - enable_irq(irq) -#endif +#define irq_canonicalize(irq) (irq) + +#endif /* __ASSEMBLY__ */ #endif /* _ASM_IRQ_H */ diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index 084e7755ed9b..7b7fce4e8469 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -77,8 +77,7 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, WARN_ON(atomic_read(&prev->context.attach_count) < 0); atomic_inc(&next->context.attach_count); /* Check for TLBs not flushed yet */ - if (next->context.flush_mm) - __tlb_flush_mm(next); + __tlb_flush_mm_lazy(next); } #define enter_lazy_tlb(mm,tsk) do { } while (0) diff --git a/arch/s390/include/asm/page.h b/arch/s390/include/asm/page.h index 5d64fb7619cc..1e51f2915b2e 100644 --- a/arch/s390/include/asm/page.h +++ b/arch/s390/include/asm/page.h @@ -32,16 +32,6 @@ void storage_key_init_range(unsigned long start, unsigned long end); -static inline unsigned long pfmf(unsigned long function, unsigned long address) -{ - asm volatile( - " .insn rre,0xb9af0000,%[function],%[address]" - : [address] "+a" (address) - : [function] "d" (function) - : "memory"); - return address; -} - static inline void clear_page(void *page) { register unsigned long reg1 asm ("1") = 0; @@ -150,15 +140,6 @@ static inline int page_reset_referenced(unsigned long addr) #define _PAGE_FP_BIT 0x08 /* HW fetch protection bit */ #define _PAGE_ACC_BITS 0xf0 /* HW access control bits */ -/* - * Test and clear referenced bit in storage key. - */ -#define __HAVE_ARCH_PAGE_TEST_AND_CLEAR_YOUNG -static inline int page_test_and_clear_young(unsigned long pfn) -{ - return page_reset_referenced(pfn << PAGE_SHIFT); -} - struct page; void arch_free_page(struct page *page, int order); void arch_alloc_page(struct page *page, int order); diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h index 6e577ba0e5da..c290f13d1c47 100644 --- a/arch/s390/include/asm/pci.h +++ b/arch/s390/include/asm/pci.h @@ -6,6 +6,7 @@ /* must be set before including pci_clp.h */ #define PCI_BAR_COUNT 6 +#include <linux/pci.h> #include <asm-generic/pci.h> #include <asm-generic/pci-dma-compat.h> #include <asm/pci_clp.h> @@ -53,14 +54,9 @@ struct zpci_fmb { atomic64_t unmapped_pages; } __packed __aligned(16); -struct msi_map { - unsigned long irq; - struct msi_desc *msi; - struct hlist_node msi_chain; -}; - -#define ZPCI_NR_MSI_VECS 64 -#define ZPCI_MSI_MASK (ZPCI_NR_MSI_VECS - 1) +#define ZPCI_MSI_VEC_BITS 11 +#define ZPCI_MSI_VEC_MAX (1 << ZPCI_MSI_VEC_BITS) +#define ZPCI_MSI_VEC_MASK (ZPCI_MSI_VEC_MAX - 1) enum zpci_state { ZPCI_FN_STATE_RESERVED, @@ -91,8 +87,7 @@ struct zpci_dev { /* IRQ stuff */ u64 msi_addr; /* MSI address */ - struct zdev_irq_map *irq_map; - struct msi_map *msi_map[ZPCI_NR_MSI_VECS]; + struct airq_iv *aibv; /* adapter interrupt bit vector */ unsigned int aisb; /* number of the summary bit */ /* DMA stuff */ @@ -122,11 +117,6 @@ struct zpci_dev { struct dentry *debugfs_perf; }; -struct pci_hp_callback_ops { - int (*create_slot) (struct zpci_dev *zdev); - void (*remove_slot) (struct zpci_dev *zdev); -}; - static inline bool zdev_enabled(struct zpci_dev *zdev) { return (zdev->fh & (1UL << 31)) ? true : false; @@ -146,32 +136,38 @@ int zpci_register_ioat(struct zpci_dev *, u8, u64, u64, u64); int zpci_unregister_ioat(struct zpci_dev *, u8); /* CLP */ -int clp_find_pci_devices(void); +int clp_scan_pci_devices(void); +int clp_rescan_pci_devices(void); +int clp_rescan_pci_devices_simple(void); int clp_add_pci_device(u32, u32, int); int clp_enable_fh(struct zpci_dev *, u8); int clp_disable_fh(struct zpci_dev *); -/* MSI */ -struct msi_desc *__irq_get_msi_desc(unsigned int); -int zpci_msi_set_mask_bits(struct msi_desc *, u32, u32); -int zpci_setup_msi_irq(struct zpci_dev *, struct msi_desc *, unsigned int, int); -void zpci_teardown_msi_irq(struct zpci_dev *, struct msi_desc *); -int zpci_msihash_init(void); -void zpci_msihash_exit(void); - #ifdef CONFIG_PCI /* Error handling and recovery */ void zpci_event_error(void *); void zpci_event_availability(void *); +void zpci_rescan(void); #else /* CONFIG_PCI */ static inline void zpci_event_error(void *e) {} static inline void zpci_event_availability(void *e) {} +static inline void zpci_rescan(void) {} #endif /* CONFIG_PCI */ +#ifdef CONFIG_HOTPLUG_PCI_S390 +int zpci_init_slot(struct zpci_dev *); +void zpci_exit_slot(struct zpci_dev *); +#else /* CONFIG_HOTPLUG_PCI_S390 */ +static inline int zpci_init_slot(struct zpci_dev *zdev) +{ + return 0; +} +static inline void zpci_exit_slot(struct zpci_dev *zdev) {} +#endif /* CONFIG_HOTPLUG_PCI_S390 */ + /* Helpers */ struct zpci_dev *get_zdev(struct pci_dev *); struct zpci_dev *get_zdev_by_fid(u32); -bool zpci_fid_present(u32); /* sysfs */ int zpci_sysfs_add_device(struct device *); @@ -181,14 +177,6 @@ void zpci_sysfs_remove_device(struct device *); int zpci_dma_init(void); void zpci_dma_exit(void); -/* Hotplug */ -extern struct mutex zpci_list_lock; -extern struct list_head zpci_list; -extern unsigned int s390_pci_probe; - -void zpci_register_hp_ops(struct pci_hp_callback_ops *); -void zpci_deregister_hp_ops(void); - /* FMB */ int zpci_fmb_enable_device(struct zpci_dev *); int zpci_fmb_disable_device(struct zpci_dev *); diff --git a/arch/s390/include/asm/pci_insn.h b/arch/s390/include/asm/pci_insn.h index e6a2bdd4d705..df6eac9f0cb4 100644 --- a/arch/s390/include/asm/pci_insn.h +++ b/arch/s390/include/asm/pci_insn.h @@ -79,11 +79,11 @@ struct zpci_fib { } __packed; -int s390pci_mod_fc(u64 req, struct zpci_fib *fib); -int s390pci_refresh_trans(u64 fn, u64 addr, u64 range); -int s390pci_load(u64 *data, u64 req, u64 offset); -int s390pci_store(u64 data, u64 req, u64 offset); -int s390pci_store_block(const u64 *data, u64 req, u64 offset); -void set_irq_ctrl(u16 ctl, char *unused, u8 isc); +int zpci_mod_fc(u64 req, struct zpci_fib *fib); +int zpci_refresh_trans(u64 fn, u64 addr, u64 range); +int zpci_load(u64 *data, u64 req, u64 offset); +int zpci_store(u64 data, u64 req, u64 offset); +int zpci_store_block(const u64 *data, u64 req, u64 offset); +void zpci_set_irq_ctrl(u16 ctl, char *unused, u8 isc); #endif diff --git a/arch/s390/include/asm/pci_io.h b/arch/s390/include/asm/pci_io.h index 83a9caa6ae53..d194d544d694 100644 --- a/arch/s390/include/asm/pci_io.h +++ b/arch/s390/include/asm/pci_io.h @@ -36,7 +36,7 @@ static inline RETTYPE zpci_read_##RETTYPE(const volatile void __iomem *addr) \ u64 data; \ int rc; \ \ - rc = s390pci_load(&data, req, ZPCI_OFFSET(addr)); \ + rc = zpci_load(&data, req, ZPCI_OFFSET(addr)); \ if (rc) \ data = -1ULL; \ return (RETTYPE) data; \ @@ -50,7 +50,7 @@ static inline void zpci_write_##VALTYPE(VALTYPE val, \ u64 req = ZPCI_CREATE_REQ(entry->fh, entry->bar, LENGTH); \ u64 data = (VALTYPE) val; \ \ - s390pci_store(data, req, ZPCI_OFFSET(addr)); \ + zpci_store(data, req, ZPCI_OFFSET(addr)); \ } zpci_read(8, u64) @@ -83,7 +83,7 @@ static inline int zpci_write_single(u64 req, const u64 *data, u64 offset, u8 len val = 0; /* let FW report error */ break; } - return s390pci_store(val, req, offset); + return zpci_store(val, req, offset); } static inline int zpci_read_single(u64 req, u64 *dst, u64 offset, u8 len) @@ -91,7 +91,7 @@ static inline int zpci_read_single(u64 req, u64 *dst, u64 offset, u8 len) u64 data; int cc; - cc = s390pci_load(&data, req, offset); + cc = zpci_load(&data, req, offset); if (cc) goto out; @@ -115,7 +115,7 @@ out: static inline int zpci_write_block(u64 req, const u64 *data, u64 offset) { - return s390pci_store_block(data, req, offset); + return zpci_store_block(data, req, offset); } static inline u8 zpci_get_max_write_size(u64 src, u64 dst, int len, int max) diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 75fb726de91f..9f215b40109e 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -217,63 +217,57 @@ extern unsigned long MODULES_END; /* Hardware bits in the page table entry */ #define _PAGE_CO 0x100 /* HW Change-bit override */ -#define _PAGE_RO 0x200 /* HW read-only bit */ +#define _PAGE_PROTECT 0x200 /* HW read-only bit */ #define _PAGE_INVALID 0x400 /* HW invalid bit */ +#define _PAGE_LARGE 0x800 /* Bit to mark a large pte */ /* Software bits in the page table entry */ -#define _PAGE_SWT 0x001 /* SW pte type bit t */ -#define _PAGE_SWX 0x002 /* SW pte type bit x */ -#define _PAGE_SWC 0x004 /* SW pte changed bit */ -#define _PAGE_SWR 0x008 /* SW pte referenced bit */ -#define _PAGE_SWW 0x010 /* SW pte write bit */ -#define _PAGE_SPECIAL 0x020 /* SW associated with special page */ +#define _PAGE_PRESENT 0x001 /* SW pte present bit */ +#define _PAGE_TYPE 0x002 /* SW pte type bit */ +#define _PAGE_YOUNG 0x004 /* SW pte young bit */ +#define _PAGE_DIRTY 0x008 /* SW pte dirty bit */ +#define _PAGE_READ 0x010 /* SW pte read bit */ +#define _PAGE_WRITE 0x020 /* SW pte write bit */ +#define _PAGE_SPECIAL 0x040 /* SW associated with special page */ #define __HAVE_ARCH_PTE_SPECIAL /* Set of bits not changed in pte_modify */ #define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_SPECIAL | _PAGE_CO | \ - _PAGE_SWC | _PAGE_SWR) - -/* Six different types of pages. */ -#define _PAGE_TYPE_EMPTY 0x400 -#define _PAGE_TYPE_NONE 0x401 -#define _PAGE_TYPE_SWAP 0x403 -#define _PAGE_TYPE_FILE 0x601 /* bit 0x002 is used for offset !! */ -#define _PAGE_TYPE_RO 0x200 -#define _PAGE_TYPE_RW 0x000 + _PAGE_DIRTY | _PAGE_YOUNG) /* - * Only four types for huge pages, using the invalid bit and protection bit - * of a segment table entry. - */ -#define _HPAGE_TYPE_EMPTY 0x020 /* _SEGMENT_ENTRY_INV */ -#define _HPAGE_TYPE_NONE 0x220 -#define _HPAGE_TYPE_RO 0x200 /* _SEGMENT_ENTRY_RO */ -#define _HPAGE_TYPE_RW 0x000 - -/* - * PTE type bits are rather complicated. handle_pte_fault uses pte_present, - * pte_none and pte_file to find out the pte type WITHOUT holding the page - * table lock. ptep_clear_flush on the other hand uses ptep_clear_flush to - * invalidate a given pte. ipte sets the hw invalid bit and clears all tlbs - * for the page. The page table entry is set to _PAGE_TYPE_EMPTY afterwards. - * This change is done while holding the lock, but the intermediate step - * of a previously valid pte with the hw invalid bit set can be observed by - * handle_pte_fault. That makes it necessary that all valid pte types with - * the hw invalid bit set must be distinguishable from the four pte types - * empty, none, swap and file. + * handle_pte_fault uses pte_present, pte_none and pte_file to find out the + * pte type WITHOUT holding the page table lock. The _PAGE_PRESENT bit + * is used to distinguish present from not-present ptes. It is changed only + * with the page table lock held. + * + * The following table gives the different possible bit combinations for + * the pte hardware and software bits in the last 12 bits of a pte: * - * irxt ipte irxt - * _PAGE_TYPE_EMPTY 1000 -> 1000 - * _PAGE_TYPE_NONE 1001 -> 1001 - * _PAGE_TYPE_SWAP 1011 -> 1011 - * _PAGE_TYPE_FILE 11?1 -> 11?1 - * _PAGE_TYPE_RO 0100 -> 1100 - * _PAGE_TYPE_RW 0000 -> 1000 + * 842100000000 + * 000084210000 + * 000000008421 + * .IR...wrdytp + * empty .10...000000 + * swap .10...xxxx10 + * file .11...xxxxx0 + * prot-none, clean, old .11...000001 + * prot-none, clean, young .11...000101 + * prot-none, dirty, old .10...001001 + * prot-none, dirty, young .10...001101 + * read-only, clean, old .11...010001 + * read-only, clean, young .01...010101 + * read-only, dirty, old .11...011001 + * read-only, dirty, young .01...011101 + * read-write, clean, old .11...110001 + * read-write, clean, young .01...110101 + * read-write, dirty, old .10...111001 + * read-write, dirty, young .00...111101 * - * pte_none is true for bits combinations 1000, 1010, 1100, 1110 - * pte_present is true for bits combinations 0000, 0010, 0100, 0110, 1001 - * pte_file is true for bits combinations 1101, 1111 - * swap pte is 1011 and 0001, 0011, 0101, 0111 are invalid. + * pte_present is true for the bit pattern .xx...xxxxx1, (pte & 0x001) == 0x001 + * pte_none is true for the bit pattern .10...xxxx00, (pte & 0x603) == 0x400 + * pte_file is true for the bit pattern .11...xxxxx0, (pte & 0x601) == 0x600 + * pte_swap is true for the bit pattern .10...xxxx10, (pte & 0x603) == 0x402 */ #ifndef CONFIG_64BIT @@ -286,14 +280,25 @@ extern unsigned long MODULES_END; #define _ASCE_TABLE_LENGTH 0x7f /* 128 x 64 entries = 8k */ /* Bits in the segment table entry */ +#define _SEGMENT_ENTRY_BITS 0x7fffffffUL /* Valid segment table bits */ #define _SEGMENT_ENTRY_ORIGIN 0x7fffffc0UL /* page table origin */ -#define _SEGMENT_ENTRY_RO 0x200 /* page protection bit */ -#define _SEGMENT_ENTRY_INV 0x20 /* invalid segment table entry */ +#define _SEGMENT_ENTRY_PROTECT 0x200 /* page protection bit */ +#define _SEGMENT_ENTRY_INVALID 0x20 /* invalid segment table entry */ #define _SEGMENT_ENTRY_COMMON 0x10 /* common segment bit */ #define _SEGMENT_ENTRY_PTL 0x0f /* page table length */ +#define _SEGMENT_ENTRY_NONE _SEGMENT_ENTRY_PROTECT #define _SEGMENT_ENTRY (_SEGMENT_ENTRY_PTL) -#define _SEGMENT_ENTRY_EMPTY (_SEGMENT_ENTRY_INV) +#define _SEGMENT_ENTRY_EMPTY (_SEGMENT_ENTRY_INVALID) + +/* + * Segment table entry encoding (I = invalid, R = read-only bit): + * ..R...I..... + * prot-none ..1...1..... + * read-only ..1...0..... + * read-write ..0...0..... + * empty ..0...1..... + */ /* Page status table bits for virtualization */ #define PGSTE_ACC_BITS 0xf0000000UL @@ -303,9 +308,7 @@ extern unsigned long MODULES_END; #define PGSTE_HC_BIT 0x00200000UL #define PGSTE_GR_BIT 0x00040000UL #define PGSTE_GC_BIT 0x00020000UL -#define PGSTE_UR_BIT 0x00008000UL -#define PGSTE_UC_BIT 0x00004000UL /* user dirty (migration) */ -#define PGSTE_IN_BIT 0x00002000UL /* IPTE notify bit */ +#define PGSTE_IN_BIT 0x00008000UL /* IPTE notify bit */ #else /* CONFIG_64BIT */ @@ -324,8 +327,8 @@ extern unsigned long MODULES_END; /* Bits in the region table entry */ #define _REGION_ENTRY_ORIGIN ~0xfffUL/* region/segment table origin */ -#define _REGION_ENTRY_RO 0x200 /* region protection bit */ -#define _REGION_ENTRY_INV 0x20 /* invalid region table entry */ +#define _REGION_ENTRY_PROTECT 0x200 /* region protection bit */ +#define _REGION_ENTRY_INVALID 0x20 /* invalid region table entry */ #define _REGION_ENTRY_TYPE_MASK 0x0c /* region/segment table type mask */ #define _REGION_ENTRY_TYPE_R1 0x0c /* region first table type */ #define _REGION_ENTRY_TYPE_R2 0x08 /* region second table type */ @@ -333,29 +336,47 @@ extern unsigned long MODULES_END; #define _REGION_ENTRY_LENGTH 0x03 /* region third length */ #define _REGION1_ENTRY (_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_LENGTH) -#define _REGION1_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_INV) +#define _REGION1_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_INVALID) #define _REGION2_ENTRY (_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_LENGTH) -#define _REGION2_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_INV) +#define _REGION2_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_INVALID) #define _REGION3_ENTRY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_LENGTH) -#define _REGION3_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_INV) +#define _REGION3_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_INVALID) #define _REGION3_ENTRY_LARGE 0x400 /* RTTE-format control, large page */ #define _REGION3_ENTRY_RO 0x200 /* page protection bit */ #define _REGION3_ENTRY_CO 0x100 /* change-recording override */ /* Bits in the segment table entry */ +#define _SEGMENT_ENTRY_BITS 0xfffffffffffffe33UL +#define _SEGMENT_ENTRY_BITS_LARGE 0xfffffffffff1ff33UL #define _SEGMENT_ENTRY_ORIGIN_LARGE ~0xfffffUL /* large page address */ #define _SEGMENT_ENTRY_ORIGIN ~0x7ffUL/* segment table origin */ -#define _SEGMENT_ENTRY_RO 0x200 /* page protection bit */ -#define _SEGMENT_ENTRY_INV 0x20 /* invalid segment table entry */ +#define _SEGMENT_ENTRY_PROTECT 0x200 /* page protection bit */ +#define _SEGMENT_ENTRY_INVALID 0x20 /* invalid segment table entry */ #define _SEGMENT_ENTRY (0) -#define _SEGMENT_ENTRY_EMPTY (_SEGMENT_ENTRY_INV) +#define _SEGMENT_ENTRY_EMPTY (_SEGMENT_ENTRY_INVALID) #define _SEGMENT_ENTRY_LARGE 0x400 /* STE-format control, large page */ #define _SEGMENT_ENTRY_CO 0x100 /* change-recording override */ +#define _SEGMENT_ENTRY_SPLIT 0x001 /* THP splitting bit */ +#define _SEGMENT_ENTRY_YOUNG 0x002 /* SW segment young bit */ +#define _SEGMENT_ENTRY_NONE _SEGMENT_ENTRY_YOUNG + +/* + * Segment table entry encoding (R = read-only, I = invalid, y = young bit): + * ..R...I...y. + * prot-none, old ..0...1...1. + * prot-none, young ..1...1...1. + * read-only, old ..1...1...0. + * read-only, young ..1...0...1. + * read-write, old ..0...1...0. + * read-write, young ..0...0...1. + * The segment table origin is used to distinguish empty (origin==0) from + * read-write, old segment table entries (origin!=0) + */ + #define _SEGMENT_ENTRY_SPLIT_BIT 0 /* THP splitting bit number */ -#define _SEGMENT_ENTRY_SPLIT (1UL << _SEGMENT_ENTRY_SPLIT_BIT) /* Set of bits not changed in pmd_modify */ #define _SEGMENT_CHG_MASK (_SEGMENT_ENTRY_ORIGIN | _SEGMENT_ENTRY_LARGE \ @@ -369,9 +390,7 @@ extern unsigned long MODULES_END; #define PGSTE_HC_BIT 0x0020000000000000UL #define PGSTE_GR_BIT 0x0004000000000000UL #define PGSTE_GC_BIT 0x0002000000000000UL -#define PGSTE_UR_BIT 0x0000800000000000UL -#define PGSTE_UC_BIT 0x0000400000000000UL /* user dirty (migration) */ -#define PGSTE_IN_BIT 0x0000200000000000UL /* IPTE notify bit */ +#define PGSTE_IN_BIT 0x0000800000000000UL /* IPTE notify bit */ #endif /* CONFIG_64BIT */ @@ -386,14 +405,18 @@ extern unsigned long MODULES_END; /* * Page protection definitions. */ -#define PAGE_NONE __pgprot(_PAGE_TYPE_NONE) -#define PAGE_RO __pgprot(_PAGE_TYPE_RO) -#define PAGE_RW __pgprot(_PAGE_TYPE_RO | _PAGE_SWW) -#define PAGE_RWC __pgprot(_PAGE_TYPE_RW | _PAGE_SWW | _PAGE_SWC) - -#define PAGE_KERNEL PAGE_RWC -#define PAGE_SHARED PAGE_KERNEL -#define PAGE_COPY PAGE_RO +#define PAGE_NONE __pgprot(_PAGE_PRESENT | _PAGE_INVALID) +#define PAGE_READ __pgprot(_PAGE_PRESENT | _PAGE_READ | \ + _PAGE_INVALID | _PAGE_PROTECT) +#define PAGE_WRITE __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \ + _PAGE_INVALID | _PAGE_PROTECT) + +#define PAGE_SHARED __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \ + _PAGE_YOUNG | _PAGE_DIRTY) +#define PAGE_KERNEL __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \ + _PAGE_YOUNG | _PAGE_DIRTY) +#define PAGE_KERNEL_RO __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_YOUNG | \ + _PAGE_PROTECT) /* * On s390 the page table entry has an invalid bit and a read-only bit. @@ -402,35 +425,31 @@ extern unsigned long MODULES_END; */ /*xwr*/ #define __P000 PAGE_NONE -#define __P001 PAGE_RO -#define __P010 PAGE_RO -#define __P011 PAGE_RO -#define __P100 PAGE_RO -#define __P101 PAGE_RO -#define __P110 PAGE_RO -#define __P111 PAGE_RO +#define __P001 PAGE_READ +#define __P010 PAGE_READ +#define __P011 PAGE_READ +#define __P100 PAGE_READ +#define __P101 PAGE_READ +#define __P110 PAGE_READ +#define __P111 PAGE_READ #define __S000 PAGE_NONE -#define __S001 PAGE_RO -#define __S010 PAGE_RW -#define __S011 PAGE_RW -#define __S100 PAGE_RO -#define __S101 PAGE_RO -#define __S110 PAGE_RW -#define __S111 PAGE_RW +#define __S001 PAGE_READ +#define __S010 PAGE_WRITE +#define __S011 PAGE_WRITE +#define __S100 PAGE_READ +#define __S101 PAGE_READ +#define __S110 PAGE_WRITE +#define __S111 PAGE_WRITE /* * Segment entry (large page) protection definitions. */ -#define SEGMENT_NONE __pgprot(_HPAGE_TYPE_NONE) -#define SEGMENT_RO __pgprot(_HPAGE_TYPE_RO) -#define SEGMENT_RW __pgprot(_HPAGE_TYPE_RW) - -static inline int mm_exclusive(struct mm_struct *mm) -{ - return likely(mm == current->active_mm && - atomic_read(&mm->context.attach_count) <= 1); -} +#define SEGMENT_NONE __pgprot(_SEGMENT_ENTRY_INVALID | \ + _SEGMENT_ENTRY_NONE) +#define SEGMENT_READ __pgprot(_SEGMENT_ENTRY_INVALID | \ + _SEGMENT_ENTRY_PROTECT) +#define SEGMENT_WRITE __pgprot(_SEGMENT_ENTRY_INVALID) static inline int mm_has_pgste(struct mm_struct *mm) { @@ -467,7 +486,7 @@ static inline int pgd_none(pgd_t pgd) { if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2) return 0; - return (pgd_val(pgd) & _REGION_ENTRY_INV) != 0UL; + return (pgd_val(pgd) & _REGION_ENTRY_INVALID) != 0UL; } static inline int pgd_bad(pgd_t pgd) @@ -478,7 +497,7 @@ static inline int pgd_bad(pgd_t pgd) * invalid for either table entry. */ unsigned long mask = - ~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INV & + ~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INVALID & ~_REGION_ENTRY_TYPE_MASK & ~_REGION_ENTRY_LENGTH; return (pgd_val(pgd) & mask) != 0; } @@ -494,7 +513,7 @@ static inline int pud_none(pud_t pud) { if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3) return 0; - return (pud_val(pud) & _REGION_ENTRY_INV) != 0UL; + return (pud_val(pud) & _REGION_ENTRY_INVALID) != 0UL; } static inline int pud_large(pud_t pud) @@ -512,7 +531,7 @@ static inline int pud_bad(pud_t pud) * invalid for either table entry. */ unsigned long mask = - ~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INV & + ~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INVALID & ~_REGION_ENTRY_TYPE_MASK & ~_REGION_ENTRY_LENGTH; return (pud_val(pud) & mask) != 0; } @@ -521,30 +540,36 @@ static inline int pud_bad(pud_t pud) static inline int pmd_present(pmd_t pmd) { - unsigned long mask = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO; - return (pmd_val(pmd) & mask) == _HPAGE_TYPE_NONE || - !(pmd_val(pmd) & _SEGMENT_ENTRY_INV); + return pmd_val(pmd) != _SEGMENT_ENTRY_INVALID; } static inline int pmd_none(pmd_t pmd) { - return (pmd_val(pmd) & _SEGMENT_ENTRY_INV) && - !(pmd_val(pmd) & _SEGMENT_ENTRY_RO); + return pmd_val(pmd) == _SEGMENT_ENTRY_INVALID; } static inline int pmd_large(pmd_t pmd) { #ifdef CONFIG_64BIT - return !!(pmd_val(pmd) & _SEGMENT_ENTRY_LARGE); + return (pmd_val(pmd) & _SEGMENT_ENTRY_LARGE) != 0; #else return 0; #endif } +static inline int pmd_prot_none(pmd_t pmd) +{ + return (pmd_val(pmd) & _SEGMENT_ENTRY_INVALID) && + (pmd_val(pmd) & _SEGMENT_ENTRY_NONE); +} + static inline int pmd_bad(pmd_t pmd) { - unsigned long mask = ~_SEGMENT_ENTRY_ORIGIN & ~_SEGMENT_ENTRY_INV; - return (pmd_val(pmd) & mask) != _SEGMENT_ENTRY; +#ifdef CONFIG_64BIT + if (pmd_large(pmd)) + return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS_LARGE) != 0; +#endif + return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS) != 0; } #define __HAVE_ARCH_PMDP_SPLITTING_FLUSH @@ -563,31 +588,40 @@ extern int pmdp_clear_flush_young(struct vm_area_struct *vma, #define __HAVE_ARCH_PMD_WRITE static inline int pmd_write(pmd_t pmd) { - return (pmd_val(pmd) & _SEGMENT_ENTRY_RO) == 0; + if (pmd_prot_none(pmd)) + return 0; + return (pmd_val(pmd) & _SEGMENT_ENTRY_PROTECT) == 0; } static inline int pmd_young(pmd_t pmd) { - return 0; + int young = 0; +#ifdef CONFIG_64BIT + if (pmd_prot_none(pmd)) + young = (pmd_val(pmd) & _SEGMENT_ENTRY_PROTECT) != 0; + else + young = (pmd_val(pmd) & _SEGMENT_ENTRY_YOUNG) != 0; +#endif + return young; } -static inline int pte_none(pte_t pte) +static inline int pte_present(pte_t pte) { - return (pte_val(pte) & _PAGE_INVALID) && !(pte_val(pte) & _PAGE_SWT); + /* Bit pattern: (pte & 0x001) == 0x001 */ + return (pte_val(pte) & _PAGE_PRESENT) != 0; } -static inline int pte_present(pte_t pte) +static inline int pte_none(pte_t pte) { - unsigned long mask = _PAGE_RO | _PAGE_INVALID | _PAGE_SWT | _PAGE_SWX; - return (pte_val(pte) & mask) == _PAGE_TYPE_NONE || - (!(pte_val(pte) & _PAGE_INVALID) && - !(pte_val(pte) & _PAGE_SWT)); + /* Bit pattern: pte == 0x400 */ + return pte_val(pte) == _PAGE_INVALID; } static inline int pte_file(pte_t pte) { - unsigned long mask = _PAGE_RO | _PAGE_INVALID | _PAGE_SWT; - return (pte_val(pte) & mask) == _PAGE_TYPE_FILE; + /* Bit pattern: (pte & 0x601) == 0x600 */ + return (pte_val(pte) & (_PAGE_INVALID | _PAGE_PROTECT | _PAGE_PRESENT)) + == (_PAGE_INVALID | _PAGE_PROTECT); } static inline int pte_special(pte_t pte) @@ -634,6 +668,15 @@ static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste) #endif } +static inline pgste_t pgste_get(pte_t *ptep) +{ + unsigned long pgste = 0; +#ifdef CONFIG_PGSTE + pgste = *(unsigned long *)(ptep + PTRS_PER_PTE); +#endif + return __pgste(pgste); +} + static inline void pgste_set(pte_t *ptep, pgste_t pgste) { #ifdef CONFIG_PGSTE @@ -644,33 +687,28 @@ static inline void pgste_set(pte_t *ptep, pgste_t pgste) static inline pgste_t pgste_update_all(pte_t *ptep, pgste_t pgste) { #ifdef CONFIG_PGSTE - unsigned long address, bits; - unsigned char skey; + unsigned long address, bits, skey; if (pte_val(*ptep) & _PAGE_INVALID) return pgste; address = pte_val(*ptep) & PAGE_MASK; - skey = page_get_storage_key(address); + skey = (unsigned long) page_get_storage_key(address); bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED); - /* Clear page changed & referenced bit in the storage key */ - if (bits & _PAGE_CHANGED) + if (!(pgste_val(pgste) & PGSTE_HC_BIT) && (bits & _PAGE_CHANGED)) { + /* Transfer dirty + referenced bit to host bits in pgste */ + pgste_val(pgste) |= bits << 52; page_set_storage_key(address, skey ^ bits, 0); - else if (bits) + } else if (!(pgste_val(pgste) & PGSTE_HR_BIT) && + (bits & _PAGE_REFERENCED)) { + /* Transfer referenced bit to host bit in pgste */ + pgste_val(pgste) |= PGSTE_HR_BIT; page_reset_referenced(address); + } /* Transfer page changed & referenced bit to guest bits in pgste */ pgste_val(pgste) |= bits << 48; /* GR bit & GC bit */ - /* Get host changed & referenced bits from pgste */ - bits |= (pgste_val(pgste) & (PGSTE_HR_BIT | PGSTE_HC_BIT)) >> 52; - /* Transfer page changed & referenced bit to kvm user bits */ - pgste_val(pgste) |= bits << 45; /* PGSTE_UR_BIT & PGSTE_UC_BIT */ - /* Clear relevant host bits in pgste. */ - pgste_val(pgste) &= ~(PGSTE_HR_BIT | PGSTE_HC_BIT); - pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT); /* Copy page access key and fetch protection bit to pgste */ - pgste_val(pgste) |= - (unsigned long) (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56; - /* Transfer referenced bit to pte */ - pte_val(*ptep) |= (bits & _PAGE_REFERENCED) << 1; + pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT); + pgste_val(pgste) |= (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56; #endif return pgste; @@ -679,24 +717,11 @@ static inline pgste_t pgste_update_all(pte_t *ptep, pgste_t pgste) static inline pgste_t pgste_update_young(pte_t *ptep, pgste_t pgste) { #ifdef CONFIG_PGSTE - int young; - if (pte_val(*ptep) & _PAGE_INVALID) return pgste; /* Get referenced bit from storage key */ - young = page_reset_referenced(pte_val(*ptep) & PAGE_MASK); - if (young) - pgste_val(pgste) |= PGSTE_GR_BIT; - /* Get host referenced bit from pgste */ - if (pgste_val(pgste) & PGSTE_HR_BIT) { - pgste_val(pgste) &= ~PGSTE_HR_BIT; - young = 1; - } - /* Transfer referenced bit to kvm user bits and pte */ - if (young) { - pgste_val(pgste) |= PGSTE_UR_BIT; - pte_val(*ptep) |= _PAGE_SWR; - } + if (page_reset_referenced(pte_val(*ptep) & PAGE_MASK)) + pgste_val(pgste) |= PGSTE_HR_BIT | PGSTE_GR_BIT; #endif return pgste; } @@ -723,13 +748,13 @@ static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry) static inline void pgste_set_pte(pte_t *ptep, pte_t entry) { - if (!MACHINE_HAS_ESOP && (pte_val(entry) & _PAGE_SWW)) { + if (!MACHINE_HAS_ESOP && (pte_val(entry) & _PAGE_WRITE)) { /* * Without enhanced suppression-on-protection force * the dirty bit on for all writable ptes. */ - pte_val(entry) |= _PAGE_SWC; - pte_val(entry) &= ~_PAGE_RO; + pte_val(entry) |= _PAGE_DIRTY; + pte_val(entry) &= ~_PAGE_PROTECT; } *ptep = entry; } @@ -841,21 +866,17 @@ static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, */ static inline int pte_write(pte_t pte) { - return (pte_val(pte) & _PAGE_SWW) != 0; + return (pte_val(pte) & _PAGE_WRITE) != 0; } static inline int pte_dirty(pte_t pte) { - return (pte_val(pte) & _PAGE_SWC) != 0; + return (pte_val(pte) & _PAGE_DIRTY) != 0; } static inline int pte_young(pte_t pte) { -#ifdef CONFIG_PGSTE - if (pte_val(pte) & _PAGE_SWR) - return 1; -#endif - return 0; + return (pte_val(pte) & _PAGE_YOUNG) != 0; } /* @@ -880,12 +901,12 @@ static inline void pud_clear(pud_t *pud) static inline void pmd_clear(pmd_t *pmdp) { - pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY; + pmd_val(*pmdp) = _SEGMENT_ENTRY_INVALID; } static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - pte_val(*ptep) = _PAGE_TYPE_EMPTY; + pte_val(*ptep) = _PAGE_INVALID; } /* @@ -896,55 +917,63 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { pte_val(pte) &= _PAGE_CHG_MASK; pte_val(pte) |= pgprot_val(newprot); - if ((pte_val(pte) & _PAGE_SWC) && (pte_val(pte) & _PAGE_SWW)) - pte_val(pte) &= ~_PAGE_RO; + /* + * newprot for PAGE_NONE, PAGE_READ and PAGE_WRITE has the + * invalid bit set, clear it again for readable, young pages + */ + if ((pte_val(pte) & _PAGE_YOUNG) && (pte_val(pte) & _PAGE_READ)) + pte_val(pte) &= ~_PAGE_INVALID; + /* + * newprot for PAGE_READ and PAGE_WRITE has the page protection + * bit set, clear it again for writable, dirty pages + */ + if ((pte_val(pte) & _PAGE_DIRTY) && (pte_val(pte) & _PAGE_WRITE)) + pte_val(pte) &= ~_PAGE_PROTECT; return pte; } static inline pte_t pte_wrprotect(pte_t pte) { - pte_val(pte) &= ~_PAGE_SWW; - /* Do not clobber _PAGE_TYPE_NONE pages! */ - if (!(pte_val(pte) & _PAGE_INVALID)) - pte_val(pte) |= _PAGE_RO; + pte_val(pte) &= ~_PAGE_WRITE; + pte_val(pte) |= _PAGE_PROTECT; return pte; } static inline pte_t pte_mkwrite(pte_t pte) { - pte_val(pte) |= _PAGE_SWW; - if (pte_val(pte) & _PAGE_SWC) - pte_val(pte) &= ~_PAGE_RO; + pte_val(pte) |= _PAGE_WRITE; + if (pte_val(pte) & _PAGE_DIRTY) + pte_val(pte) &= ~_PAGE_PROTECT; return pte; } static inline pte_t pte_mkclean(pte_t pte) { - pte_val(pte) &= ~_PAGE_SWC; - /* Do not clobber _PAGE_TYPE_NONE pages! */ - if (!(pte_val(pte) & _PAGE_INVALID)) - pte_val(pte) |= _PAGE_RO; + pte_val(pte) &= ~_PAGE_DIRTY; + pte_val(pte) |= _PAGE_PROTECT; return pte; } static inline pte_t pte_mkdirty(pte_t pte) { - pte_val(pte) |= _PAGE_SWC; - if (pte_val(pte) & _PAGE_SWW) - pte_val(pte) &= ~_PAGE_RO; + pte_val(pte) |= _PAGE_DIRTY; + if (pte_val(pte) & _PAGE_WRITE) + pte_val(pte) &= ~_PAGE_PROTECT; return pte; } static inline pte_t pte_mkold(pte_t pte) { -#ifdef CONFIG_PGSTE - pte_val(pte) &= ~_PAGE_SWR; -#endif + pte_val(pte) &= ~_PAGE_YOUNG; + pte_val(pte) |= _PAGE_INVALID; return pte; } static inline pte_t pte_mkyoung(pte_t pte) { + pte_val(pte) |= _PAGE_YOUNG; + if (pte_val(pte) & _PAGE_READ) + pte_val(pte) &= ~_PAGE_INVALID; return pte; } @@ -957,7 +986,7 @@ static inline pte_t pte_mkspecial(pte_t pte) #ifdef CONFIG_HUGETLB_PAGE static inline pte_t pte_mkhuge(pte_t pte) { - pte_val(pte) |= (_SEGMENT_ENTRY_LARGE | _SEGMENT_ENTRY_CO); + pte_val(pte) |= _PAGE_LARGE; return pte; } #endif @@ -974,8 +1003,8 @@ static inline int ptep_test_and_clear_user_dirty(struct mm_struct *mm, if (mm_has_pgste(mm)) { pgste = pgste_get_lock(ptep); pgste = pgste_update_all(ptep, pgste); - dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT); - pgste_val(pgste) &= ~PGSTE_UC_BIT; + dirty = !!(pgste_val(pgste) & PGSTE_HC_BIT); + pgste_val(pgste) &= ~PGSTE_HC_BIT; pgste_set_unlock(ptep, pgste); return dirty; } @@ -994,59 +1023,75 @@ static inline int ptep_test_and_clear_user_young(struct mm_struct *mm, if (mm_has_pgste(mm)) { pgste = pgste_get_lock(ptep); pgste = pgste_update_young(ptep, pgste); - young = !!(pgste_val(pgste) & PGSTE_UR_BIT); - pgste_val(pgste) &= ~PGSTE_UR_BIT; + young = !!(pgste_val(pgste) & PGSTE_HR_BIT); + pgste_val(pgste) &= ~PGSTE_HR_BIT; pgste_set_unlock(ptep, pgste); } return young; } +static inline void __ptep_ipte(unsigned long address, pte_t *ptep) +{ + if (!(pte_val(*ptep) & _PAGE_INVALID)) { +#ifndef CONFIG_64BIT + /* pto must point to the start of the segment table */ + pte_t *pto = (pte_t *) (((unsigned long) ptep) & 0x7ffffc00); +#else + /* ipte in zarch mode can do the math */ + pte_t *pto = ptep; +#endif + asm volatile( + " ipte %2,%3" + : "=m" (*ptep) : "m" (*ptep), + "a" (pto), "a" (address)); + } +} + +static inline void ptep_flush_lazy(struct mm_struct *mm, + unsigned long address, pte_t *ptep) +{ + int active = (mm == current->active_mm) ? 1 : 0; + + if (atomic_read(&mm->context.attach_count) > active) + __ptep_ipte(address, ptep); + else + mm->context.flush_mm = 1; +} + #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) { pgste_t pgste; pte_t pte; + int young; if (mm_has_pgste(vma->vm_mm)) { pgste = pgste_get_lock(ptep); - pgste = pgste_update_young(ptep, pgste); - pte = *ptep; - *ptep = pte_mkold(pte); - pgste_set_unlock(ptep, pgste); - return pte_young(pte); + pgste = pgste_ipte_notify(vma->vm_mm, addr, ptep, pgste); } - return 0; + + pte = *ptep; + __ptep_ipte(addr, ptep); + young = pte_young(pte); + pte = pte_mkold(pte); + + if (mm_has_pgste(vma->vm_mm)) { + pgste_set_pte(ptep, pte); + pgste_set_unlock(ptep, pgste); + } else + *ptep = pte; + + return young; } #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH static inline int ptep_clear_flush_young(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) { - /* No need to flush TLB - * On s390 reference bits are in storage key and never in TLB - * With virtualization we handle the reference bit, without we - * we can simply return */ return ptep_test_and_clear_young(vma, address, ptep); } -static inline void __ptep_ipte(unsigned long address, pte_t *ptep) -{ - if (!(pte_val(*ptep) & _PAGE_INVALID)) { -#ifndef CONFIG_64BIT - /* pto must point to the start of the segment table */ - pte_t *pto = (pte_t *) (((unsigned long) ptep) & 0x7ffffc00); -#else - /* ipte in zarch mode can do the math */ - pte_t *pto = ptep; -#endif - asm volatile( - " ipte %2,%3" - : "=m" (*ptep) : "m" (*ptep), - "a" (pto), "a" (address)); - } -} - /* * This is hard to understand. ptep_get_and_clear and ptep_clear_flush * both clear the TLB for the unmapped pte. The reason is that @@ -1067,16 +1112,14 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, pgste_t pgste; pte_t pte; - mm->context.flush_mm = 1; if (mm_has_pgste(mm)) { pgste = pgste_get_lock(ptep); pgste = pgste_ipte_notify(mm, address, ptep, pgste); } pte = *ptep; - if (!mm_exclusive(mm)) - __ptep_ipte(address, ptep); - pte_val(*ptep) = _PAGE_TYPE_EMPTY; + ptep_flush_lazy(mm, address, ptep); + pte_val(*ptep) = _PAGE_INVALID; if (mm_has_pgste(mm)) { pgste = pgste_update_all(&pte, pgste); @@ -1093,15 +1136,14 @@ static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, pgste_t pgste; pte_t pte; - mm->context.flush_mm = 1; if (mm_has_pgste(mm)) { pgste = pgste_get_lock(ptep); pgste_ipte_notify(mm, address, ptep, pgste); } pte = *ptep; - if (!mm_exclusive(mm)) - __ptep_ipte(address, ptep); + ptep_flush_lazy(mm, address, ptep); + pte_val(*ptep) |= _PAGE_INVALID; if (mm_has_pgste(mm)) { pgste = pgste_update_all(&pte, pgste); @@ -1117,7 +1159,7 @@ static inline void ptep_modify_prot_commit(struct mm_struct *mm, pgste_t pgste; if (mm_has_pgste(mm)) { - pgste = *(pgste_t *)(ptep + PTRS_PER_PTE); + pgste = pgste_get(ptep); pgste_set_key(ptep, pgste, pte); pgste_set_pte(ptep, pte); pgste_set_unlock(ptep, pgste); @@ -1139,7 +1181,7 @@ static inline pte_t ptep_clear_flush(struct vm_area_struct *vma, pte = *ptep; __ptep_ipte(address, ptep); - pte_val(*ptep) = _PAGE_TYPE_EMPTY; + pte_val(*ptep) = _PAGE_INVALID; if (mm_has_pgste(vma->vm_mm)) { pgste = pgste_update_all(&pte, pgste); @@ -1163,18 +1205,17 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, pgste_t pgste; pte_t pte; - if (mm_has_pgste(mm)) { + if (!full && mm_has_pgste(mm)) { pgste = pgste_get_lock(ptep); - if (!full) - pgste = pgste_ipte_notify(mm, address, ptep, pgste); + pgste = pgste_ipte_notify(mm, address, ptep, pgste); } pte = *ptep; if (!full) - __ptep_ipte(address, ptep); - pte_val(*ptep) = _PAGE_TYPE_EMPTY; + ptep_flush_lazy(mm, address, ptep); + pte_val(*ptep) = _PAGE_INVALID; - if (mm_has_pgste(mm)) { + if (!full && mm_has_pgste(mm)) { pgste = pgste_update_all(&pte, pgste); pgste_set_unlock(ptep, pgste); } @@ -1189,14 +1230,12 @@ static inline pte_t ptep_set_wrprotect(struct mm_struct *mm, pte_t pte = *ptep; if (pte_write(pte)) { - mm->context.flush_mm = 1; if (mm_has_pgste(mm)) { pgste = pgste_get_lock(ptep); pgste = pgste_ipte_notify(mm, address, ptep, pgste); } - if (!mm_exclusive(mm)) - __ptep_ipte(address, ptep); + ptep_flush_lazy(mm, address, ptep); pte = pte_wrprotect(pte); if (mm_has_pgste(mm)) { @@ -1240,7 +1279,7 @@ static inline pte_t mk_pte_phys(unsigned long physpage, pgprot_t pgprot) { pte_t __pte; pte_val(__pte) = physpage + pgprot_val(pgprot); - return __pte; + return pte_mkyoung(__pte); } static inline pte_t mk_pte(struct page *page, pgprot_t pgprot) @@ -1248,10 +1287,8 @@ static inline pte_t mk_pte(struct page *page, pgprot_t pgprot) unsigned long physpage = page_to_phys(page); pte_t __pte = mk_pte_phys(physpage, pgprot); - if ((pte_val(__pte) & _PAGE_SWW) && PageDirty(page)) { - pte_val(__pte) |= _PAGE_SWC; - pte_val(__pte) &= ~_PAGE_RO; - } + if (pte_write(__pte) && PageDirty(page)) + __pte = pte_mkdirty(__pte); return __pte; } @@ -1313,7 +1350,7 @@ static inline void __pmd_idte(unsigned long address, pmd_t *pmdp) unsigned long sto = (unsigned long) pmdp - pmd_index(address) * sizeof(pmd_t); - if (!(pmd_val(*pmdp) & _SEGMENT_ENTRY_INV)) { + if (!(pmd_val(*pmdp) & _SEGMENT_ENTRY_INVALID)) { asm volatile( " .insn rrf,0xb98e0000,%2,%3,0,0" : "=m" (*pmdp) @@ -1324,24 +1361,68 @@ static inline void __pmd_idte(unsigned long address, pmd_t *pmdp) } } +static inline void __pmd_csp(pmd_t *pmdp) +{ + register unsigned long reg2 asm("2") = pmd_val(*pmdp); + register unsigned long reg3 asm("3") = pmd_val(*pmdp) | + _SEGMENT_ENTRY_INVALID; + register unsigned long reg4 asm("4") = ((unsigned long) pmdp) + 5; + + asm volatile( + " csp %1,%3" + : "=m" (*pmdp) + : "d" (reg2), "d" (reg3), "d" (reg4), "m" (*pmdp) : "cc"); +} + #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE) static inline unsigned long massage_pgprot_pmd(pgprot_t pgprot) { /* - * pgprot is PAGE_NONE, PAGE_RO, or PAGE_RW (see __Pxxx / __Sxxx) + * pgprot is PAGE_NONE, PAGE_READ, or PAGE_WRITE (see __Pxxx / __Sxxx) * Convert to segment table entry format. */ if (pgprot_val(pgprot) == pgprot_val(PAGE_NONE)) return pgprot_val(SEGMENT_NONE); - if (pgprot_val(pgprot) == pgprot_val(PAGE_RO)) - return pgprot_val(SEGMENT_RO); - return pgprot_val(SEGMENT_RW); + if (pgprot_val(pgprot) == pgprot_val(PAGE_READ)) + return pgprot_val(SEGMENT_READ); + return pgprot_val(SEGMENT_WRITE); +} + +static inline pmd_t pmd_mkyoung(pmd_t pmd) +{ +#ifdef CONFIG_64BIT + if (pmd_prot_none(pmd)) { + pmd_val(pmd) |= _SEGMENT_ENTRY_PROTECT; + } else { + pmd_val(pmd) |= _SEGMENT_ENTRY_YOUNG; + pmd_val(pmd) &= ~_SEGMENT_ENTRY_INVALID; + } +#endif + return pmd; +} + +static inline pmd_t pmd_mkold(pmd_t pmd) +{ +#ifdef CONFIG_64BIT + if (pmd_prot_none(pmd)) { + pmd_val(pmd) &= ~_SEGMENT_ENTRY_PROTECT; + } else { + pmd_val(pmd) &= ~_SEGMENT_ENTRY_YOUNG; + pmd_val(pmd) |= _SEGMENT_ENTRY_INVALID; + } +#endif + return pmd; } static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) { + int young; + + young = pmd_young(pmd); pmd_val(pmd) &= _SEGMENT_CHG_MASK; pmd_val(pmd) |= massage_pgprot_pmd(newprot); + if (young) + pmd = pmd_mkyoung(pmd); return pmd; } @@ -1349,14 +1430,14 @@ static inline pmd_t mk_pmd_phys(unsigned long physpage, pgprot_t pgprot) { pmd_t __pmd; pmd_val(__pmd) = physpage + massage_pgprot_pmd(pgprot); - return __pmd; + return pmd_mkyoung(__pmd); } static inline pmd_t pmd_mkwrite(pmd_t pmd) { - /* Do not clobber _HPAGE_TYPE_NONE pages! */ - if (!(pmd_val(pmd) & _SEGMENT_ENTRY_INV)) - pmd_val(pmd) &= ~_SEGMENT_ENTRY_RO; + /* Do not clobber PROT_NONE segments! */ + if (!pmd_prot_none(pmd)) + pmd_val(pmd) &= ~_SEGMENT_ENTRY_PROTECT; return pmd; } #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLB_PAGE */ @@ -1378,7 +1459,7 @@ static inline int pmd_trans_splitting(pmd_t pmd) static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t entry) { - if (!(pmd_val(entry) & _SEGMENT_ENTRY_INV) && MACHINE_HAS_EDAT1) + if (!(pmd_val(entry) & _SEGMENT_ENTRY_INVALID) && MACHINE_HAS_EDAT1) pmd_val(entry) |= _SEGMENT_ENTRY_CO; *pmdp = entry; } @@ -1391,7 +1472,9 @@ static inline pmd_t pmd_mkhuge(pmd_t pmd) static inline pmd_t pmd_wrprotect(pmd_t pmd) { - pmd_val(pmd) |= _SEGMENT_ENTRY_RO; + /* Do not clobber PROT_NONE segments! */ + if (!pmd_prot_none(pmd)) + pmd_val(pmd) |= _SEGMENT_ENTRY_PROTECT; return pmd; } @@ -1401,50 +1484,16 @@ static inline pmd_t pmd_mkdirty(pmd_t pmd) return pmd; } -static inline pmd_t pmd_mkold(pmd_t pmd) -{ - /* No referenced bit in the segment table entry. */ - return pmd; -} - -static inline pmd_t pmd_mkyoung(pmd_t pmd) -{ - /* No referenced bit in the segment table entry. */ - return pmd; -} - #define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp) { - unsigned long pmd_addr = pmd_val(*pmdp) & HPAGE_MASK; - long tmp, rc; - int counter; + pmd_t pmd; - rc = 0; - if (MACHINE_HAS_RRBM) { - counter = PTRS_PER_PTE >> 6; - asm volatile( - "0: .insn rre,0xb9ae0000,%0,%3\n" /* rrbm */ - " ogr %1,%0\n" - " la %3,0(%4,%3)\n" - " brct %2,0b\n" - : "=&d" (tmp), "+&d" (rc), "+d" (counter), - "+a" (pmd_addr) - : "a" (64 * 4096UL) : "cc"); - rc = !!rc; - } else { - counter = PTRS_PER_PTE; - asm volatile( - "0: rrbe 0,%2\n" - " la %2,0(%3,%2)\n" - " brc 12,1f\n" - " lhi %0,1\n" - "1: brct %1,0b\n" - : "+d" (rc), "+d" (counter), "+a" (pmd_addr) - : "a" (4096UL) : "cc"); - } - return rc; + pmd = *pmdp; + __pmd_idte(address, pmdp); + *pmdp = pmd_mkold(pmd); + return pmd_young(pmd); } #define __HAVE_ARCH_PMDP_GET_AND_CLEAR @@ -1510,10 +1559,8 @@ static inline unsigned long pmd_pfn(pmd_t pmd) * exception will occur instead of a page translation exception. The * specifiation exception has the bad habit not to store necessary * information in the lowcore. - * Bit 21 and bit 22 are the page invalid bit and the page protection - * bit. We set both to indicate a swapped page. - * Bit 30 and 31 are used to distinguish the different page types. For - * a swapped page these bits need to be zero. + * Bits 21, 22, 30 and 31 are used to indicate the page type. + * A swap pte is indicated by bit pattern (pte & 0x603) == 0x402 * This leaves the bits 1-19 and bits 24-29 to store type and offset. * We use the 5 bits from 25-29 for the type and the 20 bits from 1-19 * plus 24 for the offset. @@ -1527,10 +1574,8 @@ static inline unsigned long pmd_pfn(pmd_t pmd) * exception will occur instead of a page translation exception. The * specifiation exception has the bad habit not to store necessary * information in the lowcore. - * Bit 53 and bit 54 are the page invalid bit and the page protection - * bit. We set both to indicate a swapped page. - * Bit 62 and 63 are used to distinguish the different page types. For - * a swapped page these bits need to be zero. + * Bits 53, 54, 62 and 63 are used to indicate the page type. + * A swap pte is indicated by bit pattern (pte & 0x603) == 0x402 * This leaves the bits 0-51 and bits 56-61 to store type and offset. * We use the 5 bits from 57-61 for the type and the 53 bits from 0-51 * plus 56 for the offset. @@ -1547,7 +1592,7 @@ static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset) { pte_t pte; offset &= __SWP_OFFSET_MASK; - pte_val(pte) = _PAGE_TYPE_SWAP | ((type & 0x1f) << 2) | + pte_val(pte) = _PAGE_INVALID | _PAGE_TYPE | ((type & 0x1f) << 2) | ((offset & 1UL) << 7) | ((offset & ~1UL) << 11); return pte; } @@ -1570,7 +1615,7 @@ static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset) #define pgoff_to_pte(__off) \ ((pte_t) { ((((__off) & 0x7f) << 1) + (((__off) >> 7) << 12)) \ - | _PAGE_TYPE_FILE }) + | _PAGE_INVALID | _PAGE_PROTECT }) #endif /* !__ASSEMBLY__ */ diff --git a/arch/s390/include/asm/serial.h b/arch/s390/include/asm/serial.h new file mode 100644 index 000000000000..5b3e48ef534b --- /dev/null +++ b/arch/s390/include/asm/serial.h @@ -0,0 +1,6 @@ +#ifndef _ASM_S390_SERIAL_H +#define _ASM_S390_SERIAL_H + +#define BASE_BAUD 0 + +#endif /* _ASM_S390_SERIAL_H */ diff --git a/arch/s390/include/asm/switch_to.h b/arch/s390/include/asm/switch_to.h index 80b6f11263c4..6dbd559763c9 100644 --- a/arch/s390/include/asm/switch_to.h +++ b/arch/s390/include/asm/switch_to.h @@ -8,6 +8,7 @@ #define __ASM_SWITCH_TO_H #include <linux/thread_info.h> +#include <asm/ptrace.h> extern struct task_struct *__switch_to(void *, void *); extern void update_cr_regs(struct task_struct *task); @@ -68,12 +69,16 @@ static inline void restore_fp_regs(s390_fp_regs *fpregs) static inline void save_access_regs(unsigned int *acrs) { - asm volatile("stam 0,15,%0" : "=Q" (*acrs)); + typedef struct { int _[NUM_ACRS]; } acrstype; + + asm volatile("stam 0,15,%0" : "=Q" (*(acrstype *)acrs)); } static inline void restore_access_regs(unsigned int *acrs) { - asm volatile("lam 0,15,%0" : : "Q" (*acrs)); + typedef struct { int _[NUM_ACRS]; } acrstype; + + asm volatile("lam 0,15,%0" : : "Q" (*(acrstype *)acrs)); } #define switch_to(prev,next,last) do { \ diff --git a/arch/s390/include/asm/tlb.h b/arch/s390/include/asm/tlb.h index 6d6d92b4ea11..2cb846c4b37f 100644 --- a/arch/s390/include/asm/tlb.h +++ b/arch/s390/include/asm/tlb.h @@ -63,13 +63,14 @@ static inline void tlb_gather_mmu(struct mmu_gather *tlb, static inline void tlb_flush_mmu(struct mmu_gather *tlb) { + __tlb_flush_mm_lazy(tlb->mm); tlb_table_flush(tlb); } static inline void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) { - tlb_table_flush(tlb); + tlb_flush_mmu(tlb); } /* diff --git a/arch/s390/include/asm/tlbflush.h b/arch/s390/include/asm/tlbflush.h index 6b32af30878c..f9fef0425fee 100644 --- a/arch/s390/include/asm/tlbflush.h +++ b/arch/s390/include/asm/tlbflush.h @@ -86,7 +86,7 @@ static inline void __tlb_flush_mm(struct mm_struct * mm) __tlb_flush_full(mm); } -static inline void __tlb_flush_mm_cond(struct mm_struct * mm) +static inline void __tlb_flush_mm_lazy(struct mm_struct * mm) { if (mm->context.flush_mm) { __tlb_flush_mm(mm); @@ -118,13 +118,13 @@ static inline void __tlb_flush_mm_cond(struct mm_struct * mm) static inline void flush_tlb_mm(struct mm_struct *mm) { - __tlb_flush_mm_cond(mm); + __tlb_flush_mm_lazy(mm); } static inline void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { - __tlb_flush_mm_cond(vma->vm_mm); + __tlb_flush_mm_lazy(vma->vm_mm); } static inline void flush_tlb_kernel_range(unsigned long start, diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index be7a408be7a1..cc30d1fb000c 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -18,6 +18,7 @@ #include <asm/unistd.h> #include <asm/page.h> #include <asm/sigp.h> +#include <asm/irq.h> __PT_R0 = __PT_GPRS __PT_R1 = __PT_GPRS + 4 @@ -435,6 +436,11 @@ io_skip: io_loop: l %r1,BASED(.Ldo_IRQ) lr %r2,%r11 # pass pointer to pt_regs + lhi %r3,IO_INTERRUPT + tm __PT_INT_CODE+8(%r11),0x80 # adapter interrupt ? + jz io_call + lhi %r3,THIN_INTERRUPT +io_call: basr %r14,%r1 # call do_IRQ tm __LC_MACHINE_FLAGS+2,0x10 # MACHINE_FLAG_LPAR jz io_return @@ -584,9 +590,10 @@ ext_skip: mvc __PT_INT_CODE(4,%r11),__LC_EXT_CPU_ADDR mvc __PT_INT_PARM(4,%r11),__LC_EXT_PARAMS TRACE_IRQS_OFF + l %r1,BASED(.Ldo_IRQ) lr %r2,%r11 # pass pointer to pt_regs - l %r1,BASED(.Ldo_extint) - basr %r14,%r1 # call do_extint + lhi %r3,EXT_INTERRUPT + basr %r14,%r1 # call do_IRQ j io_return /* @@ -879,13 +886,13 @@ cleanup_idle: stm %r9,%r10,__LC_SYSTEM_TIMER mvc __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2) # prepare return psw - n %r8,BASED(cleanup_idle_wait) # clear wait state bit + n %r8,BASED(cleanup_idle_wait) # clear irq & wait state bits l %r9,24(%r11) # return from psw_idle br %r14 cleanup_idle_insn: .long psw_idle_lpsw + 0x80000000 cleanup_idle_wait: - .long 0xfffdffff + .long 0xfcfdffff /* * Integer constants @@ -902,7 +909,6 @@ cleanup_idle_wait: .Ldo_machine_check: .long s390_do_machine_check .Lhandle_mcck: .long s390_handle_mcck .Ldo_IRQ: .long do_IRQ -.Ldo_extint: .long do_extint .Ldo_signal: .long do_signal .Ldo_notify_resume: .long do_notify_resume .Ldo_per_trap: .long do_per_trap diff --git a/arch/s390/kernel/entry64.S b/arch/s390/kernel/entry64.S index 1c039d0c24c7..2b2188b97c6a 100644 --- a/arch/s390/kernel/entry64.S +++ b/arch/s390/kernel/entry64.S @@ -19,6 +19,7 @@ #include <asm/unistd.h> #include <asm/page.h> #include <asm/sigp.h> +#include <asm/irq.h> __PT_R0 = __PT_GPRS __PT_R1 = __PT_GPRS + 8 @@ -468,6 +469,11 @@ io_skip: xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) io_loop: lgr %r2,%r11 # pass pointer to pt_regs + lghi %r3,IO_INTERRUPT + tm __PT_INT_CODE+8(%r11),0x80 # adapter interrupt ? + jz io_call + lghi %r3,THIN_INTERRUPT +io_call: brasl %r14,do_IRQ tm __LC_MACHINE_FLAGS+6,0x10 # MACHINE_FLAG_LPAR jz io_return @@ -623,7 +629,8 @@ ext_skip: TRACE_IRQS_OFF xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) lgr %r2,%r11 # pass pointer to pt_regs - brasl %r14,do_extint + lghi %r3,EXT_INTERRUPT + brasl %r14,do_IRQ j io_return /* @@ -922,7 +929,7 @@ cleanup_idle: stg %r9,__LC_SYSTEM_TIMER mvc __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2) # prepare return psw - nihh %r8,0xfffd # clear wait state bit + nihh %r8,0xfcfd # clear irq & wait state bits lg %r9,48(%r11) # return from psw_idle br %r14 cleanup_idle_insn: diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index 54b0995514e8..b34ba0ea96a9 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -22,6 +22,7 @@ #include <asm/cputime.h> #include <asm/lowcore.h> #include <asm/irq.h> +#include <asm/hw_irq.h> #include "entry.h" DEFINE_PER_CPU_SHARED_ALIGNED(struct irq_stat, irq_stat); @@ -42,9 +43,10 @@ struct irq_class { * Since the external and I/O interrupt fields are already sums we would end * up with having a sum which accounts each interrupt twice. */ -static const struct irq_class irqclass_main_desc[NR_IRQS] = { - [EXTERNAL_INTERRUPT] = {.name = "EXT"}, - [IO_INTERRUPT] = {.name = "I/O"} +static const struct irq_class irqclass_main_desc[NR_IRQS_BASE] = { + [EXT_INTERRUPT] = {.name = "EXT"}, + [IO_INTERRUPT] = {.name = "I/O"}, + [THIN_INTERRUPT] = {.name = "AIO"}, }; /* @@ -86,6 +88,28 @@ static const struct irq_class irqclass_sub_desc[NR_ARCH_IRQS] = { [CPU_RST] = {.name = "RST", .desc = "[CPU] CPU Restart"}, }; +void __init init_IRQ(void) +{ + irq_reserve_irqs(0, THIN_INTERRUPT); + init_cio_interrupts(); + init_airq_interrupts(); + init_ext_interrupts(); +} + +void do_IRQ(struct pt_regs *regs, int irq) +{ + struct pt_regs *old_regs; + + old_regs = set_irq_regs(regs); + irq_enter(); + if (S390_lowcore.int_clock >= S390_lowcore.clock_comparator) + /* Serve timer interrupts first. */ + clock_comparator_work(); + generic_handle_irq(irq); + irq_exit(); + set_irq_regs(old_regs); +} + /* * show_interrupts is needed by /proc/interrupts. */ @@ -100,27 +124,36 @@ int show_interrupts(struct seq_file *p, void *v) for_each_online_cpu(cpu) seq_printf(p, "CPU%d ", cpu); seq_putc(p, '\n'); + goto out; } if (irq < NR_IRQS) { + if (irq >= NR_IRQS_BASE) + goto out; seq_printf(p, "%s: ", irqclass_main_desc[irq].name); for_each_online_cpu(cpu) - seq_printf(p, "%10u ", kstat_cpu(cpu).irqs[irq]); + seq_printf(p, "%10u ", kstat_irqs_cpu(irq, cpu)); seq_putc(p, '\n'); - goto skip_arch_irqs; + goto out; } for (irq = 0; irq < NR_ARCH_IRQS; irq++) { seq_printf(p, "%s: ", irqclass_sub_desc[irq].name); for_each_online_cpu(cpu) - seq_printf(p, "%10u ", per_cpu(irq_stat, cpu).irqs[irq]); + seq_printf(p, "%10u ", + per_cpu(irq_stat, cpu).irqs[irq]); if (irqclass_sub_desc[irq].desc) seq_printf(p, " %s", irqclass_sub_desc[irq].desc); seq_putc(p, '\n'); } -skip_arch_irqs: +out: put_online_cpus(); return 0; } +int arch_show_interrupts(struct seq_file *p, int prec) +{ + return 0; +} + /* * Switch to the asynchronous interrupt stack for softirq execution. */ @@ -159,14 +192,6 @@ asmlinkage void do_softirq(void) local_irq_restore(flags); } -#ifdef CONFIG_PROC_FS -void init_irq_proc(void) -{ - if (proc_mkdir("irq", NULL)) - create_prof_cpu_mask(); -} -#endif - /* * ext_int_hash[index] is the list head for all external interrupts that hash * to this index. @@ -183,14 +208,6 @@ struct ext_int_info { /* ext_int_hash_lock protects the handler lists for external interrupts */ DEFINE_SPINLOCK(ext_int_hash_lock); -static void __init init_external_interrupts(void) -{ - int idx; - - for (idx = 0; idx < ARRAY_SIZE(ext_int_hash); idx++) - INIT_LIST_HEAD(&ext_int_hash[idx]); -} - static inline int ext_hash(u16 code) { return (code + (code >> 9)) & 0xff; @@ -234,20 +251,13 @@ int unregister_external_interrupt(u16 code, ext_int_handler_t handler) } EXPORT_SYMBOL(unregister_external_interrupt); -void __irq_entry do_extint(struct pt_regs *regs) +static irqreturn_t do_ext_interrupt(int irq, void *dummy) { + struct pt_regs *regs = get_irq_regs(); struct ext_code ext_code; - struct pt_regs *old_regs; struct ext_int_info *p; int index; - old_regs = set_irq_regs(regs); - irq_enter(); - if (S390_lowcore.int_clock >= S390_lowcore.clock_comparator) { - /* Serve timer interrupts first. */ - clock_comparator_work(); - } - kstat_incr_irqs_this_cpu(EXTERNAL_INTERRUPT, NULL); ext_code = *(struct ext_code *) ®s->int_code; if (ext_code.code != 0x1004) __get_cpu_var(s390_idle).nohz_delay = 1; @@ -259,13 +269,25 @@ void __irq_entry do_extint(struct pt_regs *regs) p->handler(ext_code, regs->int_parm, regs->int_parm_long); rcu_read_unlock(); - irq_exit(); - set_irq_regs(old_regs); + + return IRQ_HANDLED; } -void __init init_IRQ(void) +static struct irqaction external_interrupt = { + .name = "EXT", + .handler = do_ext_interrupt, +}; + +void __init init_ext_interrupts(void) { - init_external_interrupts(); + int idx; + + for (idx = 0; idx < ARRAY_SIZE(ext_int_hash); idx++) + INIT_LIST_HEAD(&ext_int_hash[idx]); + + irq_set_chip_and_handler(EXT_INTERRUPT, + &dummy_irq_chip, handle_percpu_irq); + setup_irq(EXT_INTERRUPT, &external_interrupt); } static DEFINE_SPINLOCK(sc_irq_lock); @@ -313,69 +335,3 @@ void measurement_alert_subclass_unregister(void) spin_unlock(&ma_subclass_lock); } EXPORT_SYMBOL(measurement_alert_subclass_unregister); - -#ifdef CONFIG_SMP -void synchronize_irq(unsigned int irq) -{ - /* - * Not needed, the handler is protected by a lock and IRQs that occur - * after the handler is deleted are just NOPs. - */ -} -EXPORT_SYMBOL_GPL(synchronize_irq); -#endif - -#ifndef CONFIG_PCI - -/* Only PCI devices have dynamically-defined IRQ handlers */ - -int request_irq(unsigned int irq, irq_handler_t handler, - unsigned long irqflags, const char *devname, void *dev_id) -{ - return -EINVAL; -} -EXPORT_SYMBOL_GPL(request_irq); - -void free_irq(unsigned int irq, void *dev_id) -{ - WARN_ON(1); -} -EXPORT_SYMBOL_GPL(free_irq); - -void enable_irq(unsigned int irq) -{ - WARN_ON(1); -} -EXPORT_SYMBOL_GPL(enable_irq); - -void disable_irq(unsigned int irq) -{ - WARN_ON(1); -} -EXPORT_SYMBOL_GPL(disable_irq); - -#endif /* !CONFIG_PCI */ - -void disable_irq_nosync(unsigned int irq) -{ - disable_irq(irq); -} -EXPORT_SYMBOL_GPL(disable_irq_nosync); - -unsigned long probe_irq_on(void) -{ - return 0; -} -EXPORT_SYMBOL_GPL(probe_irq_on); - -int probe_irq_off(unsigned long val) -{ - return 0; -} -EXPORT_SYMBOL_GPL(probe_irq_off); - -unsigned int probe_irq_mask(unsigned long val) -{ - return val; -} -EXPORT_SYMBOL_GPL(probe_irq_mask); diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c index 3388b2b2a07d..adbbe7f1cb0d 100644 --- a/arch/s390/kernel/kprobes.c +++ b/arch/s390/kernel/kprobes.c @@ -105,14 +105,31 @@ static int __kprobes get_fixup_type(kprobe_opcode_t *insn) fixup |= FIXUP_RETURN_REGISTER; break; case 0xeb: - if ((insn[2] & 0xff) == 0x44 || /* bxhg */ - (insn[2] & 0xff) == 0x45) /* bxleg */ + switch (insn[2] & 0xff) { + case 0x44: /* bxhg */ + case 0x45: /* bxleg */ fixup = FIXUP_BRANCH_NOT_TAKEN; + break; + } break; case 0xe3: /* bctg */ if ((insn[2] & 0xff) == 0x46) fixup = FIXUP_BRANCH_NOT_TAKEN; break; + case 0xec: + switch (insn[2] & 0xff) { + case 0xe5: /* clgrb */ + case 0xe6: /* cgrb */ + case 0xf6: /* crb */ + case 0xf7: /* clrb */ + case 0xfc: /* cgib */ + case 0xfd: /* cglib */ + case 0xfe: /* cib */ + case 0xff: /* clib */ + fixup = FIXUP_BRANCH_NOT_TAKEN; + break; + } + break; } return fixup; } diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 504175ebf8b0..c4c033819879 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -214,10 +214,7 @@ static int notrace s390_revalidate_registers(struct mci *mci) : "0", "cc"); #endif /* Revalidate clock comparator register */ - if (S390_lowcore.clock_comparator == -1) - set_clock_comparator(S390_lowcore.mcck_clock); - else - set_clock_comparator(S390_lowcore.clock_comparator); + set_clock_comparator(S390_lowcore.clock_comparator); /* Check if old PSW is valid */ if (!mci->wp) /* diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index 2bc3eddae34a..c5dbb335716d 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -71,6 +71,7 @@ void arch_cpu_idle(void) } /* Halt the cpu and keep track of cpu time accounting. */ vtime_stop_cpu(); + local_irq_enable(); } void arch_cpu_idle_exit(void) diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index e9fadb04e3c6..9556905bd3ce 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -60,11 +60,11 @@ void update_cr_regs(struct task_struct *task) __ctl_store(cr, 0, 2); cr_new[1] = cr[1]; - /* Set or clear transaction execution TXC/PIFO bits 8 and 9. */ + /* Set or clear transaction execution TXC bit 8. */ if (task->thread.per_flags & PER_FLAG_NO_TE) - cr_new[0] = cr[0] & ~(3UL << 54); + cr_new[0] = cr[0] & ~(1UL << 55); else - cr_new[0] = cr[0] | (3UL << 54); + cr_new[0] = cr[0] | (1UL << 55); /* Set or clear transaction execution TDC bits 62 and 63. */ cr_new[2] = cr[2] & ~3UL; if (task->thread.per_flags & PER_FLAG_TE_ABORT_RAND) { @@ -1299,7 +1299,7 @@ int regs_query_register_offset(const char *name) if (!name || *name != 'r') return -EINVAL; - if (strict_strtoul(name + 1, 10, &offset)) + if (kstrtoul(name + 1, 10, &offset)) return -EINVAL; if (offset >= NUM_GPRS) return -EINVAL; diff --git a/arch/s390/kernel/suspend.c b/arch/s390/kernel/suspend.c index c479d2f9605b..737bff38e3ee 100644 --- a/arch/s390/kernel/suspend.c +++ b/arch/s390/kernel/suspend.c @@ -10,6 +10,9 @@ #include <linux/suspend.h> #include <linux/mm.h> #include <asm/ctl_reg.h> +#include <asm/ipl.h> +#include <asm/cio.h> +#include <asm/pci.h> /* * References to section boundaries @@ -211,3 +214,11 @@ void restore_processor_state(void) __ctl_set_bit(0,28); local_mcck_enable(); } + +/* Called at the end of swsusp_arch_resume */ +void s390_early_resume(void) +{ + lgr_info_log(); + channel_subsystem_reinit(); + zpci_rescan(); +} diff --git a/arch/s390/kernel/swsusp_asm64.S b/arch/s390/kernel/swsusp_asm64.S index c487be4cfc81..6b09fdffbd2f 100644 --- a/arch/s390/kernel/swsusp_asm64.S +++ b/arch/s390/kernel/swsusp_asm64.S @@ -281,11 +281,8 @@ restore_registers: lghi %r2,0 brasl %r14,arch_set_page_states - /* Log potential guest relocation */ - brasl %r14,lgr_info_log - - /* Reinitialize the channel subsystem */ - brasl %r14,channel_subsystem_reinit + /* Call arch specific early resume code */ + brasl %r14,s390_early_resume /* Return 0 */ lmg %r6,%r15,STACK_FRAME_OVERHEAD + __SF_GPRS(%r15) diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index 876546b9cfa1..064c3082ab33 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -92,7 +92,6 @@ void clock_comparator_work(void) struct clock_event_device *cd; S390_lowcore.clock_comparator = -1ULL; - set_clock_comparator(S390_lowcore.clock_comparator); cd = &__get_cpu_var(comparators); cd->event_handler(cd); } diff --git a/arch/s390/kernel/vdso.c b/arch/s390/kernel/vdso.c index d7776281cb60..05d75c413137 100644 --- a/arch/s390/kernel/vdso.c +++ b/arch/s390/kernel/vdso.c @@ -63,7 +63,7 @@ static int __init vdso_setup(char *s) else if (strncmp(s, "off", 4) == 0) vdso_enabled = 0; else { - rc = strict_strtoul(s, 0, &val); + rc = kstrtoul(s, 0, &val); vdso_enabled = rc ? 0 : !!val; } return !rc; @@ -113,11 +113,11 @@ int vdso_alloc_per_cpu(struct _lowcore *lowcore) clear_table((unsigned long *) segment_table, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE << SEGMENT_ORDER); - clear_table((unsigned long *) page_table, _PAGE_TYPE_EMPTY, + clear_table((unsigned long *) page_table, _PAGE_INVALID, 256*sizeof(unsigned long)); *(unsigned long *) segment_table = _SEGMENT_ENTRY + page_table; - *(unsigned long *) page_table = _PAGE_RO + page_frame; + *(unsigned long *) page_table = _PAGE_PROTECT + page_frame; psal = (u32 *) (page_table + 256*sizeof(unsigned long)); aste = psal + 32; diff --git a/arch/s390/lib/delay.c b/arch/s390/lib/delay.c index c61b9fad43cc..57c87d7d7ede 100644 --- a/arch/s390/lib/delay.c +++ b/arch/s390/lib/delay.c @@ -44,7 +44,6 @@ static void __udelay_disabled(unsigned long long usecs) do { set_clock_comparator(end); vtime_stop_cpu(); - local_irq_disable(); } while (get_tod_clock() < end); lockdep_on(); __ctl_load(cr0, 0, 0); @@ -64,7 +63,6 @@ static void __udelay_enabled(unsigned long long usecs) set_clock_comparator(end); } vtime_stop_cpu(); - local_irq_disable(); if (clock_saved) local_tick_enable(clock_saved); } while (get_tod_clock() < end); diff --git a/arch/s390/lib/uaccess_pt.c b/arch/s390/lib/uaccess_pt.c index 50ea137a2d3c..1694d738b175 100644 --- a/arch/s390/lib/uaccess_pt.c +++ b/arch/s390/lib/uaccess_pt.c @@ -86,28 +86,28 @@ static unsigned long follow_table(struct mm_struct *mm, switch (mm->context.asce_bits & _ASCE_TYPE_MASK) { case _ASCE_TYPE_REGION1: table = table + ((address >> 53) & 0x7ff); - if (unlikely(*table & _REGION_ENTRY_INV)) + if (unlikely(*table & _REGION_ENTRY_INVALID)) return -0x39UL; table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); /* fallthrough */ case _ASCE_TYPE_REGION2: table = table + ((address >> 42) & 0x7ff); - if (unlikely(*table & _REGION_ENTRY_INV)) + if (unlikely(*table & _REGION_ENTRY_INVALID)) return -0x3aUL; table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); /* fallthrough */ case _ASCE_TYPE_REGION3: table = table + ((address >> 31) & 0x7ff); - if (unlikely(*table & _REGION_ENTRY_INV)) + if (unlikely(*table & _REGION_ENTRY_INVALID)) return -0x3bUL; table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); /* fallthrough */ case _ASCE_TYPE_SEGMENT: table = table + ((address >> 20) & 0x7ff); - if (unlikely(*table & _SEGMENT_ENTRY_INV)) + if (unlikely(*table & _SEGMENT_ENTRY_INVALID)) return -0x10UL; if (unlikely(*table & _SEGMENT_ENTRY_LARGE)) { - if (write && (*table & _SEGMENT_ENTRY_RO)) + if (write && (*table & _SEGMENT_ENTRY_PROTECT)) return -0x04UL; return (*table & _SEGMENT_ENTRY_ORIGIN_LARGE) + (address & ~_SEGMENT_ENTRY_ORIGIN_LARGE); @@ -117,7 +117,7 @@ static unsigned long follow_table(struct mm_struct *mm, table = table + ((address >> 12) & 0xff); if (unlikely(*table & _PAGE_INVALID)) return -0x11UL; - if (write && (*table & _PAGE_RO)) + if (write && (*table & _PAGE_PROTECT)) return -0x04UL; return (*table & PAGE_MASK) + (address & ~PAGE_MASK); } @@ -130,13 +130,13 @@ static unsigned long follow_table(struct mm_struct *mm, unsigned long *table = (unsigned long *)__pa(mm->pgd); table = table + ((address >> 20) & 0x7ff); - if (unlikely(*table & _SEGMENT_ENTRY_INV)) + if (unlikely(*table & _SEGMENT_ENTRY_INVALID)) return -0x10UL; table = (unsigned long *)(*table & _SEGMENT_ENTRY_ORIGIN); table = table + ((address >> 12) & 0xff); if (unlikely(*table & _PAGE_INVALID)) return -0x11UL; - if (write && (*table & _PAGE_RO)) + if (write && (*table & _PAGE_PROTECT)) return -0x04UL; return (*table & PAGE_MASK) + (address & ~PAGE_MASK); } diff --git a/arch/s390/mm/dump_pagetables.c b/arch/s390/mm/dump_pagetables.c index 3ad65b04ac15..46d517c3c763 100644 --- a/arch/s390/mm/dump_pagetables.c +++ b/arch/s390/mm/dump_pagetables.c @@ -53,7 +53,7 @@ static void print_prot(struct seq_file *m, unsigned int pr, int level) seq_printf(m, "I\n"); return; } - seq_printf(m, "%s", pr & _PAGE_RO ? "RO " : "RW "); + seq_printf(m, "%s", pr & _PAGE_PROTECT ? "RO " : "RW "); seq_printf(m, "%s", pr & _PAGE_CO ? "CO " : " "); seq_putc(m, '\n'); } @@ -105,12 +105,12 @@ static void note_page(struct seq_file *m, struct pg_state *st, } /* - * The actual page table walker functions. In order to keep the implementation - * of print_prot() short, we only check and pass _PAGE_INVALID and _PAGE_RO - * flags to note_page() if a region, segment or page table entry is invalid or - * read-only. - * After all it's just a hint that the current level being walked contains an - * invalid or read-only entry. + * The actual page table walker functions. In order to keep the + * implementation of print_prot() short, we only check and pass + * _PAGE_INVALID and _PAGE_PROTECT flags to note_page() if a region, + * segment or page table entry is invalid or read-only. + * After all it's just a hint that the current level being walked + * contains an invalid or read-only entry. */ static void walk_pte_level(struct seq_file *m, struct pg_state *st, pmd_t *pmd, unsigned long addr) @@ -122,14 +122,14 @@ static void walk_pte_level(struct seq_file *m, struct pg_state *st, for (i = 0; i < PTRS_PER_PTE && addr < max_addr; i++) { st->current_address = addr; pte = pte_offset_kernel(pmd, addr); - prot = pte_val(*pte) & (_PAGE_RO | _PAGE_INVALID); + prot = pte_val(*pte) & (_PAGE_PROTECT | _PAGE_INVALID); note_page(m, st, prot, 4); addr += PAGE_SIZE; } } #ifdef CONFIG_64BIT -#define _PMD_PROT_MASK (_SEGMENT_ENTRY_RO | _SEGMENT_ENTRY_CO) +#define _PMD_PROT_MASK (_SEGMENT_ENTRY_PROTECT | _SEGMENT_ENTRY_CO) #else #define _PMD_PROT_MASK 0 #endif diff --git a/arch/s390/mm/gup.c b/arch/s390/mm/gup.c index 1f5315d1215c..5d758db27bdc 100644 --- a/arch/s390/mm/gup.c +++ b/arch/s390/mm/gup.c @@ -24,7 +24,7 @@ static inline int gup_pte_range(pmd_t *pmdp, pmd_t pmd, unsigned long addr, pte_t *ptep, pte; struct page *page; - mask = (write ? _PAGE_RO : 0) | _PAGE_INVALID | _PAGE_SPECIAL; + mask = (write ? _PAGE_PROTECT : 0) | _PAGE_INVALID | _PAGE_SPECIAL; ptep = ((pte_t *) pmd_deref(pmd)) + pte_index(addr); do { @@ -55,8 +55,8 @@ static inline int gup_huge_pmd(pmd_t *pmdp, pmd_t pmd, unsigned long addr, struct page *head, *page, *tail; int refs; - result = write ? 0 : _SEGMENT_ENTRY_RO; - mask = result | _SEGMENT_ENTRY_INV; + result = write ? 0 : _SEGMENT_ENTRY_PROTECT; + mask = result | _SEGMENT_ENTRY_INVALID; if ((pmd_val(pmd) & mask) != result) return 0; VM_BUG_ON(!pfn_valid(pmd_val(pmd) >> PAGE_SHIFT)); diff --git a/arch/s390/mm/hugetlbpage.c b/arch/s390/mm/hugetlbpage.c index 121089d57802..248445f92604 100644 --- a/arch/s390/mm/hugetlbpage.c +++ b/arch/s390/mm/hugetlbpage.c @@ -8,21 +8,127 @@ #include <linux/mm.h> #include <linux/hugetlb.h> +static inline pmd_t __pte_to_pmd(pte_t pte) +{ + int none, young, prot; + pmd_t pmd; + + /* + * Convert encoding pte bits pmd bits + * .IR...wrdytp ..R...I...y. + * empty .10...000000 -> ..0...1...0. + * prot-none, clean, old .11...000001 -> ..0...1...1. + * prot-none, clean, young .11...000101 -> ..1...1...1. + * prot-none, dirty, old .10...001001 -> ..0...1...1. + * prot-none, dirty, young .10...001101 -> ..1...1...1. + * read-only, clean, old .11...010001 -> ..1...1...0. + * read-only, clean, young .01...010101 -> ..1...0...1. + * read-only, dirty, old .11...011001 -> ..1...1...0. + * read-only, dirty, young .01...011101 -> ..1...0...1. + * read-write, clean, old .11...110001 -> ..0...1...0. + * read-write, clean, young .01...110101 -> ..0...0...1. + * read-write, dirty, old .10...111001 -> ..0...1...0. + * read-write, dirty, young .00...111101 -> ..0...0...1. + * Huge ptes are dirty by definition, a clean pte is made dirty + * by the conversion. + */ + if (pte_present(pte)) { + pmd_val(pmd) = pte_val(pte) & PAGE_MASK; + if (pte_val(pte) & _PAGE_INVALID) + pmd_val(pmd) |= _SEGMENT_ENTRY_INVALID; + none = (pte_val(pte) & _PAGE_PRESENT) && + !(pte_val(pte) & _PAGE_READ) && + !(pte_val(pte) & _PAGE_WRITE); + prot = (pte_val(pte) & _PAGE_PROTECT) && + !(pte_val(pte) & _PAGE_WRITE); + young = pte_val(pte) & _PAGE_YOUNG; + if (none || young) + pmd_val(pmd) |= _SEGMENT_ENTRY_YOUNG; + if (prot || (none && young)) + pmd_val(pmd) |= _SEGMENT_ENTRY_PROTECT; + } else + pmd_val(pmd) = _SEGMENT_ENTRY_INVALID; + return pmd; +} + +static inline pte_t __pmd_to_pte(pmd_t pmd) +{ + pte_t pte; + + /* + * Convert encoding pmd bits pte bits + * ..R...I...y. .IR...wrdytp + * empty ..0...1...0. -> .10...000000 + * prot-none, old ..0...1...1. -> .10...001001 + * prot-none, young ..1...1...1. -> .10...001101 + * read-only, old ..1...1...0. -> .11...011001 + * read-only, young ..1...0...1. -> .01...011101 + * read-write, old ..0...1...0. -> .10...111001 + * read-write, young ..0...0...1. -> .00...111101 + * Huge ptes are dirty by definition + */ + if (pmd_present(pmd)) { + pte_val(pte) = _PAGE_PRESENT | _PAGE_LARGE | _PAGE_DIRTY | + (pmd_val(pmd) & PAGE_MASK); + if (pmd_val(pmd) & _SEGMENT_ENTRY_INVALID) + pte_val(pte) |= _PAGE_INVALID; + if (pmd_prot_none(pmd)) { + if (pmd_val(pmd) & _SEGMENT_ENTRY_PROTECT) + pte_val(pte) |= _PAGE_YOUNG; + } else { + pte_val(pte) |= _PAGE_READ; + if (pmd_val(pmd) & _SEGMENT_ENTRY_PROTECT) + pte_val(pte) |= _PAGE_PROTECT; + else + pte_val(pte) |= _PAGE_WRITE; + if (pmd_val(pmd) & _SEGMENT_ENTRY_YOUNG) + pte_val(pte) |= _PAGE_YOUNG; + } + } else + pte_val(pte) = _PAGE_INVALID; + return pte; +} void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *pteptr, pte_t pteval) + pte_t *ptep, pte_t pte) { - pmd_t *pmdp = (pmd_t *) pteptr; - unsigned long mask; + pmd_t pmd; + pmd = __pte_to_pmd(pte); if (!MACHINE_HAS_HPAGE) { - pteptr = (pte_t *) pte_page(pteval)[1].index; - mask = pte_val(pteval) & - (_SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO); - pte_val(pteval) = (_SEGMENT_ENTRY + __pa(pteptr)) | mask; + pmd_val(pmd) &= ~_SEGMENT_ENTRY_ORIGIN; + pmd_val(pmd) |= pte_page(pte)[1].index; + } else + pmd_val(pmd) |= _SEGMENT_ENTRY_LARGE | _SEGMENT_ENTRY_CO; + *(pmd_t *) ptep = pmd; +} + +pte_t huge_ptep_get(pte_t *ptep) +{ + unsigned long origin; + pmd_t pmd; + + pmd = *(pmd_t *) ptep; + if (!MACHINE_HAS_HPAGE && pmd_present(pmd)) { + origin = pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN; + pmd_val(pmd) &= ~_SEGMENT_ENTRY_ORIGIN; + pmd_val(pmd) |= *(unsigned long *) origin; } + return __pmd_to_pte(pmd); +} - pmd_val(*pmdp) = pte_val(pteval); +pte_t huge_ptep_get_and_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + pmd_t *pmdp = (pmd_t *) ptep; + pte_t pte = huge_ptep_get(ptep); + + if (MACHINE_HAS_IDTE) + __pmd_idte(addr, pmdp); + else + __pmd_csp(pmdp); + pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY; + return pte; } int arch_prepare_hugepage(struct page *page) @@ -58,7 +164,7 @@ void arch_release_hugepage(struct page *page) ptep = (pte_t *) page[1].index; if (!ptep) return; - clear_table((unsigned long *) ptep, _PAGE_TYPE_EMPTY, + clear_table((unsigned long *) ptep, _PAGE_INVALID, PTRS_PER_PTE * sizeof(pte_t)); page_table_free(&init_mm, (unsigned long *) ptep); page[1].index = 0; diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c index 80adfbf75065..990397420e6b 100644 --- a/arch/s390/mm/pageattr.c +++ b/arch/s390/mm/pageattr.c @@ -118,7 +118,7 @@ void kernel_map_pages(struct page *page, int numpages, int enable) pte = pte_offset_kernel(pmd, address); if (!enable) { __ptep_ipte(address, pte); - pte_val(*pte) = _PAGE_TYPE_EMPTY; + pte_val(*pte) = _PAGE_INVALID; continue; } pte_val(*pte) = __pa(address); diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c index a8154a1a2c94..6d16132d0850 100644 --- a/arch/s390/mm/pgtable.c +++ b/arch/s390/mm/pgtable.c @@ -161,7 +161,7 @@ static int gmap_unlink_segment(struct gmap *gmap, unsigned long *table) struct gmap_rmap *rmap; struct page *page; - if (*table & _SEGMENT_ENTRY_INV) + if (*table & _SEGMENT_ENTRY_INVALID) return 0; page = pfn_to_page(*table >> PAGE_SHIFT); mp = (struct gmap_pgtable *) page->index; @@ -172,7 +172,7 @@ static int gmap_unlink_segment(struct gmap *gmap, unsigned long *table) kfree(rmap); break; } - *table = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO | mp->vmaddr; + *table = mp->vmaddr | _SEGMENT_ENTRY_INVALID | _SEGMENT_ENTRY_PROTECT; return 1; } @@ -258,7 +258,7 @@ static int gmap_alloc_table(struct gmap *gmap, return -ENOMEM; new = (unsigned long *) page_to_phys(page); crst_table_init(new, init); - if (*table & _REGION_ENTRY_INV) { + if (*table & _REGION_ENTRY_INVALID) { list_add(&page->lru, &gmap->crst_list); *table = (unsigned long) new | _REGION_ENTRY_LENGTH | (*table & _REGION_ENTRY_TYPE_MASK); @@ -292,22 +292,22 @@ int gmap_unmap_segment(struct gmap *gmap, unsigned long to, unsigned long len) for (off = 0; off < len; off += PMD_SIZE) { /* Walk the guest addr space page table */ table = gmap->table + (((to + off) >> 53) & 0x7ff); - if (*table & _REGION_ENTRY_INV) + if (*table & _REGION_ENTRY_INVALID) goto out; table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); table = table + (((to + off) >> 42) & 0x7ff); - if (*table & _REGION_ENTRY_INV) + if (*table & _REGION_ENTRY_INVALID) goto out; table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); table = table + (((to + off) >> 31) & 0x7ff); - if (*table & _REGION_ENTRY_INV) + if (*table & _REGION_ENTRY_INVALID) goto out; table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); table = table + (((to + off) >> 20) & 0x7ff); /* Clear segment table entry in guest address space. */ flush |= gmap_unlink_segment(gmap, table); - *table = _SEGMENT_ENTRY_INV; + *table = _SEGMENT_ENTRY_INVALID; } out: spin_unlock(&gmap->mm->page_table_lock); @@ -345,17 +345,17 @@ int gmap_map_segment(struct gmap *gmap, unsigned long from, for (off = 0; off < len; off += PMD_SIZE) { /* Walk the gmap address space page table */ table = gmap->table + (((to + off) >> 53) & 0x7ff); - if ((*table & _REGION_ENTRY_INV) && + if ((*table & _REGION_ENTRY_INVALID) && gmap_alloc_table(gmap, table, _REGION2_ENTRY_EMPTY)) goto out_unmap; table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); table = table + (((to + off) >> 42) & 0x7ff); - if ((*table & _REGION_ENTRY_INV) && + if ((*table & _REGION_ENTRY_INVALID) && gmap_alloc_table(gmap, table, _REGION3_ENTRY_EMPTY)) goto out_unmap; table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); table = table + (((to + off) >> 31) & 0x7ff); - if ((*table & _REGION_ENTRY_INV) && + if ((*table & _REGION_ENTRY_INVALID) && gmap_alloc_table(gmap, table, _SEGMENT_ENTRY_EMPTY)) goto out_unmap; table = (unsigned long *) (*table & _REGION_ENTRY_ORIGIN); @@ -363,7 +363,8 @@ int gmap_map_segment(struct gmap *gmap, unsigned long from, /* Store 'from' address in an invalid segment table entry. */ flush |= gmap_unlink_segment(gmap, table); - *table = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO | (from + off); + *table = (from + off) | (_SEGMENT_ENTRY_INVALID | + _SEGMENT_ENTRY_PROTECT); } spin_unlock(&gmap->mm->page_table_lock); up_read(&gmap->mm->mmap_sem); @@ -384,15 +385,15 @@ static unsigned long *gmap_table_walk(unsigned long address, struct gmap *gmap) unsigned long *table; table = gmap->table + ((address >> 53) & 0x7ff); - if (unlikely(*table & _REGION_ENTRY_INV)) + if (unlikely(*table & _REGION_ENTRY_INVALID)) return ERR_PTR(-EFAULT); table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); table = table + ((address >> 42) & 0x7ff); - if (unlikely(*table & _REGION_ENTRY_INV)) + if (unlikely(*table & _REGION_ENTRY_INVALID)) return ERR_PTR(-EFAULT); table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); table = table + ((address >> 31) & 0x7ff); - if (unlikely(*table & _REGION_ENTRY_INV)) + if (unlikely(*table & _REGION_ENTRY_INVALID)) return ERR_PTR(-EFAULT); table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); table = table + ((address >> 20) & 0x7ff); @@ -422,11 +423,11 @@ unsigned long __gmap_translate(unsigned long address, struct gmap *gmap) return PTR_ERR(segment_ptr); /* Convert the gmap address to an mm address. */ segment = *segment_ptr; - if (!(segment & _SEGMENT_ENTRY_INV)) { + if (!(segment & _SEGMENT_ENTRY_INVALID)) { page = pfn_to_page(segment >> PAGE_SHIFT); mp = (struct gmap_pgtable *) page->index; return mp->vmaddr | (address & ~PMD_MASK); - } else if (segment & _SEGMENT_ENTRY_RO) { + } else if (segment & _SEGMENT_ENTRY_PROTECT) { vmaddr = segment & _SEGMENT_ENTRY_ORIGIN; return vmaddr | (address & ~PMD_MASK); } @@ -517,8 +518,8 @@ static void gmap_disconnect_pgtable(struct mm_struct *mm, unsigned long *table) page = pfn_to_page(__pa(table) >> PAGE_SHIFT); mp = (struct gmap_pgtable *) page->index; list_for_each_entry_safe(rmap, next, &mp->mapper, list) { - *rmap->entry = - _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO | mp->vmaddr; + *rmap->entry = mp->vmaddr | (_SEGMENT_ENTRY_INVALID | + _SEGMENT_ENTRY_PROTECT); list_del(&rmap->list); kfree(rmap); flush = 1; @@ -545,13 +546,13 @@ unsigned long __gmap_fault(unsigned long address, struct gmap *gmap) /* Convert the gmap address to an mm address. */ while (1) { segment = *segment_ptr; - if (!(segment & _SEGMENT_ENTRY_INV)) { + if (!(segment & _SEGMENT_ENTRY_INVALID)) { /* Page table is present */ page = pfn_to_page(segment >> PAGE_SHIFT); mp = (struct gmap_pgtable *) page->index; return mp->vmaddr | (address & ~PMD_MASK); } - if (!(segment & _SEGMENT_ENTRY_RO)) + if (!(segment & _SEGMENT_ENTRY_PROTECT)) /* Nothing mapped in the gmap address space. */ break; rc = gmap_connect_pgtable(address, segment, segment_ptr, gmap); @@ -586,25 +587,25 @@ void gmap_discard(unsigned long from, unsigned long to, struct gmap *gmap) while (address < to) { /* Walk the gmap address space page table */ table = gmap->table + ((address >> 53) & 0x7ff); - if (unlikely(*table & _REGION_ENTRY_INV)) { + if (unlikely(*table & _REGION_ENTRY_INVALID)) { address = (address + PMD_SIZE) & PMD_MASK; continue; } table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); table = table + ((address >> 42) & 0x7ff); - if (unlikely(*table & _REGION_ENTRY_INV)) { + if (unlikely(*table & _REGION_ENTRY_INVALID)) { address = (address + PMD_SIZE) & PMD_MASK; continue; } table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); table = table + ((address >> 31) & 0x7ff); - if (unlikely(*table & _REGION_ENTRY_INV)) { + if (unlikely(*table & _REGION_ENTRY_INVALID)) { address = (address + PMD_SIZE) & PMD_MASK; continue; } table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); table = table + ((address >> 20) & 0x7ff); - if (unlikely(*table & _SEGMENT_ENTRY_INV)) { + if (unlikely(*table & _SEGMENT_ENTRY_INVALID)) { address = (address + PMD_SIZE) & PMD_MASK; continue; } @@ -687,7 +688,7 @@ int gmap_ipte_notify(struct gmap *gmap, unsigned long start, unsigned long len) continue; /* Set notification bit in the pgste of the pte */ entry = *ptep; - if ((pte_val(entry) & (_PAGE_INVALID | _PAGE_RO)) == 0) { + if ((pte_val(entry) & (_PAGE_INVALID | _PAGE_PROTECT)) == 0) { pgste = pgste_get_lock(ptep); pgste_val(pgste) |= PGSTE_IN_BIT; pgste_set_unlock(ptep, pgste); @@ -752,8 +753,9 @@ static inline unsigned long *page_table_alloc_pgste(struct mm_struct *mm, page->index = (unsigned long) mp; atomic_set(&page->_mapcount, 3); table = (unsigned long *) page_to_phys(page); - clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE/2); - clear_table(table + PTRS_PER_PTE, 0, PAGE_SIZE/2); + clear_table(table, _PAGE_INVALID, PAGE_SIZE/2); + clear_table(table + PTRS_PER_PTE, PGSTE_HR_BIT | PGSTE_HC_BIT, + PAGE_SIZE/2); return table; } @@ -791,26 +793,21 @@ int set_guest_storage_key(struct mm_struct *mm, unsigned long addr, pgste_val(new) |= (key & (_PAGE_CHANGED | _PAGE_REFERENCED)) << 48; pgste_val(new) |= (key & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56; if (!(pte_val(*ptep) & _PAGE_INVALID)) { - unsigned long address, bits; - unsigned char skey; + unsigned long address, bits, skey; address = pte_val(*ptep) & PAGE_MASK; - skey = page_get_storage_key(address); + skey = (unsigned long) page_get_storage_key(address); bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED); + skey = key & (_PAGE_ACC_BITS | _PAGE_FP_BIT); /* Set storage key ACC and FP */ - page_set_storage_key(address, - (key & (_PAGE_ACC_BITS | _PAGE_FP_BIT)), - !nq); - + page_set_storage_key(address, skey, !nq); /* Merge host changed & referenced into pgste */ pgste_val(new) |= bits << 52; - /* Transfer skey changed & referenced bit to kvm user bits */ - pgste_val(new) |= bits << 45; /* PGSTE_UR_BIT & PGSTE_UC_BIT */ } /* changing the guest storage key is considered a change of the page */ if ((pgste_val(new) ^ pgste_val(old)) & (PGSTE_ACC_BITS | PGSTE_FP_BIT | PGSTE_GR_BIT | PGSTE_GC_BIT)) - pgste_val(new) |= PGSTE_UC_BIT; + pgste_val(new) |= PGSTE_HC_BIT; pgste_set_unlock(ptep, new); pte_unmap_unlock(*ptep, ptl); @@ -878,7 +875,7 @@ unsigned long *page_table_alloc(struct mm_struct *mm, unsigned long vmaddr) pgtable_page_ctor(page); atomic_set(&page->_mapcount, 1); table = (unsigned long *) page_to_phys(page); - clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE); + clear_table(table, _PAGE_INVALID, PAGE_SIZE); spin_lock_bh(&mm->context.list_lock); list_add(&page->lru, &mm->context.pgtable_list); } else { @@ -1007,7 +1004,6 @@ void tlb_table_flush(struct mmu_gather *tlb) struct mmu_table_batch **batch = &tlb->batch; if (*batch) { - __tlb_flush_mm(tlb->mm); call_rcu_sched(&(*batch)->rcu, tlb_remove_table_rcu); *batch = NULL; } @@ -1017,11 +1013,12 @@ void tlb_remove_table(struct mmu_gather *tlb, void *table) { struct mmu_table_batch **batch = &tlb->batch; + tlb->mm->context.flush_mm = 1; if (*batch == NULL) { *batch = (struct mmu_table_batch *) __get_free_page(GFP_NOWAIT | __GFP_NOWARN); if (*batch == NULL) { - __tlb_flush_mm(tlb->mm); + __tlb_flush_mm_lazy(tlb->mm); tlb_remove_table_one(table); return; } @@ -1029,7 +1026,7 @@ void tlb_remove_table(struct mmu_gather *tlb, void *table) } (*batch)->tables[(*batch)->nr++] = table; if ((*batch)->nr == MAX_TABLE_BATCH) - tlb_table_flush(tlb); + tlb_flush_mmu(tlb); } #ifdef CONFIG_TRANSPARENT_HUGEPAGE @@ -1198,9 +1195,9 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) list_del(lh); } ptep = (pte_t *) pgtable; - pte_val(*ptep) = _PAGE_TYPE_EMPTY; + pte_val(*ptep) = _PAGE_INVALID; ptep++; - pte_val(*ptep) = _PAGE_TYPE_EMPTY; + pte_val(*ptep) = _PAGE_INVALID; return pgtable; } #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 8b268fcc4612..bcfb70b60be6 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -69,7 +69,7 @@ static pte_t __ref *vmem_pte_alloc(unsigned long address) pte = alloc_bootmem(PTRS_PER_PTE * sizeof(pte_t)); if (!pte) return NULL; - clear_table((unsigned long *) pte, _PAGE_TYPE_EMPTY, + clear_table((unsigned long *) pte, _PAGE_INVALID, PTRS_PER_PTE * sizeof(pte_t)); return pte; } @@ -101,7 +101,7 @@ static int vmem_add_mem(unsigned long start, unsigned long size, int ro) !(address & ~PUD_MASK) && (address + PUD_SIZE <= end)) { pud_val(*pu_dir) = __pa(address) | _REGION_ENTRY_TYPE_R3 | _REGION3_ENTRY_LARGE | - (ro ? _REGION_ENTRY_RO : 0); + (ro ? _REGION_ENTRY_PROTECT : 0); address += PUD_SIZE; continue; } @@ -118,7 +118,8 @@ static int vmem_add_mem(unsigned long start, unsigned long size, int ro) !(address & ~PMD_MASK) && (address + PMD_SIZE <= end)) { pmd_val(*pm_dir) = __pa(address) | _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE | - (ro ? _SEGMENT_ENTRY_RO : 0); + _SEGMENT_ENTRY_YOUNG | + (ro ? _SEGMENT_ENTRY_PROTECT : 0); address += PMD_SIZE; continue; } @@ -131,7 +132,8 @@ static int vmem_add_mem(unsigned long start, unsigned long size, int ro) } pt_dir = pte_offset_kernel(pm_dir, address); - pte_val(*pt_dir) = __pa(address) | (ro ? _PAGE_RO : 0); + pte_val(*pt_dir) = __pa(address) | + pgprot_val(ro ? PAGE_KERNEL_RO : PAGE_KERNEL); address += PAGE_SIZE; } ret = 0; @@ -154,7 +156,7 @@ static void vmem_remove_range(unsigned long start, unsigned long size) pte_t *pt_dir; pte_t pte; - pte_val(pte) = _PAGE_TYPE_EMPTY; + pte_val(pte) = _PAGE_INVALID; while (address < end) { pg_dir = pgd_offset_k(address); if (pgd_none(*pg_dir)) { @@ -255,7 +257,8 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) new_page =__pa(vmem_alloc_pages(0)); if (!new_page) goto out; - pte_val(*pt_dir) = __pa(new_page); + pte_val(*pt_dir) = + __pa(new_page) | pgprot_val(PAGE_KERNEL); } address += PAGE_SIZE; } diff --git a/arch/s390/pci/Makefile b/arch/s390/pci/Makefile index 086a2e37935d..a9e1dc4ae442 100644 --- a/arch/s390/pci/Makefile +++ b/arch/s390/pci/Makefile @@ -2,5 +2,5 @@ # Makefile for the s390 PCI subsystem. # -obj-$(CONFIG_PCI) += pci.o pci_dma.o pci_clp.o pci_msi.o pci_sysfs.o \ +obj-$(CONFIG_PCI) += pci.o pci_dma.o pci_clp.o pci_sysfs.o \ pci_event.o pci_debug.o pci_insn.o diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index e2956ad39a4f..f17a8343e360 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -42,45 +42,26 @@ #define SIC_IRQ_MODE_SINGLE 1 #define ZPCI_NR_DMA_SPACES 1 -#define ZPCI_MSI_VEC_BITS 6 #define ZPCI_NR_DEVICES CONFIG_PCI_NR_FUNCTIONS /* list of all detected zpci devices */ -LIST_HEAD(zpci_list); -EXPORT_SYMBOL_GPL(zpci_list); -DEFINE_MUTEX(zpci_list_lock); -EXPORT_SYMBOL_GPL(zpci_list_lock); +static LIST_HEAD(zpci_list); +static DEFINE_SPINLOCK(zpci_list_lock); -static struct pci_hp_callback_ops *hotplug_ops; +static void zpci_enable_irq(struct irq_data *data); +static void zpci_disable_irq(struct irq_data *data); -static DECLARE_BITMAP(zpci_domain, ZPCI_NR_DEVICES); -static DEFINE_SPINLOCK(zpci_domain_lock); - -struct callback { - irq_handler_t handler; - void *data; +static struct irq_chip zpci_irq_chip = { + .name = "zPCI", + .irq_unmask = zpci_enable_irq, + .irq_mask = zpci_disable_irq, }; -struct zdev_irq_map { - unsigned long aibv; /* AI bit vector */ - int msi_vecs; /* consecutive MSI-vectors used */ - int __unused; - struct callback cb[ZPCI_NR_MSI_VECS]; /* callback handler array */ - spinlock_t lock; /* protect callbacks against de-reg */ -}; - -struct intr_bucket { - /* amap of adapters, one bit per dev, corresponds to one irq nr */ - unsigned long *alloc; - /* AI summary bit, global page for all devices */ - unsigned long *aisb; - /* pointer to aibv and callback data in zdev */ - struct zdev_irq_map *imap[ZPCI_NR_DEVICES]; - /* protects the whole bucket struct */ - spinlock_t lock; -}; +static DECLARE_BITMAP(zpci_domain, ZPCI_NR_DEVICES); +static DEFINE_SPINLOCK(zpci_domain_lock); -static struct intr_bucket *bucket; +static struct airq_iv *zpci_aisb_iv; +static struct airq_iv *zpci_aibv[ZPCI_NR_DEVICES]; /* Adapter interrupt definitions */ static void zpci_irq_handler(struct airq_struct *airq); @@ -96,27 +77,8 @@ static DECLARE_BITMAP(zpci_iomap, ZPCI_IOMAP_MAX_ENTRIES); struct zpci_iomap_entry *zpci_iomap_start; EXPORT_SYMBOL_GPL(zpci_iomap_start); -/* highest irq summary bit */ -static int __read_mostly aisb_max; - -static struct kmem_cache *zdev_irq_cache; static struct kmem_cache *zdev_fmb_cache; -static inline int irq_to_msi_nr(unsigned int irq) -{ - return irq & ZPCI_MSI_MASK; -} - -static inline int irq_to_dev_nr(unsigned int irq) -{ - return irq >> ZPCI_MSI_VEC_BITS; -} - -static inline struct zdev_irq_map *get_imap(unsigned int irq) -{ - return bucket->imap[irq_to_dev_nr(irq)]; -} - struct zpci_dev *get_zdev(struct pci_dev *pdev) { return (struct zpci_dev *) pdev->sysdata; @@ -126,22 +88,17 @@ struct zpci_dev *get_zdev_by_fid(u32 fid) { struct zpci_dev *tmp, *zdev = NULL; - mutex_lock(&zpci_list_lock); + spin_lock(&zpci_list_lock); list_for_each_entry(tmp, &zpci_list, entry) { if (tmp->fid == fid) { zdev = tmp; break; } } - mutex_unlock(&zpci_list_lock); + spin_unlock(&zpci_list_lock); return zdev; } -bool zpci_fid_present(u32 fid) -{ - return (get_zdev_by_fid(fid) != NULL) ? true : false; -} - static struct zpci_dev *get_zdev_by_bus(struct pci_bus *bus) { return (bus && bus->sysdata) ? (struct zpci_dev *) bus->sysdata : NULL; @@ -160,8 +117,7 @@ int pci_proc_domain(struct pci_bus *bus) EXPORT_SYMBOL_GPL(pci_proc_domain); /* Modify PCI: Register adapter interruptions */ -static int zpci_register_airq(struct zpci_dev *zdev, unsigned int aisb, - u64 aibv) +static int zpci_set_airq(struct zpci_dev *zdev) { u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_REG_INT); struct zpci_fib *fib; @@ -172,14 +128,14 @@ static int zpci_register_airq(struct zpci_dev *zdev, unsigned int aisb, return -ENOMEM; fib->isc = PCI_ISC; - fib->noi = zdev->irq_map->msi_vecs; fib->sum = 1; /* enable summary notifications */ - fib->aibv = aibv; - fib->aibvo = 0; /* every function has its own page */ - fib->aisb = (u64) bucket->aisb + aisb / 8; - fib->aisbo = aisb & ZPCI_MSI_MASK; + fib->noi = airq_iv_end(zdev->aibv); + fib->aibv = (unsigned long) zdev->aibv->vector; + fib->aibvo = 0; /* each zdev has its own interrupt vector */ + fib->aisb = (unsigned long) zpci_aisb_iv->vector + (zdev->aisb/64)*8; + fib->aisbo = zdev->aisb & 63; - rc = s390pci_mod_fc(req, fib); + rc = zpci_mod_fc(req, fib); pr_debug("%s mpcifc returned noi: %d\n", __func__, fib->noi); free_page((unsigned long) fib); @@ -209,7 +165,7 @@ static int mod_pci(struct zpci_dev *zdev, int fn, u8 dmaas, struct mod_pci_args fib->iota = args->iota; fib->fmb_addr = args->fmb_addr; - rc = s390pci_mod_fc(req, fib); + rc = zpci_mod_fc(req, fib); free_page((unsigned long) fib); return rc; } @@ -234,7 +190,7 @@ int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas) } /* Modify PCI: Unregister adapter interruptions */ -static int zpci_unregister_airq(struct zpci_dev *zdev) +static int zpci_clear_airq(struct zpci_dev *zdev) { struct mod_pci_args args = { 0, 0, 0, 0 }; @@ -283,7 +239,7 @@ static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len) u64 data; int rc; - rc = s390pci_load(&data, req, offset); + rc = zpci_load(&data, req, offset); if (!rc) { data = data << ((8 - len) * 8); data = le64_to_cpu(data); @@ -301,25 +257,46 @@ static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len) data = cpu_to_le64(data); data = data >> ((8 - len) * 8); - rc = s390pci_store(data, req, offset); + rc = zpci_store(data, req, offset); return rc; } -void enable_irq(unsigned int irq) +static int zpci_msi_set_mask_bits(struct msi_desc *msi, u32 mask, u32 flag) +{ + int offset, pos; + u32 mask_bits; + + if (msi->msi_attrib.is_msix) { + offset = msi->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + + PCI_MSIX_ENTRY_VECTOR_CTRL; + msi->masked = readl(msi->mask_base + offset); + writel(flag, msi->mask_base + offset); + } else if (msi->msi_attrib.maskbit) { + pos = (long) msi->mask_base; + pci_read_config_dword(msi->dev, pos, &mask_bits); + mask_bits &= ~(mask); + mask_bits |= flag & mask; + pci_write_config_dword(msi->dev, pos, mask_bits); + } else + return 0; + + msi->msi_attrib.maskbit = !!flag; + return 1; +} + +static void zpci_enable_irq(struct irq_data *data) { - struct msi_desc *msi = irq_get_msi_desc(irq); + struct msi_desc *msi = irq_get_msi_desc(data->irq); zpci_msi_set_mask_bits(msi, 1, 0); } -EXPORT_SYMBOL_GPL(enable_irq); -void disable_irq(unsigned int irq) +static void zpci_disable_irq(struct irq_data *data) { - struct msi_desc *msi = irq_get_msi_desc(irq); + struct msi_desc *msi = irq_get_msi_desc(data->irq); zpci_msi_set_mask_bits(msi, 1, 1); } -EXPORT_SYMBOL_GPL(disable_irq); void pcibios_fixup_bus(struct pci_bus *bus) { @@ -404,152 +381,147 @@ static struct pci_ops pci_root_ops = { .write = pci_write, }; -/* store the last handled bit to implement fair scheduling of devices */ -static DEFINE_PER_CPU(unsigned long, next_sbit); - static void zpci_irq_handler(struct airq_struct *airq) { - unsigned long sbit, mbit, last = 0, start = __get_cpu_var(next_sbit); - int rescan = 0, max = aisb_max; - struct zdev_irq_map *imap; + unsigned long si, ai; + struct airq_iv *aibv; + int irqs_on = 0; inc_irq_stat(IRQIO_PCI); - sbit = start; - -scan: - /* find summary_bit */ - for_each_set_bit_left_cont(sbit, bucket->aisb, max) { - clear_bit(63 - (sbit & 63), bucket->aisb + (sbit >> 6)); - last = sbit; + for (si = 0;;) { + /* Scan adapter summary indicator bit vector */ + si = airq_iv_scan(zpci_aisb_iv, si, airq_iv_end(zpci_aisb_iv)); + if (si == -1UL) { + if (irqs_on++) + /* End of second scan with interrupts on. */ + break; + /* First scan complete, reenable interrupts. */ + zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC); + si = 0; + continue; + } - /* find vector bit */ - imap = bucket->imap[sbit]; - for_each_set_bit_left(mbit, &imap->aibv, imap->msi_vecs) { + /* Scan the adapter interrupt vector for this device. */ + aibv = zpci_aibv[si]; + for (ai = 0;;) { + ai = airq_iv_scan(aibv, ai, airq_iv_end(aibv)); + if (ai == -1UL) + break; inc_irq_stat(IRQIO_MSI); - clear_bit(63 - mbit, &imap->aibv); - - spin_lock(&imap->lock); - if (imap->cb[mbit].handler) - imap->cb[mbit].handler(mbit, - imap->cb[mbit].data); - spin_unlock(&imap->lock); + airq_iv_lock(aibv, ai); + generic_handle_irq(airq_iv_get_data(aibv, ai)); + airq_iv_unlock(aibv, ai); } } - - if (rescan) - goto out; - - /* scan the skipped bits */ - if (start > 0) { - sbit = 0; - max = start; - start = 0; - goto scan; - } - - /* enable interrupts again */ - set_irq_ctrl(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC); - - /* check again to not lose initiative */ - rmb(); - max = aisb_max; - sbit = find_first_bit_left(bucket->aisb, max); - if (sbit != max) { - rescan++; - goto scan; - } -out: - /* store next device bit to scan */ - __get_cpu_var(next_sbit) = (++last >= aisb_max) ? 0 : last; } -/* msi_vecs - number of requested interrupts, 0 place function to error state */ -static int zpci_setup_msi(struct pci_dev *pdev, int msi_vecs) +int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) { struct zpci_dev *zdev = get_zdev(pdev); - unsigned int aisb, msi_nr; + unsigned int hwirq, irq, msi_vecs; + unsigned long aisb; struct msi_desc *msi; + struct msi_msg msg; int rc; - /* store the number of used MSI vectors */ - zdev->irq_map->msi_vecs = min(msi_vecs, ZPCI_NR_MSI_VECS); - - spin_lock(&bucket->lock); - aisb = find_first_zero_bit(bucket->alloc, PAGE_SIZE); - /* alloc map exhausted? */ - if (aisb == PAGE_SIZE) { - spin_unlock(&bucket->lock); - return -EIO; - } - set_bit(aisb, bucket->alloc); - spin_unlock(&bucket->lock); + pr_debug("%s: requesting %d MSI-X interrupts...", __func__, nvec); + if (type != PCI_CAP_ID_MSIX && type != PCI_CAP_ID_MSI) + return -EINVAL; + msi_vecs = min(nvec, ZPCI_MSI_VEC_MAX); + msi_vecs = min_t(unsigned int, msi_vecs, CONFIG_PCI_NR_MSI); + /* Allocate adapter summary indicator bit */ + rc = -EIO; + aisb = airq_iv_alloc_bit(zpci_aisb_iv); + if (aisb == -1UL) + goto out; zdev->aisb = aisb; - if (aisb + 1 > aisb_max) - aisb_max = aisb + 1; - /* wire up IRQ shortcut pointer */ - bucket->imap[zdev->aisb] = zdev->irq_map; - pr_debug("%s: imap[%u] linked to %p\n", __func__, zdev->aisb, zdev->irq_map); + /* Create adapter interrupt vector */ + rc = -ENOMEM; + zdev->aibv = airq_iv_create(msi_vecs, AIRQ_IV_DATA | AIRQ_IV_BITLOCK); + if (!zdev->aibv) + goto out_si; - /* TODO: irq number 0 wont be found if we return less than requested MSIs. - * ignore it for now and fix in common code. - */ - msi_nr = aisb << ZPCI_MSI_VEC_BITS; + /* Wire up shortcut pointer */ + zpci_aibv[aisb] = zdev->aibv; + /* Request MSI interrupts */ + hwirq = 0; list_for_each_entry(msi, &pdev->msi_list, list) { - rc = zpci_setup_msi_irq(zdev, msi, msi_nr, - aisb << ZPCI_MSI_VEC_BITS); + rc = -EIO; + irq = irq_alloc_desc(0); /* Alloc irq on node 0 */ + if (irq == NO_IRQ) + goto out_msi; + rc = irq_set_msi_desc(irq, msi); if (rc) - return rc; - msi_nr++; + goto out_msi; + irq_set_chip_and_handler(irq, &zpci_irq_chip, + handle_simple_irq); + msg.data = hwirq; + msg.address_lo = zdev->msi_addr & 0xffffffff; + msg.address_hi = zdev->msi_addr >> 32; + write_msi_msg(irq, &msg); + airq_iv_set_data(zdev->aibv, hwirq, irq); + hwirq++; } - rc = zpci_register_airq(zdev, aisb, (u64) &zdev->irq_map->aibv); - if (rc) { - clear_bit(aisb, bucket->alloc); - dev_err(&pdev->dev, "register MSI failed with: %d\n", rc); - return rc; + /* Enable adapter interrupts */ + rc = zpci_set_airq(zdev); + if (rc) + goto out_msi; + + return (msi_vecs == nvec) ? 0 : msi_vecs; + +out_msi: + list_for_each_entry(msi, &pdev->msi_list, list) { + if (hwirq-- == 0) + break; + irq_set_msi_desc(msi->irq, NULL); + irq_free_desc(msi->irq); + msi->msg.address_lo = 0; + msi->msg.address_hi = 0; + msi->msg.data = 0; + msi->irq = 0; } - return (zdev->irq_map->msi_vecs == msi_vecs) ? - 0 : zdev->irq_map->msi_vecs; + zpci_aibv[aisb] = NULL; + airq_iv_release(zdev->aibv); +out_si: + airq_iv_free_bit(zpci_aisb_iv, aisb); +out: + dev_err(&pdev->dev, "register MSI failed with: %d\n", rc); + return rc; } -static void zpci_teardown_msi(struct pci_dev *pdev) +void arch_teardown_msi_irqs(struct pci_dev *pdev) { struct zpci_dev *zdev = get_zdev(pdev); struct msi_desc *msi; - int aisb, rc; + int rc; - rc = zpci_unregister_airq(zdev); + pr_info("%s: on pdev: %p\n", __func__, pdev); + + /* Disable adapter interrupts */ + rc = zpci_clear_airq(zdev); if (rc) { dev_err(&pdev->dev, "deregister MSI failed with: %d\n", rc); return; } - msi = list_first_entry(&pdev->msi_list, struct msi_desc, list); - aisb = irq_to_dev_nr(msi->irq); - - list_for_each_entry(msi, &pdev->msi_list, list) - zpci_teardown_msi_irq(zdev, msi); - - clear_bit(aisb, bucket->alloc); - if (aisb + 1 == aisb_max) - aisb_max--; -} - -int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) -{ - pr_debug("%s: requesting %d MSI-X interrupts...", __func__, nvec); - if (type != PCI_CAP_ID_MSIX && type != PCI_CAP_ID_MSI) - return -EINVAL; - return zpci_setup_msi(pdev, nvec); -} + /* Release MSI interrupts */ + list_for_each_entry(msi, &pdev->msi_list, list) { + zpci_msi_set_mask_bits(msi, 1, 1); + irq_set_msi_desc(msi->irq, NULL); + irq_free_desc(msi->irq); + msi->msg.address_lo = 0; + msi->msg.address_hi = 0; + msi->msg.data = 0; + msi->irq = 0; + } -void arch_teardown_msi_irqs(struct pci_dev *pdev) -{ - pr_info("%s: on pdev: %p\n", __func__, pdev); - zpci_teardown_msi(pdev); + zpci_aibv[zdev->aisb] = NULL; + airq_iv_release(zdev->aibv); + airq_iv_free_bit(zpci_aisb_iv, zdev->aisb); } static void zpci_map_resources(struct zpci_dev *zdev) @@ -564,8 +536,6 @@ static void zpci_map_resources(struct zpci_dev *zdev) continue; pdev->resource[i].start = (resource_size_t) pci_iomap(pdev, i, 0); pdev->resource[i].end = pdev->resource[i].start + len - 1; - pr_debug("BAR%i: -> start: %Lx end: %Lx\n", - i, pdev->resource[i].start, pdev->resource[i].end); } } @@ -589,162 +559,47 @@ struct zpci_dev *zpci_alloc_device(void) /* Alloc memory for our private pci device data */ zdev = kzalloc(sizeof(*zdev), GFP_KERNEL); - if (!zdev) - return ERR_PTR(-ENOMEM); - - /* Alloc aibv & callback space */ - zdev->irq_map = kmem_cache_zalloc(zdev_irq_cache, GFP_KERNEL); - if (!zdev->irq_map) - goto error; - WARN_ON((u64) zdev->irq_map & 0xff); - return zdev; - -error: - kfree(zdev); - return ERR_PTR(-ENOMEM); + return zdev ? : ERR_PTR(-ENOMEM); } void zpci_free_device(struct zpci_dev *zdev) { - kmem_cache_free(zdev_irq_cache, zdev->irq_map); kfree(zdev); } -/* - * Too late for any s390 specific setup, since interrupts must be set up - * already which requires DMA setup too and the pci scan will access the - * config space, which only works if the function handle is enabled. - */ -int pcibios_enable_device(struct pci_dev *pdev, int mask) -{ - struct resource *res; - u16 cmd; - int i; - - pci_read_config_word(pdev, PCI_COMMAND, &cmd); - - for (i = 0; i < PCI_BAR_COUNT; i++) { - res = &pdev->resource[i]; - - if (res->flags & IORESOURCE_IO) - return -EINVAL; - - if (res->flags & IORESOURCE_MEM) - cmd |= PCI_COMMAND_MEMORY; - } - pci_write_config_word(pdev, PCI_COMMAND, cmd); - return 0; -} - int pcibios_add_platform_entries(struct pci_dev *pdev) { return zpci_sysfs_add_device(&pdev->dev); } -int zpci_request_irq(unsigned int irq, irq_handler_t handler, void *data) -{ - int msi_nr = irq_to_msi_nr(irq); - struct zdev_irq_map *imap; - struct msi_desc *msi; - - msi = irq_get_msi_desc(irq); - if (!msi) - return -EIO; - - imap = get_imap(irq); - spin_lock_init(&imap->lock); - - pr_debug("%s: register handler for IRQ:MSI %d:%d\n", __func__, irq >> 6, msi_nr); - imap->cb[msi_nr].handler = handler; - imap->cb[msi_nr].data = data; - - /* - * The generic MSI code returns with the interrupt disabled on the - * card, using the MSI mask bits. Firmware doesn't appear to unmask - * at that level, so we do it here by hand. - */ - zpci_msi_set_mask_bits(msi, 1, 0); - return 0; -} - -void zpci_free_irq(unsigned int irq) -{ - struct zdev_irq_map *imap = get_imap(irq); - int msi_nr = irq_to_msi_nr(irq); - unsigned long flags; - - pr_debug("%s: for irq: %d\n", __func__, irq); - - spin_lock_irqsave(&imap->lock, flags); - imap->cb[msi_nr].handler = NULL; - imap->cb[msi_nr].data = NULL; - spin_unlock_irqrestore(&imap->lock, flags); -} - -int request_irq(unsigned int irq, irq_handler_t handler, - unsigned long irqflags, const char *devname, void *dev_id) -{ - pr_debug("%s: irq: %d handler: %p flags: %lx dev: %s\n", - __func__, irq, handler, irqflags, devname); - - return zpci_request_irq(irq, handler, dev_id); -} -EXPORT_SYMBOL_GPL(request_irq); - -void free_irq(unsigned int irq, void *dev_id) -{ - zpci_free_irq(irq); -} -EXPORT_SYMBOL_GPL(free_irq); - static int __init zpci_irq_init(void) { - int cpu, rc; - - bucket = kzalloc(sizeof(*bucket), GFP_KERNEL); - if (!bucket) - return -ENOMEM; - - bucket->aisb = (unsigned long *) get_zeroed_page(GFP_KERNEL); - if (!bucket->aisb) { - rc = -ENOMEM; - goto out_aisb; - } - - bucket->alloc = (unsigned long *) get_zeroed_page(GFP_KERNEL); - if (!bucket->alloc) { - rc = -ENOMEM; - goto out_alloc; - } + int rc; rc = register_adapter_interrupt(&zpci_airq); if (rc) - goto out_ai; + goto out; /* Set summary to 1 to be called every time for the ISC. */ *zpci_airq.lsi_ptr = 1; - for_each_online_cpu(cpu) - per_cpu(next_sbit, cpu) = 0; + rc = -ENOMEM; + zpci_aisb_iv = airq_iv_create(ZPCI_NR_DEVICES, AIRQ_IV_ALLOC); + if (!zpci_aisb_iv) + goto out_airq; - spin_lock_init(&bucket->lock); - set_irq_ctrl(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC); + zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC); return 0; -out_ai: - free_page((unsigned long) bucket->alloc); -out_alloc: - free_page((unsigned long) bucket->aisb); -out_aisb: - kfree(bucket); +out_airq: + unregister_adapter_interrupt(&zpci_airq); +out: return rc; } static void zpci_irq_exit(void) { - free_page((unsigned long) bucket->alloc); - free_page((unsigned long) bucket->aisb); + airq_iv_release(zpci_aisb_iv); unregister_adapter_interrupt(&zpci_airq); - kfree(bucket); } static struct resource *zpci_alloc_bus_resource(unsigned long start, unsigned long size, @@ -801,16 +656,49 @@ static void zpci_free_iomap(struct zpci_dev *zdev, int entry) int pcibios_add_device(struct pci_dev *pdev) { struct zpci_dev *zdev = get_zdev(pdev); + struct resource *res; + int i; + + zdev->pdev = pdev; + zpci_map_resources(zdev); + + for (i = 0; i < PCI_BAR_COUNT; i++) { + res = &pdev->resource[i]; + if (res->parent || !res->flags) + continue; + pci_claim_resource(pdev, i); + } + + return 0; +} + +int pcibios_enable_device(struct pci_dev *pdev, int mask) +{ + struct zpci_dev *zdev = get_zdev(pdev); + struct resource *res; + u16 cmd; + int i; zdev->pdev = pdev; zpci_debug_init_device(zdev); zpci_fmb_enable_device(zdev); zpci_map_resources(zdev); + pci_read_config_word(pdev, PCI_COMMAND, &cmd); + for (i = 0; i < PCI_BAR_COUNT; i++) { + res = &pdev->resource[i]; + + if (res->flags & IORESOURCE_IO) + return -EINVAL; + + if (res->flags & IORESOURCE_MEM) + cmd |= PCI_COMMAND_MEMORY; + } + pci_write_config_word(pdev, PCI_COMMAND, cmd); return 0; } -void pcibios_release_device(struct pci_dev *pdev) +void pcibios_disable_device(struct pci_dev *pdev) { struct zpci_dev *zdev = get_zdev(pdev); @@ -898,6 +786,8 @@ int zpci_enable_device(struct zpci_dev *zdev) rc = zpci_dma_init_device(zdev); if (rc) goto out_dma; + + zdev->state = ZPCI_FN_STATE_ONLINE; return 0; out_dma: @@ -926,18 +816,16 @@ int zpci_create_device(struct zpci_dev *zdev) rc = zpci_enable_device(zdev); if (rc) goto out_free; - - zdev->state = ZPCI_FN_STATE_ONLINE; } rc = zpci_scan_bus(zdev); if (rc) goto out_disable; - mutex_lock(&zpci_list_lock); + spin_lock(&zpci_list_lock); list_add_tail(&zdev->entry, &zpci_list); - if (hotplug_ops) - hotplug_ops->create_slot(zdev); - mutex_unlock(&zpci_list_lock); + spin_unlock(&zpci_list_lock); + + zpci_init_slot(zdev); return 0; @@ -967,15 +855,10 @@ static inline int barsize(u8 size) static int zpci_mem_init(void) { - zdev_irq_cache = kmem_cache_create("PCI_IRQ_cache", sizeof(struct zdev_irq_map), - L1_CACHE_BYTES, SLAB_HWCACHE_ALIGN, NULL); - if (!zdev_irq_cache) - goto error_zdev; - zdev_fmb_cache = kmem_cache_create("PCI_FMB_cache", sizeof(struct zpci_fmb), 16, 0, NULL); if (!zdev_fmb_cache) - goto error_fmb; + goto error_zdev; /* TODO: use realloc */ zpci_iomap_start = kzalloc(ZPCI_IOMAP_MAX_ENTRIES * sizeof(*zpci_iomap_start), @@ -986,8 +869,6 @@ static int zpci_mem_init(void) error_iomap: kmem_cache_destroy(zdev_fmb_cache); -error_fmb: - kmem_cache_destroy(zdev_irq_cache); error_zdev: return -ENOMEM; } @@ -995,28 +876,10 @@ error_zdev: static void zpci_mem_exit(void) { kfree(zpci_iomap_start); - kmem_cache_destroy(zdev_irq_cache); kmem_cache_destroy(zdev_fmb_cache); } -void zpci_register_hp_ops(struct pci_hp_callback_ops *ops) -{ - mutex_lock(&zpci_list_lock); - hotplug_ops = ops; - mutex_unlock(&zpci_list_lock); -} -EXPORT_SYMBOL_GPL(zpci_register_hp_ops); - -void zpci_deregister_hp_ops(void) -{ - mutex_lock(&zpci_list_lock); - hotplug_ops = NULL; - mutex_unlock(&zpci_list_lock); -} -EXPORT_SYMBOL_GPL(zpci_deregister_hp_ops); - -unsigned int s390_pci_probe; -EXPORT_SYMBOL_GPL(s390_pci_probe); +static unsigned int s390_pci_probe; char * __init pcibios_setup(char *str) { @@ -1044,16 +907,12 @@ static int __init pci_base_init(void) rc = zpci_debug_init(); if (rc) - return rc; + goto out; rc = zpci_mem_init(); if (rc) goto out_mem; - rc = zpci_msihash_init(); - if (rc) - goto out_hash; - rc = zpci_irq_init(); if (rc) goto out_irq; @@ -1062,7 +921,7 @@ static int __init pci_base_init(void) if (rc) goto out_dma; - rc = clp_find_pci_devices(); + rc = clp_scan_pci_devices(); if (rc) goto out_find; @@ -1073,11 +932,15 @@ out_find: out_dma: zpci_irq_exit(); out_irq: - zpci_msihash_exit(); -out_hash: zpci_mem_exit(); out_mem: zpci_debug_exit(); +out: return rc; } -subsys_initcall(pci_base_init); +subsys_initcall_sync(pci_base_init); + +void zpci_rescan(void) +{ + clp_rescan_pci_devices_simple(); +} diff --git a/arch/s390/pci/pci_clp.c b/arch/s390/pci/pci_clp.c index 2e9539625d93..475563c3d1e4 100644 --- a/arch/s390/pci/pci_clp.c +++ b/arch/s390/pci/pci_clp.c @@ -36,9 +36,9 @@ static inline u8 clp_instr(void *data) return cc; } -static void *clp_alloc_block(void) +static void *clp_alloc_block(gfp_t gfp_mask) { - return (void *) __get_free_pages(GFP_KERNEL, get_order(CLP_BLK_SIZE)); + return (void *) __get_free_pages(gfp_mask, get_order(CLP_BLK_SIZE)); } static void clp_free_block(void *ptr) @@ -70,7 +70,7 @@ static int clp_query_pci_fngrp(struct zpci_dev *zdev, u8 pfgid) struct clp_req_rsp_query_pci_grp *rrb; int rc; - rrb = clp_alloc_block(); + rrb = clp_alloc_block(GFP_KERNEL); if (!rrb) return -ENOMEM; @@ -113,7 +113,7 @@ static int clp_query_pci_fn(struct zpci_dev *zdev, u32 fh) struct clp_req_rsp_query_pci *rrb; int rc; - rrb = clp_alloc_block(); + rrb = clp_alloc_block(GFP_KERNEL); if (!rrb) return -ENOMEM; @@ -179,9 +179,9 @@ error: static int clp_set_pci_fn(u32 *fh, u8 nr_dma_as, u8 command) { struct clp_req_rsp_set_pci *rrb; - int rc, retries = 1000; + int rc, retries = 100; - rrb = clp_alloc_block(); + rrb = clp_alloc_block(GFP_KERNEL); if (!rrb) return -ENOMEM; @@ -199,7 +199,7 @@ static int clp_set_pci_fn(u32 *fh, u8 nr_dma_as, u8 command) retries--; if (retries < 0) break; - msleep(1); + msleep(20); } } while (rrb->response.hdr.rsp == CLP_RC_SETPCIFN_BUSY); @@ -245,49 +245,12 @@ int clp_disable_fh(struct zpci_dev *zdev) return rc; } -static void clp_check_pcifn_entry(struct clp_fh_list_entry *entry) +static int clp_list_pci(struct clp_req_rsp_list_pci *rrb, + void (*cb)(struct clp_fh_list_entry *entry)) { - int present, rc; - - if (!entry->vendor_id) - return; - - /* TODO: be a little bit more scalable */ - present = zpci_fid_present(entry->fid); - - if (present) - pr_debug("%s: device %x already present\n", __func__, entry->fid); - - /* skip already used functions */ - if (present && entry->config_state) - return; - - /* aev 306: function moved to stand-by state */ - if (present && !entry->config_state) { - /* - * The handle is already disabled, that means no iota/irq freeing via - * the firmware interfaces anymore. Need to free resources manually - * (DMA memory, debug, sysfs)... - */ - zpci_stop_device(get_zdev_by_fid(entry->fid)); - return; - } - - rc = clp_add_pci_device(entry->fid, entry->fh, entry->config_state); - if (rc) - pr_err("Failed to add fid: 0x%x\n", entry->fid); -} - -int clp_find_pci_devices(void) -{ - struct clp_req_rsp_list_pci *rrb; u64 resume_token = 0; int entries, i, rc; - rrb = clp_alloc_block(); - if (!rrb) - return -ENOMEM; - do { memset(rrb, 0, sizeof(*rrb)); rrb->request.hdr.len = sizeof(rrb->request); @@ -316,12 +279,101 @@ int clp_find_pci_devices(void) resume_token = rrb->response.resume_token; for (i = 0; i < entries; i++) - clp_check_pcifn_entry(&rrb->response.fh_list[i]); + cb(&rrb->response.fh_list[i]); } while (resume_token); pr_debug("Maximum number of supported PCI functions: %u\n", rrb->response.max_fn); out: + return rc; +} + +static void __clp_add(struct clp_fh_list_entry *entry) +{ + if (!entry->vendor_id) + return; + + clp_add_pci_device(entry->fid, entry->fh, entry->config_state); +} + +static void __clp_rescan(struct clp_fh_list_entry *entry) +{ + struct zpci_dev *zdev; + + if (!entry->vendor_id) + return; + + zdev = get_zdev_by_fid(entry->fid); + if (!zdev) { + clp_add_pci_device(entry->fid, entry->fh, entry->config_state); + return; + } + + if (!entry->config_state) { + /* + * The handle is already disabled, that means no iota/irq freeing via + * the firmware interfaces anymore. Need to free resources manually + * (DMA memory, debug, sysfs)... + */ + zpci_stop_device(zdev); + } +} + +static void __clp_update(struct clp_fh_list_entry *entry) +{ + struct zpci_dev *zdev; + + if (!entry->vendor_id) + return; + + zdev = get_zdev_by_fid(entry->fid); + if (!zdev) + return; + + zdev->fh = entry->fh; +} + +int clp_scan_pci_devices(void) +{ + struct clp_req_rsp_list_pci *rrb; + int rc; + + rrb = clp_alloc_block(GFP_KERNEL); + if (!rrb) + return -ENOMEM; + + rc = clp_list_pci(rrb, __clp_add); + + clp_free_block(rrb); + return rc; +} + +int clp_rescan_pci_devices(void) +{ + struct clp_req_rsp_list_pci *rrb; + int rc; + + rrb = clp_alloc_block(GFP_KERNEL); + if (!rrb) + return -ENOMEM; + + rc = clp_list_pci(rrb, __clp_rescan); + + clp_free_block(rrb); + return rc; +} + +int clp_rescan_pci_devices_simple(void) +{ + struct clp_req_rsp_list_pci *rrb; + int rc; + + rrb = clp_alloc_block(GFP_NOWAIT); + if (!rrb) + return -ENOMEM; + + rc = clp_list_pci(rrb, __clp_update); + clp_free_block(rrb); return rc; } diff --git a/arch/s390/pci/pci_dma.c b/arch/s390/pci/pci_dma.c index a2343c1f6e04..7e5573acb063 100644 --- a/arch/s390/pci/pci_dma.c +++ b/arch/s390/pci/pci_dma.c @@ -10,6 +10,7 @@ #include <linux/export.h> #include <linux/iommu-helper.h> #include <linux/dma-mapping.h> +#include <linux/vmalloc.h> #include <linux/pci.h> #include <asm/pci_dma.h> @@ -170,8 +171,8 @@ static int dma_update_trans(struct zpci_dev *zdev, unsigned long pa, */ goto no_refresh; - rc = s390pci_refresh_trans((u64) zdev->fh << 32, start_dma_addr, - nr_pages * PAGE_SIZE); + rc = zpci_refresh_trans((u64) zdev->fh << 32, start_dma_addr, + nr_pages * PAGE_SIZE); no_refresh: spin_unlock_irqrestore(&zdev->dma_table_lock, irq_flags); @@ -407,7 +408,6 @@ static void s390_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int zpci_dma_init_device(struct zpci_dev *zdev) { - unsigned int bitmap_order; int rc; spin_lock_init(&zdev->iommu_bitmap_lock); @@ -421,12 +421,7 @@ int zpci_dma_init_device(struct zpci_dev *zdev) zdev->iommu_size = (unsigned long) high_memory - PAGE_OFFSET; zdev->iommu_pages = zdev->iommu_size >> PAGE_SHIFT; - bitmap_order = get_order(zdev->iommu_pages / 8); - pr_info("iommu_size: 0x%lx iommu_pages: 0x%lx bitmap_order: %i\n", - zdev->iommu_size, zdev->iommu_pages, bitmap_order); - - zdev->iommu_bitmap = (void *) __get_free_pages(GFP_KERNEL | __GFP_ZERO, - bitmap_order); + zdev->iommu_bitmap = vzalloc(zdev->iommu_pages / 8); if (!zdev->iommu_bitmap) { rc = -ENOMEM; goto out_reg; @@ -451,8 +446,7 @@ void zpci_dma_exit_device(struct zpci_dev *zdev) { zpci_unregister_ioat(zdev, 0); dma_cleanup_tables(zdev); - free_pages((unsigned long) zdev->iommu_bitmap, - get_order(zdev->iommu_pages / 8)); + vfree(zdev->iommu_bitmap); zdev->iommu_bitmap = NULL; zdev->next_bit = 0; } diff --git a/arch/s390/pci/pci_event.c b/arch/s390/pci/pci_event.c index ec62e3a0dc09..0aecaf954845 100644 --- a/arch/s390/pci/pci_event.c +++ b/arch/s390/pci/pci_event.c @@ -69,7 +69,7 @@ static void zpci_event_log_avail(struct zpci_ccdf_avail *ccdf) clp_add_pci_device(ccdf->fid, ccdf->fh, 0); break; case 0x0306: - clp_find_pci_devices(); + clp_rescan_pci_devices(); break; default: break; diff --git a/arch/s390/pci/pci_insn.c b/arch/s390/pci/pci_insn.c index 22eeb9d7ffeb..85267c058af8 100644 --- a/arch/s390/pci/pci_insn.c +++ b/arch/s390/pci/pci_insn.c @@ -27,7 +27,7 @@ static inline u8 __mpcifc(u64 req, struct zpci_fib *fib, u8 *status) return cc; } -int s390pci_mod_fc(u64 req, struct zpci_fib *fib) +int zpci_mod_fc(u64 req, struct zpci_fib *fib) { u8 cc, status; @@ -61,7 +61,7 @@ static inline u8 __rpcit(u64 fn, u64 addr, u64 range, u8 *status) return cc; } -int s390pci_refresh_trans(u64 fn, u64 addr, u64 range) +int zpci_refresh_trans(u64 fn, u64 addr, u64 range) { u8 cc, status; @@ -78,7 +78,7 @@ int s390pci_refresh_trans(u64 fn, u64 addr, u64 range) } /* Set Interruption Controls */ -void set_irq_ctrl(u16 ctl, char *unused, u8 isc) +void zpci_set_irq_ctrl(u16 ctl, char *unused, u8 isc) { asm volatile ( " .insn rsy,0xeb00000000d1,%[ctl],%[isc],%[u]\n" @@ -109,7 +109,7 @@ static inline int __pcilg(u64 *data, u64 req, u64 offset, u8 *status) return cc; } -int s390pci_load(u64 *data, u64 req, u64 offset) +int zpci_load(u64 *data, u64 req, u64 offset) { u8 status; int cc; @@ -125,7 +125,7 @@ int s390pci_load(u64 *data, u64 req, u64 offset) __func__, cc, status, req, offset); return (cc > 0) ? -EIO : cc; } -EXPORT_SYMBOL_GPL(s390pci_load); +EXPORT_SYMBOL_GPL(zpci_load); /* PCI Store */ static inline int __pcistg(u64 data, u64 req, u64 offset, u8 *status) @@ -147,7 +147,7 @@ static inline int __pcistg(u64 data, u64 req, u64 offset, u8 *status) return cc; } -int s390pci_store(u64 data, u64 req, u64 offset) +int zpci_store(u64 data, u64 req, u64 offset) { u8 status; int cc; @@ -163,7 +163,7 @@ int s390pci_store(u64 data, u64 req, u64 offset) __func__, cc, status, req, offset); return (cc > 0) ? -EIO : cc; } -EXPORT_SYMBOL_GPL(s390pci_store); +EXPORT_SYMBOL_GPL(zpci_store); /* PCI Store Block */ static inline int __pcistb(const u64 *data, u64 req, u64 offset, u8 *status) @@ -183,7 +183,7 @@ static inline int __pcistb(const u64 *data, u64 req, u64 offset, u8 *status) return cc; } -int s390pci_store_block(const u64 *data, u64 req, u64 offset) +int zpci_store_block(const u64 *data, u64 req, u64 offset) { u8 status; int cc; @@ -199,4 +199,4 @@ int s390pci_store_block(const u64 *data, u64 req, u64 offset) __func__, cc, status, req, offset); return (cc > 0) ? -EIO : cc; } -EXPORT_SYMBOL_GPL(s390pci_store_block); +EXPORT_SYMBOL_GPL(zpci_store_block); diff --git a/arch/s390/pci/pci_msi.c b/arch/s390/pci/pci_msi.c deleted file mode 100644 index b097aed05a9b..000000000000 --- a/arch/s390/pci/pci_msi.c +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright IBM Corp. 2012 - * - * Author(s): - * Jan Glauber <jang@linux.vnet.ibm.com> - */ - -#define COMPONENT "zPCI" -#define pr_fmt(fmt) COMPONENT ": " fmt - -#include <linux/kernel.h> -#include <linux/err.h> -#include <linux/rculist.h> -#include <linux/hash.h> -#include <linux/pci.h> -#include <linux/msi.h> -#include <asm/hw_irq.h> - -/* mapping of irq numbers to msi_desc */ -static struct hlist_head *msi_hash; -static const unsigned int msi_hash_bits = 8; -#define MSI_HASH_BUCKETS (1U << msi_hash_bits) -#define msi_hashfn(nr) hash_long(nr, msi_hash_bits) - -static DEFINE_SPINLOCK(msi_map_lock); - -struct msi_desc *__irq_get_msi_desc(unsigned int irq) -{ - struct msi_map *map; - - hlist_for_each_entry_rcu(map, - &msi_hash[msi_hashfn(irq)], msi_chain) - if (map->irq == irq) - return map->msi; - return NULL; -} - -int zpci_msi_set_mask_bits(struct msi_desc *msi, u32 mask, u32 flag) -{ - if (msi->msi_attrib.is_msix) { - int offset = msi->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + - PCI_MSIX_ENTRY_VECTOR_CTRL; - msi->masked = readl(msi->mask_base + offset); - writel(flag, msi->mask_base + offset); - } else { - if (msi->msi_attrib.maskbit) { - int pos; - u32 mask_bits; - - pos = (long) msi->mask_base; - pci_read_config_dword(msi->dev, pos, &mask_bits); - mask_bits &= ~(mask); - mask_bits |= flag & mask; - pci_write_config_dword(msi->dev, pos, mask_bits); - } else { - return 0; - } - } - - msi->msi_attrib.maskbit = !!flag; - return 1; -} - -int zpci_setup_msi_irq(struct zpci_dev *zdev, struct msi_desc *msi, - unsigned int nr, int offset) -{ - struct msi_map *map; - struct msi_msg msg; - int rc; - - map = kmalloc(sizeof(*map), GFP_KERNEL); - if (map == NULL) - return -ENOMEM; - - map->irq = nr; - map->msi = msi; - zdev->msi_map[nr & ZPCI_MSI_MASK] = map; - INIT_HLIST_NODE(&map->msi_chain); - - pr_debug("%s hashing irq: %u to bucket nr: %llu\n", - __func__, nr, msi_hashfn(nr)); - hlist_add_head_rcu(&map->msi_chain, &msi_hash[msi_hashfn(nr)]); - - spin_lock(&msi_map_lock); - rc = irq_set_msi_desc(nr, msi); - if (rc) { - spin_unlock(&msi_map_lock); - hlist_del_rcu(&map->msi_chain); - kfree(map); - zdev->msi_map[nr & ZPCI_MSI_MASK] = NULL; - return rc; - } - spin_unlock(&msi_map_lock); - - msg.data = nr - offset; - msg.address_lo = zdev->msi_addr & 0xffffffff; - msg.address_hi = zdev->msi_addr >> 32; - write_msi_msg(nr, &msg); - return 0; -} - -void zpci_teardown_msi_irq(struct zpci_dev *zdev, struct msi_desc *msi) -{ - int irq = msi->irq & ZPCI_MSI_MASK; - struct msi_map *map; - - msi->msg.address_lo = 0; - msi->msg.address_hi = 0; - msi->msg.data = 0; - msi->irq = 0; - zpci_msi_set_mask_bits(msi, 1, 1); - - spin_lock(&msi_map_lock); - map = zdev->msi_map[irq]; - hlist_del_rcu(&map->msi_chain); - kfree(map); - zdev->msi_map[irq] = NULL; - spin_unlock(&msi_map_lock); -} - -/* - * The msi hash table has 256 entries which is good for 4..20 - * devices (a typical device allocates 10 + CPUs MSI's). Maybe make - * the hash table size adjustable later. - */ -int __init zpci_msihash_init(void) -{ - unsigned int i; - - msi_hash = kmalloc(MSI_HASH_BUCKETS * sizeof(*msi_hash), GFP_KERNEL); - if (!msi_hash) - return -ENOMEM; - - for (i = 0; i < MSI_HASH_BUCKETS; i++) - INIT_HLIST_HEAD(&msi_hash[i]); - return 0; -} - -void __init zpci_msihash_exit(void) -{ - kfree(msi_hash); -} diff --git a/arch/s390/pci/pci_sysfs.c b/arch/s390/pci/pci_sysfs.c index e99a2557f186..cf8a12ff733b 100644 --- a/arch/s390/pci/pci_sysfs.c +++ b/arch/s390/pci/pci_sysfs.c @@ -48,11 +48,38 @@ static ssize_t show_pfgid(struct device *dev, struct device_attribute *attr, } static DEVICE_ATTR(pfgid, S_IRUGO, show_pfgid, NULL); +static void recover_callback(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct zpci_dev *zdev = get_zdev(pdev); + int ret; + + pci_stop_and_remove_bus_device(pdev); + ret = zpci_disable_device(zdev); + if (ret) + return; + + ret = zpci_enable_device(zdev); + if (ret) + return; + + pci_rescan_bus(zdev->bus); +} + +static ssize_t store_recover(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + int rc = device_schedule_callback(dev, recover_callback); + return rc ? rc : count; +} +static DEVICE_ATTR(recover, S_IWUSR, NULL, store_recover); + static struct device_attribute *zpci_dev_attrs[] = { &dev_attr_function_id, &dev_attr_function_handle, &dev_attr_pchid, &dev_attr_pfgid, + &dev_attr_recover, NULL, }; diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig index 1020dd85431a..1018ed3a3ca5 100644 --- a/arch/sh/Kconfig +++ b/arch/sh/Kconfig @@ -643,9 +643,9 @@ config KEXEC It is an ongoing process to be certain the hardware in a machine is properly shutdown, so do not be surprised if this code does not - initially work for you. It may help to enable device hotplugging - support. As of this writing the exact hardware interface is - strongly in flux, so no good recommendation can be made. + initially work for you. As of this writing the exact hardware + interface is strongly in flux, so no good recommendation can be + made. config CRASH_DUMP bool "kernel crash dumps (EXPERIMENTAL)" diff --git a/arch/sh/drivers/pci/pci.c b/arch/sh/drivers/pci/pci.c index 102f5d58b037..60ed3e1c4b75 100644 --- a/arch/sh/drivers/pci/pci.c +++ b/arch/sh/drivers/pci/pci.c @@ -69,7 +69,6 @@ static void pcibios_scanbus(struct pci_channel *hose) pci_bus_size_bridges(bus); pci_bus_assign_resources(bus); - pci_enable_bridges(bus); } else { pci_free_resource_list(&resources); } diff --git a/arch/sh/kernel/cpu/shmobile/cpuidle.c b/arch/sh/kernel/cpu/shmobile/cpuidle.c index d30622592116..e3abfd4277e2 100644 --- a/arch/sh/kernel/cpu/shmobile/cpuidle.c +++ b/arch/sh/kernel/cpu/shmobile/cpuidle.c @@ -91,13 +91,11 @@ static struct cpuidle_driver cpuidle_driver = { int __init sh_mobile_setup_cpuidle(void) { - int ret; - if (sh_mobile_sleep_supported & SUSP_SH_SF) cpuidle_driver.states[1].disabled = false; if (sh_mobile_sleep_supported & SUSP_SH_STANDBY) cpuidle_driver.states[2].disabled = false; - return cpuidle_register(&cpuidle_driver); + return cpuidle_register(&cpuidle_driver, NULL); } diff --git a/arch/tile/include/asm/topology.h b/arch/tile/include/asm/topology.h index d5e86c9f74fd..d15c0d8d550f 100644 --- a/arch/tile/include/asm/topology.h +++ b/arch/tile/include/asm/topology.h @@ -89,9 +89,6 @@ static inline const struct cpumask *cpumask_of_node(int node) #define topology_core_id(cpu) (cpu) #define topology_core_cpumask(cpu) ((void)(cpu), cpu_online_mask) #define topology_thread_cpumask(cpu) cpumask_of(cpu) - -/* indicates that pointers to the topology struct cpumask maps are valid */ -#define arch_provides_topology_pointers yes #endif #endif /* _ASM_TILE_TOPOLOGY_H */ diff --git a/arch/tile/kernel/pci_gx.c b/arch/tile/kernel/pci_gx.c index 11425633b2d7..6640e7bbeaa2 100644 --- a/arch/tile/kernel/pci_gx.c +++ b/arch/tile/kernel/pci_gx.c @@ -508,13 +508,8 @@ static void fixup_read_and_payload_sizes(struct pci_controller *controller) rc_dev_cap.word); /* Configure PCI Express MPS setting. */ - list_for_each_entry(child, &root_bus->children, node) { - struct pci_dev *self = child->self; - if (!self) - continue; - - pcie_bus_configure_settings(child, self->pcie_mpss); - } + list_for_each_entry(child, &root_bus->children, node) + pcie_bus_configure_settings(child); /* * Set the mac_config register in trio based on the MPS/MRS of the link. diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index b32ebf92b0ce..f16fc34e6608 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -16,6 +16,7 @@ config X86_64 def_bool y depends on 64BIT select X86_DEV_DMA_OPS + select ARCH_USE_CMPXCHG_LOCKREF ### Arch settings config X86 @@ -81,7 +82,6 @@ config X86 select HAVE_USER_RETURN_NOTIFIER select ARCH_BINFMT_ELF_RANDOMIZE_PIE select HAVE_ARCH_JUMP_LABEL - select HAVE_TEXT_POKE_SMP select HAVE_GENERIC_HARDIRQS select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE select SPARSE_IRQ @@ -1627,9 +1627,9 @@ config KEXEC It is an ongoing process to be certain the hardware in a machine is properly shutdown, so do not be surprised if this code does not - initially work for you. It may help to enable device hotplugging - support. As of this writing the exact hardware interface is - strongly in flux, so no good recommendation can be made. + initially work for you. As of this writing the exact hardware + interface is strongly in flux, so no good recommendation can be + made. config CRASH_DUMP bool "kernel crash dumps" @@ -2332,10 +2332,6 @@ config HAVE_ATOMIC_IOMAP def_bool y depends on X86_32 -config HAVE_TEXT_POKE_SMP - bool - select STOP_MACHINE if SMP - config X86_DEV_DMA_OPS bool depends on X86_64 || STA2X11 diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h index 58ed6d96a6ac..0a3f9c9f98d5 100644 --- a/arch/x86/include/asm/alternative.h +++ b/arch/x86/include/asm/alternative.h @@ -5,6 +5,7 @@ #include <linux/stddef.h> #include <linux/stringify.h> #include <asm/asm.h> +#include <asm/ptrace.h> /* * Alternative inline assembly for SMP. @@ -220,20 +221,11 @@ extern void *text_poke_early(void *addr, const void *opcode, size_t len); * no thread can be preempted in the instructions being modified (no iret to an * invalid instruction possible) or if the instructions are changed from a * consistent state to another consistent state atomically. - * More care must be taken when modifying code in the SMP case because of - * Intel's errata. text_poke_smp() takes care that errata, but still - * doesn't support NMI/MCE handler code modifying. * On the local CPU you need to be protected again NMI or MCE handlers seeing an * inconsistent instruction while you patch. */ -struct text_poke_param { - void *addr; - const void *opcode; - size_t len; -}; - extern void *text_poke(void *addr, const void *opcode, size_t len); -extern void *text_poke_smp(void *addr, const void *opcode, size_t len); -extern void text_poke_smp_batch(struct text_poke_param *params, int n); +extern int poke_int3_handler(struct pt_regs *regs); +extern void *text_poke_bp(void *addr, const void *opcode, size_t len, void *handler); #endif /* _ASM_X86_ALTERNATIVE_H */ diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 24cf5aefb704..4f4a3d98c170 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -942,35 +942,6 @@ extern int set_tsc_mode(unsigned int val); extern u16 amd_get_nb_id(int cpu); -struct aperfmperf { - u64 aperf, mperf; -}; - -static inline void get_aperfmperf(struct aperfmperf *am) -{ - WARN_ON_ONCE(!boot_cpu_has(X86_FEATURE_APERFMPERF)); - - rdmsrl(MSR_IA32_APERF, am->aperf); - rdmsrl(MSR_IA32_MPERF, am->mperf); -} - -#define APERFMPERF_SHIFT 10 - -static inline -unsigned long calc_aperfmperf_ratio(struct aperfmperf *old, - struct aperfmperf *new) -{ - u64 aperf = new->aperf - old->aperf; - u64 mperf = new->mperf - old->mperf; - unsigned long ratio = aperf; - - mperf >>= APERFMPERF_SHIFT; - if (mperf) - ratio = div64_u64(aperf, mperf); - - return ratio; -} - extern unsigned long arch_align_stack(unsigned long sp); extern void free_init_pages(char *what, unsigned long begin, unsigned long end); diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h index e3ddd7db723f..e0e668422c75 100644 --- a/arch/x86/include/asm/spinlock.h +++ b/arch/x86/include/asm/spinlock.h @@ -34,6 +34,11 @@ # define UNLOCK_LOCK_PREFIX #endif +static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock) +{ + return lock.tickets.head == lock.tickets.tail; +} + /* * Ticket locks are conceptually two parts, one indicating the current head of * the queue, and the other indicating the current tail. The lock is acquired diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h index 095b21507b6a..d35f24e231cd 100644 --- a/arch/x86/include/asm/topology.h +++ b/arch/x86/include/asm/topology.h @@ -124,9 +124,6 @@ extern const struct cpumask *cpu_coregroup_mask(int cpu); #define topology_core_id(cpu) (cpu_data(cpu).cpu_core_id) #define topology_core_cpumask(cpu) (per_cpu(cpu_core_map, cpu)) #define topology_thread_cpumask(cpu) (per_cpu(cpu_sibling_map, cpu)) - -/* indicates that pointers to the topology cpumask_t maps are valid */ -#define arch_provides_topology_pointers yes #endif static inline void arch_fix_phys_package_id(int num, u32 slot) diff --git a/arch/x86/include/asm/tsc.h b/arch/x86/include/asm/tsc.h index c91e8b9d588b..235be70d5bb4 100644 --- a/arch/x86/include/asm/tsc.h +++ b/arch/x86/include/asm/tsc.h @@ -49,6 +49,7 @@ extern void tsc_init(void); extern void mark_tsc_unstable(char *reason); extern int unsynchronized_tsc(void); extern int check_tsc_unstable(void); +extern int check_tsc_disabled(void); extern unsigned long native_calibrate_tsc(void); extern int tsc_clocksource_reliable; diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index c15cf9a25e27..15e8563e5c24 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -11,6 +11,7 @@ #include <linux/memory.h> #include <linux/stop_machine.h> #include <linux/slab.h> +#include <linux/kdebug.h> #include <asm/alternative.h> #include <asm/sections.h> #include <asm/pgtable.h> @@ -596,97 +597,93 @@ void *__kprobes text_poke(void *addr, const void *opcode, size_t len) return addr; } -/* - * Cross-modifying kernel text with stop_machine(). - * This code originally comes from immediate value. - */ -static atomic_t stop_machine_first; -static int wrote_text; +static void do_sync_core(void *info) +{ + sync_core(); +} -struct text_poke_params { - struct text_poke_param *params; - int nparams; -}; +static bool bp_patching_in_progress; +static void *bp_int3_handler, *bp_int3_addr; -static int __kprobes stop_machine_text_poke(void *data) +int poke_int3_handler(struct pt_regs *regs) { - struct text_poke_params *tpp = data; - struct text_poke_param *p; - int i; + /* bp_patching_in_progress */ + smp_rmb(); - if (atomic_xchg(&stop_machine_first, 0)) { - for (i = 0; i < tpp->nparams; i++) { - p = &tpp->params[i]; - text_poke(p->addr, p->opcode, p->len); - } - smp_wmb(); /* Make sure other cpus see that this has run */ - wrote_text = 1; - } else { - while (!wrote_text) - cpu_relax(); - smp_mb(); /* Load wrote_text before following execution */ - } + if (likely(!bp_patching_in_progress)) + return 0; - for (i = 0; i < tpp->nparams; i++) { - p = &tpp->params[i]; - flush_icache_range((unsigned long)p->addr, - (unsigned long)p->addr + p->len); - } - /* - * Intel Archiecture Software Developer's Manual section 7.1.3 specifies - * that a core serializing instruction such as "cpuid" should be - * executed on _each_ core before the new instruction is made visible. - */ - sync_core(); - return 0; -} + if (user_mode_vm(regs) || regs->ip != (unsigned long)bp_int3_addr) + return 0; + + /* set up the specified breakpoint handler */ + regs->ip = (unsigned long) bp_int3_handler; + + return 1; -/** - * text_poke_smp - Update instructions on a live kernel on SMP - * @addr: address to modify - * @opcode: source of the copy - * @len: length to copy - * - * Modify multi-byte instruction by using stop_machine() on SMP. This allows - * user to poke/set multi-byte text on SMP. Only non-NMI/MCE code modifying - * should be allowed, since stop_machine() does _not_ protect code against - * NMI and MCE. - * - * Note: Must be called under get_online_cpus() and text_mutex. - */ -void *__kprobes text_poke_smp(void *addr, const void *opcode, size_t len) -{ - struct text_poke_params tpp; - struct text_poke_param p; - - p.addr = addr; - p.opcode = opcode; - p.len = len; - tpp.params = &p; - tpp.nparams = 1; - atomic_set(&stop_machine_first, 1); - wrote_text = 0; - /* Use __stop_machine() because the caller already got online_cpus. */ - __stop_machine(stop_machine_text_poke, (void *)&tpp, cpu_online_mask); - return addr; } /** - * text_poke_smp_batch - Update instructions on a live kernel on SMP - * @params: an array of text_poke parameters - * @n: the number of elements in params. + * text_poke_bp() -- update instructions on live kernel on SMP + * @addr: address to patch + * @opcode: opcode of new instruction + * @len: length to copy + * @handler: address to jump to when the temporary breakpoint is hit * - * Modify multi-byte instruction by using stop_machine() on SMP. Since the - * stop_machine() is heavy task, it is better to aggregate text_poke requests - * and do it once if possible. + * Modify multi-byte instruction by using int3 breakpoint on SMP. + * We completely avoid stop_machine() here, and achieve the + * synchronization using int3 breakpoint. * - * Note: Must be called under get_online_cpus() and text_mutex. + * The way it is done: + * - add a int3 trap to the address that will be patched + * - sync cores + * - update all but the first byte of the patched range + * - sync cores + * - replace the first byte (int3) by the first byte of + * replacing opcode + * - sync cores + * + * Note: must be called under text_mutex. */ -void __kprobes text_poke_smp_batch(struct text_poke_param *params, int n) +void *text_poke_bp(void *addr, const void *opcode, size_t len, void *handler) { - struct text_poke_params tpp = {.params = params, .nparams = n}; + unsigned char int3 = 0xcc; + + bp_int3_handler = handler; + bp_int3_addr = (u8 *)addr + sizeof(int3); + bp_patching_in_progress = true; + /* + * Corresponding read barrier in int3 notifier for + * making sure the in_progress flags is correctly ordered wrt. + * patching + */ + smp_wmb(); + + text_poke(addr, &int3, sizeof(int3)); - atomic_set(&stop_machine_first, 1); - wrote_text = 0; - __stop_machine(stop_machine_text_poke, (void *)&tpp, cpu_online_mask); + on_each_cpu(do_sync_core, NULL, 1); + + if (len - sizeof(int3) > 0) { + /* patch all but the first byte */ + text_poke((char *)addr + sizeof(int3), + (const char *) opcode + sizeof(int3), + len - sizeof(int3)); + /* + * According to Intel, this core syncing is very likely + * not necessary and we'd be safe even without it. But + * better safe than sorry (plus there's not only Intel). + */ + on_each_cpu(do_sync_core, NULL, 1); + } + + /* patch the first byte */ + text_poke(addr, opcode, sizeof(int3)); + + on_each_cpu(do_sync_core, NULL, 1); + + bp_patching_in_progress = false; + smp_wmb(); + + return addr; } + diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c index a7c7305030cc..8355c84b9729 100644 --- a/arch/x86/kernel/cpu/perf_event.c +++ b/arch/x86/kernel/cpu/perf_event.c @@ -1884,6 +1884,7 @@ static struct pmu pmu = { void arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now) { userpg->cap_usr_time = 0; + userpg->cap_usr_time_zero = 0; userpg->cap_usr_rdpmc = x86_pmu.attr_rdpmc; userpg->pmc_width = x86_pmu.cntval_bits; @@ -1897,6 +1898,11 @@ void arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now) userpg->time_mult = this_cpu_read(cyc2ns); userpg->time_shift = CYC2NS_SCALE_FACTOR; userpg->time_offset = this_cpu_read(cyc2ns_offset) - now; + + if (sched_clock_stable && !check_tsc_disabled()) { + userpg->cap_usr_time_zero = 1; + userpg->time_zero = this_cpu_read(cyc2ns_offset); + } } /* diff --git a/arch/x86/kernel/cpu/perf_event.h b/arch/x86/kernel/cpu/perf_event.h index 97e557bc4c91..cc16faae0538 100644 --- a/arch/x86/kernel/cpu/perf_event.h +++ b/arch/x86/kernel/cpu/perf_event.h @@ -641,6 +641,8 @@ extern struct event_constraint intel_core2_pebs_event_constraints[]; extern struct event_constraint intel_atom_pebs_event_constraints[]; +extern struct event_constraint intel_slm_pebs_event_constraints[]; + extern struct event_constraint intel_nehalem_pebs_event_constraints[]; extern struct event_constraint intel_westmere_pebs_event_constraints[]; diff --git a/arch/x86/kernel/cpu/perf_event_amd.c b/arch/x86/kernel/cpu/perf_event_amd.c index 4cbe03287b08..beeb7cc07044 100644 --- a/arch/x86/kernel/cpu/perf_event_amd.c +++ b/arch/x86/kernel/cpu/perf_event_amd.c @@ -347,8 +347,7 @@ static struct amd_nb *amd_alloc_nb(int cpu) struct amd_nb *nb; int i; - nb = kmalloc_node(sizeof(struct amd_nb), GFP_KERNEL | __GFP_ZERO, - cpu_to_node(cpu)); + nb = kzalloc_node(sizeof(struct amd_nb), GFP_KERNEL, cpu_to_node(cpu)); if (!nb) return NULL; diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c index a45d8d4ace10..0abf6742a8b0 100644 --- a/arch/x86/kernel/cpu/perf_event_intel.c +++ b/arch/x86/kernel/cpu/perf_event_intel.c @@ -81,7 +81,8 @@ static struct event_constraint intel_nehalem_event_constraints[] __read_mostly = static struct extra_reg intel_nehalem_extra_regs[] __read_mostly = { - INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff, RSP_0), + /* must define OFFCORE_RSP_X first, see intel_fixup_er() */ + INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0xffff, RSP_0), INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x100b), EVENT_EXTRA_END }; @@ -143,8 +144,9 @@ static struct event_constraint intel_ivb_event_constraints[] __read_mostly = static struct extra_reg intel_westmere_extra_regs[] __read_mostly = { - INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff, RSP_0), - INTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0xffff, RSP_1), + /* must define OFFCORE_RSP_X first, see intel_fixup_er() */ + INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0xffff, RSP_0), + INTEL_UEVENT_EXTRA_REG(0x01bb, MSR_OFFCORE_RSP_1, 0xffff, RSP_1), INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x100b), EVENT_EXTRA_END }; @@ -162,16 +164,27 @@ static struct event_constraint intel_gen_event_constraints[] __read_mostly = EVENT_CONSTRAINT_END }; +static struct event_constraint intel_slm_event_constraints[] __read_mostly = +{ + FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ + FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ + FIXED_EVENT_CONSTRAINT(0x013c, 2), /* CPU_CLK_UNHALTED.REF */ + FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF */ + EVENT_CONSTRAINT_END +}; + static struct extra_reg intel_snb_extra_regs[] __read_mostly = { - INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0x3f807f8fffull, RSP_0), - INTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0x3f807f8fffull, RSP_1), + /* must define OFFCORE_RSP_X first, see intel_fixup_er() */ + INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x3f807f8fffull, RSP_0), + INTEL_UEVENT_EXTRA_REG(0x01bb, MSR_OFFCORE_RSP_1, 0x3f807f8fffull, RSP_1), INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd), EVENT_EXTRA_END }; static struct extra_reg intel_snbep_extra_regs[] __read_mostly = { - INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0x3fffff8fffull, RSP_0), - INTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0x3fffff8fffull, RSP_1), + /* must define OFFCORE_RSP_X first, see intel_fixup_er() */ + INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x3fffff8fffull, RSP_0), + INTEL_UEVENT_EXTRA_REG(0x01bb, MSR_OFFCORE_RSP_1, 0x3fffff8fffull, RSP_1), INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd), EVENT_EXTRA_END }; @@ -882,6 +895,140 @@ static __initconst const u64 atom_hw_cache_event_ids }, }; +static struct extra_reg intel_slm_extra_regs[] __read_mostly = +{ + /* must define OFFCORE_RSP_X first, see intel_fixup_er() */ + INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x768005ffff, RSP_0), + INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OFFCORE_RSP_1, 0x768005ffff, RSP_1), + EVENT_EXTRA_END +}; + +#define SLM_DMND_READ SNB_DMND_DATA_RD +#define SLM_DMND_WRITE SNB_DMND_RFO +#define SLM_DMND_PREFETCH (SNB_PF_DATA_RD|SNB_PF_RFO) + +#define SLM_SNP_ANY (SNB_SNP_NONE|SNB_SNP_MISS|SNB_NO_FWD|SNB_HITM) +#define SLM_LLC_ACCESS SNB_RESP_ANY +#define SLM_LLC_MISS (SLM_SNP_ANY|SNB_NON_DRAM) + +static __initconst const u64 slm_hw_cache_extra_regs + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = +{ + [ C(LL ) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = SLM_DMND_READ|SLM_LLC_ACCESS, + [ C(RESULT_MISS) ] = SLM_DMND_READ|SLM_LLC_MISS, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = SLM_DMND_WRITE|SLM_LLC_ACCESS, + [ C(RESULT_MISS) ] = SLM_DMND_WRITE|SLM_LLC_MISS, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = SLM_DMND_PREFETCH|SLM_LLC_ACCESS, + [ C(RESULT_MISS) ] = SLM_DMND_PREFETCH|SLM_LLC_MISS, + }, + }, +}; + +static __initconst const u64 slm_hw_cache_event_ids + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = +{ + [ C(L1D) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = 0, + [ C(RESULT_MISS) ] = 0x0104, /* LD_DCU_MISS */ + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = 0, + [ C(RESULT_MISS) ] = 0, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = 0, + [ C(RESULT_MISS) ] = 0, + }, + }, + [ C(L1I ) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = 0x0380, /* ICACHE.ACCESSES */ + [ C(RESULT_MISS) ] = 0x0280, /* ICACGE.MISSES */ + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = -1, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = 0, + [ C(RESULT_MISS) ] = 0, + }, + }, + [ C(LL ) ] = { + [ C(OP_READ) ] = { + /* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */ + [ C(RESULT_ACCESS) ] = 0x01b7, + /* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */ + [ C(RESULT_MISS) ] = 0x01b7, + }, + [ C(OP_WRITE) ] = { + /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */ + [ C(RESULT_ACCESS) ] = 0x01b7, + /* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */ + [ C(RESULT_MISS) ] = 0x01b7, + }, + [ C(OP_PREFETCH) ] = { + /* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */ + [ C(RESULT_ACCESS) ] = 0x01b7, + /* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */ + [ C(RESULT_MISS) ] = 0x01b7, + }, + }, + [ C(DTLB) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = 0, + [ C(RESULT_MISS) ] = 0x0804, /* LD_DTLB_MISS */ + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = 0, + [ C(RESULT_MISS) ] = 0, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = 0, + [ C(RESULT_MISS) ] = 0, + }, + }, + [ C(ITLB) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P */ + [ C(RESULT_MISS) ] = 0x0282, /* ITLB.MISSES */ + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = -1, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = -1, + }, + }, + [ C(BPU ) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY */ + [ C(RESULT_MISS) ] = 0x00c5, /* BP_INST_RETIRED.MISPRED */ + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = -1, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = -1, + }, + }, +}; + static inline bool intel_pmu_needs_lbr_smpl(struct perf_event *event) { /* user explicitly requested branch sampling */ @@ -1301,11 +1448,11 @@ static void intel_fixup_er(struct perf_event *event, int idx) if (idx == EXTRA_REG_RSP_0) { event->hw.config &= ~INTEL_ARCH_EVENT_MASK; - event->hw.config |= 0x01b7; + event->hw.config |= x86_pmu.extra_regs[EXTRA_REG_RSP_0].event; event->hw.extra_reg.reg = MSR_OFFCORE_RSP_0; } else if (idx == EXTRA_REG_RSP_1) { event->hw.config &= ~INTEL_ARCH_EVENT_MASK; - event->hw.config |= 0x01bb; + event->hw.config |= x86_pmu.extra_regs[EXTRA_REG_RSP_1].event; event->hw.extra_reg.reg = MSR_OFFCORE_RSP_1; } } @@ -2176,6 +2323,21 @@ __init int intel_pmu_init(void) pr_cont("Atom events, "); break; + case 55: /* Atom 22nm "Silvermont" */ + memcpy(hw_cache_event_ids, slm_hw_cache_event_ids, + sizeof(hw_cache_event_ids)); + memcpy(hw_cache_extra_regs, slm_hw_cache_extra_regs, + sizeof(hw_cache_extra_regs)); + + intel_pmu_lbr_init_atom(); + + x86_pmu.event_constraints = intel_slm_event_constraints; + x86_pmu.pebs_constraints = intel_slm_pebs_event_constraints; + x86_pmu.extra_regs = intel_slm_extra_regs; + x86_pmu.er_flags |= ERF_HAS_RSP_1; + pr_cont("Silvermont events, "); + break; + case 37: /* 32 nm nehalem, "Clarkdale" */ case 44: /* 32 nm nehalem, "Gulftown" */ case 47: /* 32 nm Xeon E7 */ diff --git a/arch/x86/kernel/cpu/perf_event_intel_ds.c b/arch/x86/kernel/cpu/perf_event_intel_ds.c index 3065c57a63c1..63438aad177f 100644 --- a/arch/x86/kernel/cpu/perf_event_intel_ds.c +++ b/arch/x86/kernel/cpu/perf_event_intel_ds.c @@ -224,7 +224,7 @@ static int alloc_pebs_buffer(int cpu) if (!x86_pmu.pebs) return 0; - buffer = kmalloc_node(PEBS_BUFFER_SIZE, GFP_KERNEL | __GFP_ZERO, node); + buffer = kzalloc_node(PEBS_BUFFER_SIZE, GFP_KERNEL, node); if (unlikely(!buffer)) return -ENOMEM; @@ -262,7 +262,7 @@ static int alloc_bts_buffer(int cpu) if (!x86_pmu.bts) return 0; - buffer = kmalloc_node(BTS_BUFFER_SIZE, GFP_KERNEL | __GFP_ZERO, node); + buffer = kzalloc_node(BTS_BUFFER_SIZE, GFP_KERNEL, node); if (unlikely(!buffer)) return -ENOMEM; @@ -295,7 +295,7 @@ static int alloc_ds_buffer(int cpu) int node = cpu_to_node(cpu); struct debug_store *ds; - ds = kmalloc_node(sizeof(*ds), GFP_KERNEL | __GFP_ZERO, node); + ds = kzalloc_node(sizeof(*ds), GFP_KERNEL, node); if (unlikely(!ds)) return -ENOMEM; @@ -517,6 +517,32 @@ struct event_constraint intel_atom_pebs_event_constraints[] = { EVENT_CONSTRAINT_END }; +struct event_constraint intel_slm_pebs_event_constraints[] = { + INTEL_UEVENT_CONSTRAINT(0x0103, 0x1), /* REHABQ.LD_BLOCK_ST_FORWARD_PS */ + INTEL_UEVENT_CONSTRAINT(0x0803, 0x1), /* REHABQ.LD_SPLITS_PS */ + INTEL_UEVENT_CONSTRAINT(0x0204, 0x1), /* MEM_UOPS_RETIRED.L2_HIT_LOADS_PS */ + INTEL_UEVENT_CONSTRAINT(0x0404, 0x1), /* MEM_UOPS_RETIRED.L2_MISS_LOADS_PS */ + INTEL_UEVENT_CONSTRAINT(0x0804, 0x1), /* MEM_UOPS_RETIRED.DTLB_MISS_LOADS_PS */ + INTEL_UEVENT_CONSTRAINT(0x2004, 0x1), /* MEM_UOPS_RETIRED.HITM_PS */ + INTEL_UEVENT_CONSTRAINT(0x00c0, 0x1), /* INST_RETIRED.ANY_PS */ + INTEL_UEVENT_CONSTRAINT(0x00c4, 0x1), /* BR_INST_RETIRED.ALL_BRANCHES_PS */ + INTEL_UEVENT_CONSTRAINT(0x7ec4, 0x1), /* BR_INST_RETIRED.JCC_PS */ + INTEL_UEVENT_CONSTRAINT(0xbfc4, 0x1), /* BR_INST_RETIRED.FAR_BRANCH_PS */ + INTEL_UEVENT_CONSTRAINT(0xebc4, 0x1), /* BR_INST_RETIRED.NON_RETURN_IND_PS */ + INTEL_UEVENT_CONSTRAINT(0xf7c4, 0x1), /* BR_INST_RETIRED.RETURN_PS */ + INTEL_UEVENT_CONSTRAINT(0xf9c4, 0x1), /* BR_INST_RETIRED.CALL_PS */ + INTEL_UEVENT_CONSTRAINT(0xfbc4, 0x1), /* BR_INST_RETIRED.IND_CALL_PS */ + INTEL_UEVENT_CONSTRAINT(0xfdc4, 0x1), /* BR_INST_RETIRED.REL_CALL_PS */ + INTEL_UEVENT_CONSTRAINT(0xfec4, 0x1), /* BR_INST_RETIRED.TAKEN_JCC_PS */ + INTEL_UEVENT_CONSTRAINT(0x00c5, 0x1), /* BR_INST_MISP_RETIRED.ALL_BRANCHES_PS */ + INTEL_UEVENT_CONSTRAINT(0x7ec5, 0x1), /* BR_INST_MISP_RETIRED.JCC_PS */ + INTEL_UEVENT_CONSTRAINT(0xebc5, 0x1), /* BR_INST_MISP_RETIRED.NON_RETURN_IND_PS */ + INTEL_UEVENT_CONSTRAINT(0xf7c5, 0x1), /* BR_INST_MISP_RETIRED.RETURN_PS */ + INTEL_UEVENT_CONSTRAINT(0xfbc5, 0x1), /* BR_INST_MISP_RETIRED.IND_CALL_PS */ + INTEL_UEVENT_CONSTRAINT(0xfec5, 0x1), /* BR_INST_MISP_RETIRED.TAKEN_JCC_PS */ + EVENT_CONSTRAINT_END +}; + struct event_constraint intel_nehalem_pebs_event_constraints[] = { INTEL_PLD_CONSTRAINT(0x100b, 0xf), /* MEM_INST_RETIRED.* */ INTEL_EVENT_CONSTRAINT(0x0f, 0xf), /* MEM_UNCORE_RETIRED.* */ diff --git a/arch/x86/kernel/cpu/perf_event_intel_uncore.c b/arch/x86/kernel/cpu/perf_event_intel_uncore.c index 1fb6c72717bd..fd8011ed4dcd 100644 --- a/arch/x86/kernel/cpu/perf_event_intel_uncore.c +++ b/arch/x86/kernel/cpu/perf_event_intel_uncore.c @@ -6,6 +6,8 @@ static struct intel_uncore_type **pci_uncores = empty_uncore; /* pci bus to socket mapping */ static int pcibus_to_physid[256] = { [0 ... 255] = -1, }; +static struct pci_dev *extra_pci_dev[UNCORE_SOCKET_MAX][UNCORE_EXTRA_PCI_DEV_MAX]; + static DEFINE_RAW_SPINLOCK(uncore_box_lock); /* mask of cpus that collect uncore events */ @@ -45,6 +47,24 @@ DEFINE_UNCORE_FORMAT_ATTR(filter_band0, filter_band0, "config1:0-7"); DEFINE_UNCORE_FORMAT_ATTR(filter_band1, filter_band1, "config1:8-15"); DEFINE_UNCORE_FORMAT_ATTR(filter_band2, filter_band2, "config1:16-23"); DEFINE_UNCORE_FORMAT_ATTR(filter_band3, filter_band3, "config1:24-31"); +DEFINE_UNCORE_FORMAT_ATTR(match_rds, match_rds, "config1:48-51"); +DEFINE_UNCORE_FORMAT_ATTR(match_rnid30, match_rnid30, "config1:32-35"); +DEFINE_UNCORE_FORMAT_ATTR(match_rnid4, match_rnid4, "config1:31"); +DEFINE_UNCORE_FORMAT_ATTR(match_dnid, match_dnid, "config1:13-17"); +DEFINE_UNCORE_FORMAT_ATTR(match_mc, match_mc, "config1:9-12"); +DEFINE_UNCORE_FORMAT_ATTR(match_opc, match_opc, "config1:5-8"); +DEFINE_UNCORE_FORMAT_ATTR(match_vnw, match_vnw, "config1:3-4"); +DEFINE_UNCORE_FORMAT_ATTR(match0, match0, "config1:0-31"); +DEFINE_UNCORE_FORMAT_ATTR(match1, match1, "config1:32-63"); +DEFINE_UNCORE_FORMAT_ATTR(mask_rds, mask_rds, "config2:48-51"); +DEFINE_UNCORE_FORMAT_ATTR(mask_rnid30, mask_rnid30, "config2:32-35"); +DEFINE_UNCORE_FORMAT_ATTR(mask_rnid4, mask_rnid4, "config2:31"); +DEFINE_UNCORE_FORMAT_ATTR(mask_dnid, mask_dnid, "config2:13-17"); +DEFINE_UNCORE_FORMAT_ATTR(mask_mc, mask_mc, "config2:9-12"); +DEFINE_UNCORE_FORMAT_ATTR(mask_opc, mask_opc, "config2:5-8"); +DEFINE_UNCORE_FORMAT_ATTR(mask_vnw, mask_vnw, "config2:3-4"); +DEFINE_UNCORE_FORMAT_ATTR(mask0, mask0, "config2:0-31"); +DEFINE_UNCORE_FORMAT_ATTR(mask1, mask1, "config2:32-63"); static u64 uncore_msr_read_counter(struct intel_uncore_box *box, struct perf_event *event) { @@ -281,7 +301,7 @@ static struct attribute *snbep_uncore_cbox_formats_attr[] = { }; static struct attribute *snbep_uncore_pcu_formats_attr[] = { - &format_attr_event.attr, + &format_attr_event_ext.attr, &format_attr_occ_sel.attr, &format_attr_edge.attr, &format_attr_inv.attr, @@ -301,6 +321,24 @@ static struct attribute *snbep_uncore_qpi_formats_attr[] = { &format_attr_edge.attr, &format_attr_inv.attr, &format_attr_thresh8.attr, + &format_attr_match_rds.attr, + &format_attr_match_rnid30.attr, + &format_attr_match_rnid4.attr, + &format_attr_match_dnid.attr, + &format_attr_match_mc.attr, + &format_attr_match_opc.attr, + &format_attr_match_vnw.attr, + &format_attr_match0.attr, + &format_attr_match1.attr, + &format_attr_mask_rds.attr, + &format_attr_mask_rnid30.attr, + &format_attr_mask_rnid4.attr, + &format_attr_mask_dnid.attr, + &format_attr_mask_mc.attr, + &format_attr_mask_opc.attr, + &format_attr_mask_vnw.attr, + &format_attr_mask0.attr, + &format_attr_mask1.attr, NULL, }; @@ -356,13 +394,16 @@ static struct intel_uncore_ops snbep_uncore_msr_ops = { SNBEP_UNCORE_MSR_OPS_COMMON_INIT(), }; +#define SNBEP_UNCORE_PCI_OPS_COMMON_INIT() \ + .init_box = snbep_uncore_pci_init_box, \ + .disable_box = snbep_uncore_pci_disable_box, \ + .enable_box = snbep_uncore_pci_enable_box, \ + .disable_event = snbep_uncore_pci_disable_event, \ + .read_counter = snbep_uncore_pci_read_counter + static struct intel_uncore_ops snbep_uncore_pci_ops = { - .init_box = snbep_uncore_pci_init_box, - .disable_box = snbep_uncore_pci_disable_box, - .enable_box = snbep_uncore_pci_enable_box, - .disable_event = snbep_uncore_pci_disable_event, - .enable_event = snbep_uncore_pci_enable_event, - .read_counter = snbep_uncore_pci_read_counter, + SNBEP_UNCORE_PCI_OPS_COMMON_INIT(), + .enable_event = snbep_uncore_pci_enable_event, \ }; static struct event_constraint snbep_uncore_cbox_constraints[] = { @@ -726,6 +767,61 @@ static struct intel_uncore_type *snbep_msr_uncores[] = { NULL, }; +enum { + SNBEP_PCI_QPI_PORT0_FILTER, + SNBEP_PCI_QPI_PORT1_FILTER, +}; + +static int snbep_qpi_hw_config(struct intel_uncore_box *box, struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + struct hw_perf_event_extra *reg1 = &hwc->extra_reg; + struct hw_perf_event_extra *reg2 = &hwc->branch_reg; + + if ((hwc->config & SNBEP_PMON_CTL_EV_SEL_MASK) == 0x38) { + reg1->idx = 0; + reg1->reg = SNBEP_Q_Py_PCI_PMON_PKT_MATCH0; + reg1->config = event->attr.config1; + reg2->reg = SNBEP_Q_Py_PCI_PMON_PKT_MASK0; + reg2->config = event->attr.config2; + } + return 0; +} + +static void snbep_qpi_enable_event(struct intel_uncore_box *box, struct perf_event *event) +{ + struct pci_dev *pdev = box->pci_dev; + struct hw_perf_event *hwc = &event->hw; + struct hw_perf_event_extra *reg1 = &hwc->extra_reg; + struct hw_perf_event_extra *reg2 = &hwc->branch_reg; + + if (reg1->idx != EXTRA_REG_NONE) { + int idx = box->pmu->pmu_idx + SNBEP_PCI_QPI_PORT0_FILTER; + struct pci_dev *filter_pdev = extra_pci_dev[box->phys_id][idx]; + WARN_ON_ONCE(!filter_pdev); + if (filter_pdev) { + pci_write_config_dword(filter_pdev, reg1->reg, + (u32)reg1->config); + pci_write_config_dword(filter_pdev, reg1->reg + 4, + (u32)(reg1->config >> 32)); + pci_write_config_dword(filter_pdev, reg2->reg, + (u32)reg2->config); + pci_write_config_dword(filter_pdev, reg2->reg + 4, + (u32)(reg2->config >> 32)); + } + } + + pci_write_config_dword(pdev, hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN); +} + +static struct intel_uncore_ops snbep_uncore_qpi_ops = { + SNBEP_UNCORE_PCI_OPS_COMMON_INIT(), + .enable_event = snbep_qpi_enable_event, + .hw_config = snbep_qpi_hw_config, + .get_constraint = uncore_get_constraint, + .put_constraint = uncore_put_constraint, +}; + #define SNBEP_UNCORE_PCI_COMMON_INIT() \ .perf_ctr = SNBEP_PCI_PMON_CTR0, \ .event_ctl = SNBEP_PCI_PMON_CTL0, \ @@ -755,17 +851,18 @@ static struct intel_uncore_type snbep_uncore_imc = { }; static struct intel_uncore_type snbep_uncore_qpi = { - .name = "qpi", - .num_counters = 4, - .num_boxes = 2, - .perf_ctr_bits = 48, - .perf_ctr = SNBEP_PCI_PMON_CTR0, - .event_ctl = SNBEP_PCI_PMON_CTL0, - .event_mask = SNBEP_QPI_PCI_PMON_RAW_EVENT_MASK, - .box_ctl = SNBEP_PCI_PMON_BOX_CTL, - .ops = &snbep_uncore_pci_ops, - .event_descs = snbep_uncore_qpi_events, - .format_group = &snbep_uncore_qpi_format_group, + .name = "qpi", + .num_counters = 4, + .num_boxes = 2, + .perf_ctr_bits = 48, + .perf_ctr = SNBEP_PCI_PMON_CTR0, + .event_ctl = SNBEP_PCI_PMON_CTL0, + .event_mask = SNBEP_QPI_PCI_PMON_RAW_EVENT_MASK, + .box_ctl = SNBEP_PCI_PMON_BOX_CTL, + .num_shared_regs = 1, + .ops = &snbep_uncore_qpi_ops, + .event_descs = snbep_uncore_qpi_events, + .format_group = &snbep_uncore_qpi_format_group, }; @@ -807,43 +904,53 @@ static struct intel_uncore_type *snbep_pci_uncores[] = { static DEFINE_PCI_DEVICE_TABLE(snbep_uncore_pci_ids) = { { /* Home Agent */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_HA), - .driver_data = SNBEP_PCI_UNCORE_HA, + .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_HA, 0), }, { /* MC Channel 0 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC0), - .driver_data = SNBEP_PCI_UNCORE_IMC, + .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_IMC, 0), }, { /* MC Channel 1 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC1), - .driver_data = SNBEP_PCI_UNCORE_IMC, + .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_IMC, 1), }, { /* MC Channel 2 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC2), - .driver_data = SNBEP_PCI_UNCORE_IMC, + .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_IMC, 2), }, { /* MC Channel 3 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC3), - .driver_data = SNBEP_PCI_UNCORE_IMC, + .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_IMC, 3), }, { /* QPI Port 0 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_QPI0), - .driver_data = SNBEP_PCI_UNCORE_QPI, + .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_QPI, 0), }, { /* QPI Port 1 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_QPI1), - .driver_data = SNBEP_PCI_UNCORE_QPI, + .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_QPI, 1), }, { /* R2PCIe */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R2PCIE), - .driver_data = SNBEP_PCI_UNCORE_R2PCIE, + .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_R2PCIE, 0), }, { /* R3QPI Link 0 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R3QPI0), - .driver_data = SNBEP_PCI_UNCORE_R3QPI, + .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_R3QPI, 0), }, { /* R3QPI Link 1 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R3QPI1), - .driver_data = SNBEP_PCI_UNCORE_R3QPI, + .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_R3QPI, 1), + }, + { /* QPI Port 0 filter */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3c86), + .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV, + SNBEP_PCI_QPI_PORT0_FILTER), + }, + { /* QPI Port 0 filter */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3c96), + .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV, + SNBEP_PCI_QPI_PORT1_FILTER), }, { /* end: all zeroes */ } }; @@ -1256,71 +1363,71 @@ static struct intel_uncore_type *ivt_pci_uncores[] = { static DEFINE_PCI_DEVICE_TABLE(ivt_uncore_pci_ids) = { { /* Home Agent 0 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe30), - .driver_data = IVT_PCI_UNCORE_HA, + .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_HA, 0), }, { /* Home Agent 1 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe38), - .driver_data = IVT_PCI_UNCORE_HA, + .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_HA, 1), }, { /* MC0 Channel 0 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb4), - .driver_data = IVT_PCI_UNCORE_IMC, + .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_IMC, 0), }, { /* MC0 Channel 1 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb5), - .driver_data = IVT_PCI_UNCORE_IMC, + .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_IMC, 1), }, { /* MC0 Channel 3 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb0), - .driver_data = IVT_PCI_UNCORE_IMC, + .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_IMC, 2), }, { /* MC0 Channel 4 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb1), - .driver_data = IVT_PCI_UNCORE_IMC, + .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_IMC, 3), }, { /* MC1 Channel 0 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef4), - .driver_data = IVT_PCI_UNCORE_IMC, + .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_IMC, 4), }, { /* MC1 Channel 1 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef5), - .driver_data = IVT_PCI_UNCORE_IMC, + .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_IMC, 5), }, { /* MC1 Channel 3 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef0), - .driver_data = IVT_PCI_UNCORE_IMC, + .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_IMC, 6), }, { /* MC1 Channel 4 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef1), - .driver_data = IVT_PCI_UNCORE_IMC, + .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_IMC, 7), }, { /* QPI0 Port 0 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe32), - .driver_data = IVT_PCI_UNCORE_QPI, + .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_QPI, 0), }, { /* QPI0 Port 1 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe33), - .driver_data = IVT_PCI_UNCORE_QPI, + .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_QPI, 1), }, { /* QPI1 Port 2 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe3a), - .driver_data = IVT_PCI_UNCORE_QPI, + .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_QPI, 2), }, { /* R2PCIe */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe34), - .driver_data = IVT_PCI_UNCORE_R2PCIE, + .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_R2PCIE, 0), }, { /* R3QPI0 Link 0 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe36), - .driver_data = IVT_PCI_UNCORE_R3QPI, + .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_R3QPI, 0), }, { /* R3QPI0 Link 1 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe37), - .driver_data = IVT_PCI_UNCORE_R3QPI, + .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_R3QPI, 1), }, { /* R3QPI1 Link 2 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe3e), - .driver_data = IVT_PCI_UNCORE_R3QPI, + .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_R3QPI, 2), }, { /* end: all zeroes */ } }; @@ -2606,7 +2713,7 @@ struct intel_uncore_box *uncore_alloc_box(struct intel_uncore_type *type, int cp size = sizeof(*box) + type->num_shared_regs * sizeof(struct intel_uncore_extra_reg); - box = kmalloc_node(size, GFP_KERNEL | __GFP_ZERO, cpu_to_node(cpu)); + box = kzalloc_node(size, GFP_KERNEL, cpu_to_node(cpu)); if (!box) return NULL; @@ -3167,16 +3274,24 @@ static bool pcidrv_registered; /* * add a pci uncore device */ -static int uncore_pci_add(struct intel_uncore_type *type, struct pci_dev *pdev) +static int uncore_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct intel_uncore_pmu *pmu; struct intel_uncore_box *box; - int i, phys_id; + struct intel_uncore_type *type; + int phys_id; phys_id = pcibus_to_physid[pdev->bus->number]; if (phys_id < 0) return -ENODEV; + if (UNCORE_PCI_DEV_TYPE(id->driver_data) == UNCORE_EXTRA_PCI_DEV) { + extra_pci_dev[phys_id][UNCORE_PCI_DEV_IDX(id->driver_data)] = pdev; + pci_set_drvdata(pdev, NULL); + return 0; + } + + type = pci_uncores[UNCORE_PCI_DEV_TYPE(id->driver_data)]; box = uncore_alloc_box(type, 0); if (!box) return -ENOMEM; @@ -3185,21 +3300,11 @@ static int uncore_pci_add(struct intel_uncore_type *type, struct pci_dev *pdev) * for performance monitoring unit with multiple boxes, * each box has a different function id. */ - for (i = 0; i < type->num_boxes; i++) { - pmu = &type->pmus[i]; - if (pmu->func_id == pdev->devfn) - break; - if (pmu->func_id < 0) { - pmu->func_id = pdev->devfn; - break; - } - pmu = NULL; - } - - if (!pmu) { - kfree(box); - return -EINVAL; - } + pmu = &type->pmus[UNCORE_PCI_DEV_IDX(id->driver_data)]; + if (pmu->func_id < 0) + pmu->func_id = pdev->devfn; + else + WARN_ON_ONCE(pmu->func_id != pdev->devfn); box->phys_id = phys_id; box->pci_dev = pdev; @@ -3217,9 +3322,22 @@ static int uncore_pci_add(struct intel_uncore_type *type, struct pci_dev *pdev) static void uncore_pci_remove(struct pci_dev *pdev) { struct intel_uncore_box *box = pci_get_drvdata(pdev); - struct intel_uncore_pmu *pmu = box->pmu; - int cpu, phys_id = pcibus_to_physid[pdev->bus->number]; + struct intel_uncore_pmu *pmu; + int i, cpu, phys_id = pcibus_to_physid[pdev->bus->number]; + box = pci_get_drvdata(pdev); + if (!box) { + for (i = 0; i < UNCORE_EXTRA_PCI_DEV_MAX; i++) { + if (extra_pci_dev[phys_id][i] == pdev) { + extra_pci_dev[phys_id][i] = NULL; + break; + } + } + WARN_ON_ONCE(i >= UNCORE_EXTRA_PCI_DEV_MAX); + return; + } + + pmu = box->pmu; if (WARN_ON_ONCE(phys_id != box->phys_id)) return; @@ -3240,12 +3358,6 @@ static void uncore_pci_remove(struct pci_dev *pdev) kfree(box); } -static int uncore_pci_probe(struct pci_dev *pdev, - const struct pci_device_id *id) -{ - return uncore_pci_add(pci_uncores[id->driver_data], pdev); -} - static int __init uncore_pci_init(void) { int ret; diff --git a/arch/x86/kernel/cpu/perf_event_intel_uncore.h b/arch/x86/kernel/cpu/perf_event_intel_uncore.h index 47b3d00c9d89..a80ab71a883d 100644 --- a/arch/x86/kernel/cpu/perf_event_intel_uncore.h +++ b/arch/x86/kernel/cpu/perf_event_intel_uncore.h @@ -12,6 +12,15 @@ #define UNCORE_PMC_IDX_FIXED UNCORE_PMC_IDX_MAX_GENERIC #define UNCORE_PMC_IDX_MAX (UNCORE_PMC_IDX_FIXED + 1) +#define UNCORE_PCI_DEV_DATA(type, idx) ((type << 8) | idx) +#define UNCORE_PCI_DEV_TYPE(data) ((data >> 8) & 0xff) +#define UNCORE_PCI_DEV_IDX(data) (data & 0xff) +#define UNCORE_EXTRA_PCI_DEV 0xff +#define UNCORE_EXTRA_PCI_DEV_MAX 2 + +/* support up to 8 sockets */ +#define UNCORE_SOCKET_MAX 8 + #define UNCORE_EVENT_CONSTRAINT(c, n) EVENT_CONSTRAINT(c, n, 0xff) /* SNB event control */ @@ -108,6 +117,7 @@ (SNBEP_PMON_CTL_EV_SEL_MASK | \ SNBEP_PCU_MSR_PMON_CTL_OCC_SEL_MASK | \ SNBEP_PMON_CTL_EDGE_DET | \ + SNBEP_PMON_CTL_EV_SEL_EXT | \ SNBEP_PMON_CTL_INVERT | \ SNBEP_PCU_MSR_PMON_CTL_TRESH_MASK | \ SNBEP_PCU_MSR_PMON_CTL_OCC_INVERT | \ diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c index 2889b3d43882..460f5d9ceebb 100644 --- a/arch/x86/kernel/jump_label.c +++ b/arch/x86/kernel/jump_label.c @@ -37,7 +37,19 @@ static void __jump_label_transform(struct jump_entry *entry, } else memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE); - (*poker)((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE); + /* + * Make text_poke_bp() a default fallback poker. + * + * At the time the change is being done, just ignore whether we + * are doing nop -> jump or jump -> nop transition, and assume + * always nop being the 'currently valid' instruction + * + */ + if (poker) + (*poker)((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE); + else + text_poke_bp((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE, + (void *)entry->code + JUMP_LABEL_NOP_SIZE); } void arch_jump_label_transform(struct jump_entry *entry, @@ -45,7 +57,7 @@ void arch_jump_label_transform(struct jump_entry *entry, { get_online_cpus(); mutex_lock(&text_mutex); - __jump_label_transform(entry, type, text_poke_smp); + __jump_label_transform(entry, type, NULL); mutex_unlock(&text_mutex); put_online_cpus(); } diff --git a/arch/x86/kernel/kprobes/common.h b/arch/x86/kernel/kprobes/common.h index 2e9d4b5af036..c6ee63f927ab 100644 --- a/arch/x86/kernel/kprobes/common.h +++ b/arch/x86/kernel/kprobes/common.h @@ -82,14 +82,9 @@ extern void synthesize_reljump(void *from, void *to); extern void synthesize_relcall(void *from, void *to); #ifdef CONFIG_OPTPROBES -extern int arch_init_optprobes(void); extern int setup_detour_execution(struct kprobe *p, struct pt_regs *regs, int reenter); extern unsigned long __recover_optprobed_insn(kprobe_opcode_t *buf, unsigned long addr); #else /* !CONFIG_OPTPROBES */ -static inline int arch_init_optprobes(void) -{ - return 0; -} static inline int setup_detour_execution(struct kprobe *p, struct pt_regs *regs, int reenter) { return 0; diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c index 211bce445522..cd49b2c96d32 100644 --- a/arch/x86/kernel/kprobes/core.c +++ b/arch/x86/kernel/kprobes/core.c @@ -1068,7 +1068,7 @@ int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) int __init arch_init_kprobes(void) { - return arch_init_optprobes(); + return 0; } int __kprobes arch_trampoline_kprobe(struct kprobe *p) diff --git a/arch/x86/kernel/kprobes/opt.c b/arch/x86/kernel/kprobes/opt.c index 76dc6f095724..d71e99439376 100644 --- a/arch/x86/kernel/kprobes/opt.c +++ b/arch/x86/kernel/kprobes/opt.c @@ -371,31 +371,6 @@ int __kprobes arch_prepare_optimized_kprobe(struct optimized_kprobe *op) return 0; } -#define MAX_OPTIMIZE_PROBES 256 -static struct text_poke_param *jump_poke_params; -static struct jump_poke_buffer { - u8 buf[RELATIVEJUMP_SIZE]; -} *jump_poke_bufs; - -static void __kprobes setup_optimize_kprobe(struct text_poke_param *tprm, - u8 *insn_buf, - struct optimized_kprobe *op) -{ - s32 rel = (s32)((long)op->optinsn.insn - - ((long)op->kp.addr + RELATIVEJUMP_SIZE)); - - /* Backup instructions which will be replaced by jump address */ - memcpy(op->optinsn.copied_insn, op->kp.addr + INT3_SIZE, - RELATIVE_ADDR_SIZE); - - insn_buf[0] = RELATIVEJUMP_OPCODE; - *(s32 *)(&insn_buf[1]) = rel; - - tprm->addr = op->kp.addr; - tprm->opcode = insn_buf; - tprm->len = RELATIVEJUMP_SIZE; -} - /* * Replace breakpoints (int3) with relative jumps. * Caller must call with locking kprobe_mutex and text_mutex. @@ -403,37 +378,38 @@ static void __kprobes setup_optimize_kprobe(struct text_poke_param *tprm, void __kprobes arch_optimize_kprobes(struct list_head *oplist) { struct optimized_kprobe *op, *tmp; - int c = 0; + u8 insn_buf[RELATIVEJUMP_SIZE]; list_for_each_entry_safe(op, tmp, oplist, list) { + s32 rel = (s32)((long)op->optinsn.insn - + ((long)op->kp.addr + RELATIVEJUMP_SIZE)); + WARN_ON(kprobe_disabled(&op->kp)); - /* Setup param */ - setup_optimize_kprobe(&jump_poke_params[c], - jump_poke_bufs[c].buf, op); + + /* Backup instructions which will be replaced by jump address */ + memcpy(op->optinsn.copied_insn, op->kp.addr + INT3_SIZE, + RELATIVE_ADDR_SIZE); + + insn_buf[0] = RELATIVEJUMP_OPCODE; + *(s32 *)(&insn_buf[1]) = rel; + + text_poke_bp(op->kp.addr, insn_buf, RELATIVEJUMP_SIZE, + op->optinsn.insn); + list_del_init(&op->list); - if (++c >= MAX_OPTIMIZE_PROBES) - break; } - - /* - * text_poke_smp doesn't support NMI/MCE code modifying. - * However, since kprobes itself also doesn't support NMI/MCE - * code probing, it's not a problem. - */ - text_poke_smp_batch(jump_poke_params, c); } -static void __kprobes setup_unoptimize_kprobe(struct text_poke_param *tprm, - u8 *insn_buf, - struct optimized_kprobe *op) +/* Replace a relative jump with a breakpoint (int3). */ +void __kprobes arch_unoptimize_kprobe(struct optimized_kprobe *op) { + u8 insn_buf[RELATIVEJUMP_SIZE]; + /* Set int3 to first byte for kprobes */ insn_buf[0] = BREAKPOINT_INSTRUCTION; memcpy(insn_buf + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE); - - tprm->addr = op->kp.addr; - tprm->opcode = insn_buf; - tprm->len = RELATIVEJUMP_SIZE; + text_poke_bp(op->kp.addr, insn_buf, RELATIVEJUMP_SIZE, + op->optinsn.insn); } /* @@ -444,34 +420,11 @@ extern void arch_unoptimize_kprobes(struct list_head *oplist, struct list_head *done_list) { struct optimized_kprobe *op, *tmp; - int c = 0; list_for_each_entry_safe(op, tmp, oplist, list) { - /* Setup param */ - setup_unoptimize_kprobe(&jump_poke_params[c], - jump_poke_bufs[c].buf, op); + arch_unoptimize_kprobe(op); list_move(&op->list, done_list); - if (++c >= MAX_OPTIMIZE_PROBES) - break; } - - /* - * text_poke_smp doesn't support NMI/MCE code modifying. - * However, since kprobes itself also doesn't support NMI/MCE - * code probing, it's not a problem. - */ - text_poke_smp_batch(jump_poke_params, c); -} - -/* Replace a relative jump with a breakpoint (int3). */ -void __kprobes arch_unoptimize_kprobe(struct optimized_kprobe *op) -{ - u8 buf[RELATIVEJUMP_SIZE]; - - /* Set int3 to first byte for kprobes */ - buf[0] = BREAKPOINT_INSTRUCTION; - memcpy(buf + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE); - text_poke_smp(op->kp.addr, buf, RELATIVEJUMP_SIZE); } int __kprobes @@ -491,22 +444,3 @@ setup_detour_execution(struct kprobe *p, struct pt_regs *regs, int reenter) } return 0; } - -int __kprobes arch_init_optprobes(void) -{ - /* Allocate code buffer and parameter array */ - jump_poke_bufs = kmalloc(sizeof(struct jump_poke_buffer) * - MAX_OPTIMIZE_PROBES, GFP_KERNEL); - if (!jump_poke_bufs) - return -ENOMEM; - - jump_poke_params = kmalloc(sizeof(struct text_poke_param) * - MAX_OPTIMIZE_PROBES, GFP_KERNEL); - if (!jump_poke_params) { - kfree(jump_poke_bufs); - jump_poke_bufs = NULL; - return -ENOMEM; - } - - return 0; -} diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c index addf7b58f4e8..91a4496db434 100644 --- a/arch/x86/kernel/tboot.c +++ b/arch/x86/kernel/tboot.c @@ -301,6 +301,15 @@ static int tboot_sleep(u8 sleep_state, u32 pm1a_control, u32 pm1b_control) return 0; } +static int tboot_extended_sleep(u8 sleep_state, u32 val_a, u32 val_b) +{ + if (!tboot_enabled()) + return 0; + + pr_warning("tboot is not able to suspend on platforms with reduced hardware sleep (ACPIv5)"); + return -ENODEV; +} + static atomic_t ap_wfs_count; static int tboot_wait_for_aps(int num_aps) @@ -422,6 +431,7 @@ static __init int tboot_late_init(void) #endif acpi_os_set_prepare_sleep(&tboot_sleep); + acpi_os_set_prepare_extended_sleep(&tboot_extended_sleep); return 0; } diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index 1b23a1c92746..8c8093b146ca 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -58,6 +58,7 @@ #include <asm/mce.h> #include <asm/fixmap.h> #include <asm/mach_traps.h> +#include <asm/alternative.h> #ifdef CONFIG_X86_64 #include <asm/x86_init.h> @@ -327,6 +328,9 @@ dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_co ftrace_int3_handler(regs)) return; #endif + if (poke_int3_handler(regs)) + return; + prev_state = exception_enter(); #ifdef CONFIG_KGDB_LOW_LEVEL_TRAP if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP, diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 6ff49247edf8..930e5d48f560 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -89,6 +89,12 @@ int check_tsc_unstable(void) } EXPORT_SYMBOL_GPL(check_tsc_unstable); +int check_tsc_disabled(void) +{ + return tsc_disabled; +} +EXPORT_SYMBOL_GPL(check_tsc_disabled); + #ifdef CONFIG_X86_TSC int __init notsc_setup(char *str) { diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c index 2ec29ac78ae6..04664cdb7fda 100644 --- a/arch/x86/mm/init.c +++ b/arch/x86/mm/init.c @@ -78,8 +78,8 @@ __ref void *alloc_low_pages(unsigned int num) return __va(pfn << PAGE_SHIFT); } -/* need 4 4k for initial PMD_SIZE, 4k for 0-ISA_END_ADDRESS */ -#define INIT_PGT_BUF_SIZE (5 * PAGE_SIZE) +/* need 3 4k for initial PMD_SIZE, 3 4k for 0-ISA_END_ADDRESS */ +#define INIT_PGT_BUF_SIZE (6 * PAGE_SIZE) RESERVE_BRK(early_pgt_alloc, INIT_PGT_BUF_SIZE); void __init early_alloc_pgt_buf(void) { diff --git a/arch/x86/mm/srat.c b/arch/x86/mm/srat.c index cdd0da9dd530..266ca912f62e 100644 --- a/arch/x86/mm/srat.c +++ b/arch/x86/mm/srat.c @@ -146,6 +146,7 @@ int __init acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma) { u64 start, end; + u32 hotpluggable; int node, pxm; if (srat_disabled()) @@ -154,7 +155,8 @@ acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma) goto out_err_bad_srat; if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0) goto out_err; - if ((ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) && !save_add_info()) + hotpluggable = ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE; + if (hotpluggable && !save_add_info()) goto out_err; start = ma->base_address; @@ -174,9 +176,10 @@ acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma) node_set(node, numa_nodes_parsed); - printk(KERN_INFO "SRAT: Node %u PXM %u [mem %#010Lx-%#010Lx]\n", - node, pxm, - (unsigned long long) start, (unsigned long long) end - 1); + pr_info("SRAT: Node %u PXM %u [mem %#010Lx-%#010Lx]%s\n", + node, pxm, + (unsigned long long) start, (unsigned long long) end - 1, + hotpluggable ? " hotplug" : ""); return 0; out_err_bad_srat: diff --git a/arch/x86/pci/acpi.c b/arch/x86/pci/acpi.c index d641897a1f4e..b30e937689d6 100644 --- a/arch/x86/pci/acpi.c +++ b/arch/x86/pci/acpi.c @@ -568,13 +568,8 @@ struct pci_bus *pci_acpi_scan_root(struct acpi_pci_root *root) */ if (bus) { struct pci_bus *child; - list_for_each_entry(child, &bus->children, node) { - struct pci_dev *self = child->self; - if (!self) - continue; - - pcie_bus_configure_settings(child, self->pcie_mpss); - } + list_for_each_entry(child, &bus->children, node) + pcie_bus_configure_settings(child); } if (bus && node != -1) { diff --git a/arch/x86/pci/i386.c b/arch/x86/pci/i386.c index 94919e307f8e..db6b1ab43255 100644 --- a/arch/x86/pci/i386.c +++ b/arch/x86/pci/i386.c @@ -210,6 +210,8 @@ static void pcibios_allocate_bridge_resources(struct pci_dev *dev) r = &dev->resource[idx]; if (!r->flags) continue; + if (r->parent) /* Already allocated */ + continue; if (!r->start || pci_claim_resource(dev, idx) < 0) { /* * Something is wrong with the region. @@ -318,6 +320,8 @@ static void pcibios_allocate_dev_rom_resource(struct pci_dev *dev) r = &dev->resource[PCI_ROM_RESOURCE]; if (!r->flags || !r->start) return; + if (r->parent) /* Already allocated */ + return; if (pci_claim_resource(dev, PCI_ROM_RESOURCE) < 0) { r->end -= r->start; diff --git a/arch/x86/pci/mmconfig-shared.c b/arch/x86/pci/mmconfig-shared.c index 082e88129712..5596c7bdd327 100644 --- a/arch/x86/pci/mmconfig-shared.c +++ b/arch/x86/pci/mmconfig-shared.c @@ -700,7 +700,7 @@ int pci_mmconfig_insert(struct device *dev, u16 seg, u8 start, u8 end, if (!(pci_probe & PCI_PROBE_MMCONF) || pci_mmcfg_arch_init_failed) return -ENODEV; - if (start > end) + if (start > end || !addr) return -EINVAL; mutex_lock(&pci_mmcfg_lock); @@ -716,11 +716,6 @@ int pci_mmconfig_insert(struct device *dev, u16 seg, u8 start, u8 end, return -EEXIST; } - if (!addr) { - mutex_unlock(&pci_mmcfg_lock); - return -EINVAL; - } - rc = -EBUSY; cfg = pci_mmconfig_alloc(seg, start, end, addr); if (cfg == NULL) { diff --git a/arch/x86/pci/mrst.c b/arch/x86/pci/mrst.c index 6eb18c42a28a..903fded50786 100644 --- a/arch/x86/pci/mrst.c +++ b/arch/x86/pci/mrst.c @@ -23,11 +23,11 @@ #include <linux/ioport.h> #include <linux/init.h> #include <linux/dmi.h> +#include <linux/acpi.h> +#include <linux/io.h> +#include <linux/smp.h> -#include <asm/acpi.h> #include <asm/segment.h> -#include <asm/io.h> -#include <asm/smp.h> #include <asm/pci_x86.h> #include <asm/hw_irq.h> #include <asm/io_apic.h> @@ -43,7 +43,7 @@ #define PCI_FIXED_BAR_4_SIZE 0x14 #define PCI_FIXED_BAR_5_SIZE 0x1c -static int pci_soc_mode = 0; +static int pci_soc_mode; /** * fixed_bar_cap - return the offset of the fixed BAR cap if found @@ -141,7 +141,8 @@ static int pci_device_update_fixed(struct pci_bus *bus, unsigned int devfn, */ static bool type1_access_ok(unsigned int bus, unsigned int devfn, int reg) { - /* This is a workaround for A0 LNC bug where PCI status register does + /* + * This is a workaround for A0 LNC bug where PCI status register does * not have new CAP bit set. can not be written by SW either. * * PCI header type in real LNC indicates a single function device, this @@ -154,7 +155,7 @@ static bool type1_access_ok(unsigned int bus, unsigned int devfn, int reg) || devfn == PCI_DEVFN(0, 0) || devfn == PCI_DEVFN(3, 0))) return 1; - return 0; /* langwell on others */ + return 0; /* Langwell on others */ } static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, @@ -172,7 +173,8 @@ static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, { int offset; - /* On MRST, there is no PCI ROM BAR, this will cause a subsequent read + /* + * On MRST, there is no PCI ROM BAR, this will cause a subsequent read * to ROM BAR return 0 then being ignored. */ if (where == PCI_ROM_ADDRESS) @@ -210,7 +212,8 @@ static int mrst_pci_irq_enable(struct pci_dev *dev) pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &pin); - /* MRST only have IOAPIC, the PCI irq lines are 1:1 mapped to + /* + * MRST only have IOAPIC, the PCI irq lines are 1:1 mapped to * IOAPIC RTE entries, so we just enable RTE for the device. */ irq_attr.ioapic = mp_find_ioapic(dev->irq); @@ -235,7 +238,7 @@ struct pci_ops pci_mrst_ops = { */ int __init pci_mrst_init(void) { - printk(KERN_INFO "Intel MID platform detected, using MID PCI ops\n"); + pr_info("Intel MID platform detected, using MID PCI ops\n"); pci_mmcfg_late_init(); pcibios_enable_irq = mrst_pci_irq_enable; pci_root_ops = pci_mrst_ops; @@ -244,17 +247,21 @@ int __init pci_mrst_init(void) return 1; } -/* Langwell devices are not true pci devices, they are not subject to 10 ms - * d3 to d0 delay required by pci spec. +/* + * Langwell devices are not true PCI devices; they are not subject to 10 ms + * d3 to d0 delay required by PCI spec. */ static void pci_d3delay_fixup(struct pci_dev *dev) { - /* PCI fixups are effectively decided compile time. If we have a dual - SoC/non-SoC kernel we don't want to mangle d3 on non SoC devices */ - if (!pci_soc_mode) - return; - /* true pci devices in lincroft should allow type 1 access, the rest - * are langwell fake pci devices. + /* + * PCI fixups are effectively decided compile time. If we have a dual + * SoC/non-SoC kernel we don't want to mangle d3 on non-SoC devices. + */ + if (!pci_soc_mode) + return; + /* + * True PCI devices in Lincroft should allow type 1 access, the rest + * are Langwell fake PCI devices. */ if (type1_access_ok(dev->bus->number, dev->devfn, PCI_DEVICE_ID)) return; |